diff --git a/.changeset/violet-buses-tease.md b/.changeset/violet-buses-tease.md new file mode 100644 index 000000000..191cff776 --- /dev/null +++ b/.changeset/violet-buses-tease.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Send the CLI version header on all API requests so deployments are attributable to a CLI version diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c1155d377..2b1fba869 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -940,6 +940,10 @@ const EnvironmentSchema = z DISABLE_HTTP_INSTRUMENTATION: BoolEnv.default(false), INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(), + + // Second trace exporter receiving only `deployment.*` spans; they still flow to the main one + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_ENABLED: z.string().default("0"), diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 5be291bae..184fa996d 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -42,7 +42,9 @@ export async function action({ request, params }: ActionFunctionArgs) { const service = new InitializeDeploymentService(); try { - const result = await service.call(authenticatedEnv, body.data); + const result = await service.call(authenticatedEnv, body.data, { + cliVersion: parseCliVersionHeader(request), + }); const { deployment, imageRef } = result; const responseBody: InitializeDeploymentResponseBody = { @@ -75,6 +77,14 @@ export async function action({ request, params }: ActionFunctionArgs) { } } +// Client-controlled and persisted, so cap what we accept +const CLI_VERSION_MAX_LENGTH = 128; + +function parseCliVersionHeader(request: Request): string | undefined { + const value = request.headers.get("x-trigger-cli-version"); + return value && value.length <= CLI_VERSION_MAX_LENGTH ? value : undefined; +} + export const loader = createLoaderApiRoute( { searchParams: ApiDeploymentListSearchParams, diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts new file mode 100644 index 000000000..2dd0725f8 --- /dev/null +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -0,0 +1,116 @@ +import { BuildServerMetadata } from "@trigger.dev/core/v3"; + +/** + * Attribute names for the `deployment.finished` and `deployment.initialized` + * telemetry events (emitted by services/recordDeploymentFinished.server.ts). + * This module is the single owner of these names — external queries, + * dashboards, and monitors reference them, so treat renames as breaking. + * + * Query gotchas: dedup with `arg_max(_time, *) by deployment.id` (job retries + * can double-emit); the span's `_time` is the deployment's createdAt, so a + * TIMED_OUT event lands backdated by up to the full deploy timeout — monitor + * windows must exceed it; phase durations are omitted (not zero) when a + * boundary timestamp is missing, and `total_ms` excludes local-bundle's + * pre-init client work (esbuild + upload) until the CLI reports timings. + */ +export const DeploymentTelemetryAttributes = { + ORG_ID: "$trigger.org.id", + PROJECT_ID: "$trigger.project.id", + // Project external ref ("proj_…") + PROJECT_REF: "$trigger.project.ref", + ENV_ID: "$trigger.env.id", + // PRODUCTION / STAGING / PREVIEW / DEVELOPMENT + ENV_TYPE: "$trigger.env.type", + // Deployment friendly id — the dedup key + DEPLOYMENT_ID: "deployment.id", + VERSION: "deployment.version", + // finished: terminal status; initialized: initial status (PENDING/BUILDING) + STATUS: "deployment.status", + // status === DEPLOYED; CANCELED is excluded from failure rates + SUCCESS: "deployment.success", + // depot / native / native_local_bundle (see deriveBuildPath) + BUILD_PATH: "deployment.build_path", + // V1 / MANAGED (run engine) + WORKER_TYPE: "deployment.worker_type", + RUNTIME: "deployment.runtime", + // Set at indexing; null for pre-index failures + RUNTIME_VERSION: "deployment.runtime_version", + // From x-trigger-cli-version at init; null for pre-column history + CLI_VERSION: "deployment.cli_version", + TRIGGERED_VIA: "deployment.triggered_via", + COMMIT_SHA: "deployment.commit_sha", + // error.* only on FAILED/TIMED_OUT; CANCELED uses canceled_reason + ERROR_NAME: "deployment.error.name", + ERROR_MESSAGE: "deployment.error.message", + CANCELED_REASON: "deployment.canceled_reason", + // createdAt → terminal (also the span's own duration) + DURATION_TOTAL_MS: "deployment.duration.total_ms", + // createdAt → startedAt; ≈0 when created directly in BUILDING (depot) + DURATION_QUEUE_MS: "deployment.duration.queue_ms", + // startedAt → installedAt; build-server paths only (depot never sets it) + DURATION_INSTALL_MS: "deployment.duration.install_ms", + // (installedAt ?? startedAt) → builtAt + DURATION_BUILDING_MS: "deployment.duration.building_ms", + // builtAt → terminal; for depot dominated by the server-side registry push + DURATION_DEPLOYING_MS: "deployment.duration.deploying_ms", +} as const; + +export type DeploymentBuildPath = "native_local_bundle" | "native" | "depot"; + +/** + * Everything that is not a native-build-server deployment falls into the depot + * bucket, including rare `--local-build` deploys (their flag is not persisted). + * `externalBuildData` is NOT a usable depot signal: init writes a placeholder + * for every path. + */ +export function deriveBuildPath(buildServerMetadata: unknown): DeploymentBuildPath { + const metadata = BuildServerMetadata.safeParse(buildServerMetadata); + + if (metadata.success && metadata.data.isNativeBuild) { + return metadata.data.fromBundle ? "native_local_bundle" : "native"; + } + + return "depot"; +} + +export type DeploymentTimestamps = { + createdAt: Date; + startedAt?: Date | null; + installedAt?: Date | null; + builtAt?: Date | null; +}; + +export type DeploymentDurations = { + totalMs: number; + queueMs?: number; + installMs?: number; + buildingMs?: number; + deployingMs?: number; +}; + +/** + * Timestamp chains are path-shaped (e.g. depot never sets installedAt), so + * each phase is derived only when both of its boundary timestamps exist and + * are ordered. + */ +export function deriveDeploymentDurations( + timestamps: DeploymentTimestamps, + terminalAt: Date +): DeploymentDurations { + const { createdAt, startedAt, installedAt, builtAt } = timestamps; + const buildingFrom = installedAt ?? startedAt; + + return { + totalMs: Math.max(terminalAt.getTime() - createdAt.getTime(), 0), + queueMs: msBetween(createdAt, startedAt), + installMs: msBetween(startedAt, installedAt), + buildingMs: msBetween(buildingFrom, builtAt), + deployingMs: msBetween(builtAt, terminalAt), + }; +} + +function msBetween(from?: Date | null, to?: Date | null): number | undefined { + if (!from || !to) return undefined; + const ms = to.getTime() - from.getTime(); + return ms >= 0 ? ms : undefined; +} diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index d09707a0e..7215c09c3 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -18,7 +18,7 @@ import { } from "./createBackgroundWorker.server"; import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { webhookPrisma } from "~/db.server"; @@ -298,6 +298,12 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { error: Error, environment: AuthenticatedEnvironment ) { + const failedAt = new Date(); + const errorData = { + name: error.name, + message: error.message, + }; + // Guarded BUILDING → FAILED transition, symmetric with the BUILDING → DEPLOYING // transition in `call()`. With idempotent retries, two attempts can run side-by-side; // without the predicate, one attempt's failure could downgrade the deployment after @@ -309,11 +315,8 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { }, data: { status: "FAILED", - failedAt: new Date(), - errorData: { - name: error.name, - message: error.message, - }, + failedAt, + errorData, buildEnvVars: Prisma.DbNull, }, }); @@ -332,13 +335,16 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { // BUILDING → DEPLOYING transition. await TimeoutDeploymentService.dequeue(deployment.id, this._prisma); - recordDeploymentOutcome({ + recordDeploymentFinished({ status: "FAILED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: environment.organizationId, - projectId: environment.projectId, - environmentId: environment.id, - environmentType: environment.type, + deployment: { ...deployment, status: "FAILED", failedAt, errorData }, + environment: { + organizationId: environment.organizationId, + projectId: environment.projectId, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, reason: error.message, }); } diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 7a891ae4f..e8ac53cc4 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -9,6 +9,7 @@ import { type DeploymentEvent, } from "@trigger.dev/core/v3"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { createRemoteImageBuild } from "../remoteImageBuilder.server"; import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server"; @@ -195,10 +196,8 @@ export class DeploymentService extends BaseService { friendlyId: string, data?: Partial> ) { - const validateDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } + const validateDeployment = >( + deployment: T ) => { if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) { logger.warn("Attempted cancelling deployment in a final state", { @@ -210,11 +209,7 @@ export class DeploymentService extends BaseService { return okAsync(deployment); }; - const cancelDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } - ) => + const cancelDeployment = >(deployment: T) => fromPromise( this._prisma.workerDeployment.updateMany({ where: { @@ -250,7 +245,25 @@ export class DeploymentService extends BaseService { return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) - .andThen(({ deployment }) => + .andTee(({ deployment }) => + recordDeploymentFinished({ + status: "CANCELED", + deployment: { + ...deployment, + status: "CANCELED", + canceledAt: new Date(), + canceledReason: data?.canceledReason ?? null, + }, + environment: { + organizationId: deployment.environment.project.organizationId, + projectId: deployment.environment.project.id, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environment.id, + environmentType: deployment.environment.type, + }, + }) + ) + .andTee(({ deployment }) => this.appendToEventLog(deployment.environment.project, deployment, [ { type: "finalized", @@ -259,14 +272,11 @@ export class DeploymentService extends BaseService { message: data?.canceledReason ?? undefined, }, }, - ]) - .orElse((error) => { - logger.error("Failed to append event to deployment event log", { error }); - return okAsync(deployment); - }) - .map(() => deployment) + ]).orTee((error) => { + logger.error("Failed to append event to deployment event log", { error }); + }) ) - .andThen(deleteTimeout) + .andThen(({ deployment }) => deleteTimeout(deployment)) .map(() => undefined); } @@ -484,6 +494,23 @@ export class DeploymentService extends BaseService { select: { status: true, id: true, + friendlyId: true, + version: true, + type: true, + createdAt: true, + startedAt: true, + installedAt: true, + builtAt: true, + deployedAt: true, + failedAt: true, + canceledAt: true, + canceledReason: true, + errorData: true, + runtime: true, + runtimeVersion: true, + cliVersion: true, + triggeredVia: true, + commitSHA: true, buildServerMetadata: true, imageReference: true, shortCode: true, @@ -491,6 +518,8 @@ export class DeploymentService extends BaseService { include: { project: { select: { + id: true, + organizationId: true, externalRef: true, }, }, diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index cb5c622b7..534158308 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -1,11 +1,11 @@ import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { BaseService } from "./baseService.server"; import { logger } from "~/services/logger.server"; -import { Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { boundedIn, Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [ "CANCELED", @@ -41,25 +41,53 @@ export class FailDeploymentService extends BaseService { return; } - const failedDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + + // Guarded: a concurrent terminal transition can win after the check above + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: { notIn: boundedIn(FINAL_DEPLOYMENT_STATUSES) }, }, data: { status: "FAILED", - failedAt: new Date(), + failedAt, errorData: params.error, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment reached a final state concurrently, skipping fail", { + id: deployment.id, + friendlyId, + }); + return; + } + + // Re-read: the row can gain phase timestamps between the read and the guarded write + const failedDeployment = await this._prisma.workerDeployment.findFirst({ + where: { id: deployment.id }, + }); + + if (!failedDeployment) { + logger.error("Worker deployment disappeared after fail transition", { + id: deployment.id, + friendlyId, + }); + return; + } + + recordDeploymentFinished({ status: "FAILED", - deploymentFriendlyId: friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: failedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + projectId: authenticatedEnv.projectId, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, reason: params.error.message, }); diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index 51f5b1e37..3ee7a1beb 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -10,7 +10,7 @@ import { projectPubSub } from "./projectPubSub.server"; import { FailDeploymentService } from "./failDeployment.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { engine } from "../runEngine.server"; import { tryCatch } from "@trigger.dev/core"; import { externalDeploymentCacheInstance } from "~/services/externalDeploymentCacheInstance.server"; @@ -66,28 +66,51 @@ export class FinalizeDeploymentService extends BaseService { } const imageDigest = validatedImageDigest(body.imageDigest); + const deployedAt = new Date(); + const imageReference = imageDigest + ? `${deployment.imageReference}@${imageDigest}` + : deployment.imageReference; - // Link the deployment with the background worker - const finalizedDeployment = await this._prisma.workerDeployment.update({ + // Guarded: stops a concurrent transition (e.g. a late timeout) from double-committing + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: "DEPLOYING", }, data: { status: "DEPLOYED", - deployedAt: new Date(), + deployedAt, // Only add the digest, if any - imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined, + imageReference: imageDigest ? imageReference : undefined, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment left DEPLOYING concurrently, skipping finalize", { + id: deployment.id, + }); + throw new ServiceValidationError("Worker deployment is not in DEPLOYING status"); + } + + const finalizedDeployment = { + ...deployment, + status: "DEPLOYED" as const, + deployedAt, + imageReference, + buildEnvVars: null, + }; + + recordDeploymentFinished({ status: "DEPLOYED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: finalizedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + projectId: authenticatedEnv.projectId, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, }); const deploymentService = new DeploymentService(); diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index ee55d8bd8..56cf50e68 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -16,6 +16,7 @@ import { getDeploymentImageRef } from "../getDeploymentImageRef.server"; import { tryCatch } from "@trigger.dev/core"; import { getRegistryConfig } from "../registryConfig.server"; import { DeploymentService } from "./deployment.server"; +import { recordDeploymentInitialized } from "./recordDeploymentFinished.server"; import { createDeploymentWithNextVersion } from "./initializeDeployment/createDeploymentWithNextVersion.server"; import { cancelSupersededDeployments, @@ -56,7 +57,8 @@ export type InitializeDeploymentResult = export class InitializeDeploymentService extends BaseService { public async call( environment: AuthenticatedEnvironment, - payload: InitializeDeploymentRequestBody + payload: InitializeDeploymentRequestBody, + options?: { cliVersion?: string } ): Promise { return this.traceWithEnv("call", environment, async (span) => { if (payload.externalId) { @@ -386,12 +388,24 @@ export class InitializeDeploymentService extends BaseService { commitSHA: payload.gitMeta?.commitSha ?? undefined, externalId: payload.externalId, runtime: payload.runtime ?? environment.project.defaultRuntime ?? undefined, + cliVersion: options?.cliVersion, triggeredVia: payload.triggeredVia ?? undefined, startedAt: initialStatus === "BUILDING" ? new Date() : undefined, }; } ); + recordDeploymentInitialized({ + deployment, + environment: { + organizationId: environment.organizationId, + projectId: environment.projectId, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, + }); + const timeoutMs = deployment.status === "PENDING" ? env.DEPLOY_QUEUE_TIMEOUT_MS : env.DEPLOY_TIMEOUT_MS; diff --git a/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts new file mode 100644 index 000000000..8d6727cac --- /dev/null +++ b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts @@ -0,0 +1,183 @@ +import { ROOT_CONTEXT, SpanStatusCode } from "@opentelemetry/api"; +import { type WorkerDeployment, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { logger } from "~/services/logger.server"; +import { SEMINTATTRS_FORCE_RECORDING, tracer } from "~/v3/tracer.server"; +import { + DeploymentTelemetryAttributes as ATTRS, + deriveBuildPath, + deriveDeploymentDurations, +} from "~/v3/deploymentTelemetry"; + +type TerminalDeploymentStatus = Extract< + WorkerDeploymentStatus, + "DEPLOYED" | "FAILED" | "TIMED_OUT" | "CANCELED" +>; + +type FinishedDeployment = Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "startedAt" + | "installedAt" + | "builtAt" + | "deployedAt" + | "failedAt" + | "canceledAt" + | "canceledReason" + | "errorData" + | "runtime" + | "runtimeVersion" + | "cliVersion" + | "triggeredVia" + | "commitSHA" +> & { buildServerMetadata: unknown }; + +type EnvironmentInfo = { + organizationId?: string; + projectId?: string; + projectRef?: string; + environmentId?: string; + environmentType?: string; +}; + +/** + * Records a deployment's terminal transition as a single wide + * `deployment.finished` span, backdated createdAt → terminal (attribute + * contract in ../deploymentTelemetry.ts). Call exactly once, only after a + * guarded status write confirmed this caller won the transition. Emitted on + * ROOT_CONTEXT with forceRecording so the sampler can never drop it; never + * throws. + */ +export function recordDeploymentFinished(params: { + status: TerminalDeploymentStatus; + deployment: FinishedDeployment; + environment: EnvironmentInfo; + reason?: string; +}): void { + try { + const { status, deployment, environment, reason } = params; + + const isFailure = status === "FAILED" || status === "TIMED_OUT"; + const terminalAt = + deployment.deployedAt ?? deployment.failedAt ?? deployment.canceledAt ?? new Date(); + const durations = deriveDeploymentDurations(deployment, terminalAt); + const errorData = parseErrorData(deployment.errorData); + + const span = tracer.startSpan( + "deployment.finished", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: status, + [ATTRS.SUCCESS]: status === "DEPLOYED", + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.RUNTIME_VERSION]: deployment.runtimeVersion ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + [ATTRS.COMMIT_SHA]: deployment.commitSHA ?? undefined, + [ATTRS.ERROR_NAME]: isFailure ? errorData?.name : undefined, + [ATTRS.ERROR_MESSAGE]: isFailure ? (reason ?? errorData?.message) : undefined, + [ATTRS.CANCELED_REASON]: deployment.canceledReason ?? undefined, + [ATTRS.DURATION_TOTAL_MS]: durations.totalMs, + [ATTRS.DURATION_QUEUE_MS]: durations.queueMs, + [ATTRS.DURATION_INSTALL_MS]: durations.installMs, + [ATTRS.DURATION_BUILDING_MS]: durations.buildingMs, + [ATTRS.DURATION_DEPLOYING_MS]: durations.deployingMs, + }, + }, + ROOT_CONTEXT + ); + + // CANCELED is deliberately not an error: it stays out of failure rates + if (isFailure) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: reason ?? errorData?.message, + }); + } + + span.end(terminalAt); + } catch (error) { + logger.debug("recordDeploymentFinished failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +/** + * Records a deployment's creation as a zero-duration `deployment.initialized` + * event — the funnel counterpart to `deployment.finished` for detecting + * stuck deployments. Never throws. + */ +export function recordDeploymentInitialized(params: { + deployment: Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "runtime" + | "cliVersion" + | "triggeredVia" + > & { buildServerMetadata: unknown }; + environment: EnvironmentInfo; +}): void { + try { + const { deployment, environment } = params; + + const span = tracer.startSpan( + "deployment.initialized", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: deployment.status, + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + }, + }, + ROOT_CONTEXT + ); + + span.end(deployment.createdAt); + } catch (error) { + logger.debug("recordDeploymentInitialized failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +function parseErrorData(errorData: unknown): { name?: string; message?: string } | undefined { + if (!errorData || typeof errorData !== "object") return undefined; + const record = errorData as Record; + return { + name: typeof record.name === "string" ? record.name : undefined, + message: typeof record.message === "string" ? record.message : undefined, + }; +} diff --git a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts b/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts deleted file mode 100644 index e66a7a6a9..000000000 --- a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { SpanStatusCode } from "@opentelemetry/api"; -import { type WorkerDeploymentStatus } from "@trigger.dev/database"; -import { logger } from "~/services/logger.server"; -import { tracer } from "~/v3/tracer.server"; - -type TerminalDeploymentStatus = Extract< - WorkerDeploymentStatus, - "DEPLOYED" | "FAILED" | "TIMED_OUT" ->; - -/** - * Records a deployment's terminal status as a `deployment.outcome` span so - * deploy success/failure is queryable from traces (no DB read). Call after each - * terminal-status write. Org/project/env are best-effort; never throws. - */ -export function recordDeploymentOutcome(params: { - status: TerminalDeploymentStatus; - deploymentFriendlyId: string; - organizationId?: string; - projectId?: string; - environmentId?: string; - environmentType?: string; - reason?: string; -}): void { - try { - const span = tracer.startSpan("deployment.outcome", { - attributes: { - "$trigger.org.id": params.organizationId, - "$trigger.project.id": params.projectId, - "$trigger.env.id": params.environmentId, - "$trigger.env.type": params.environmentType, - "deployment.outcome.status": params.status, - "deployment.outcome.success": params.status === "DEPLOYED", - "deployment.outcome.deployment_id": params.deploymentFriendlyId, - "deployment.outcome.reason": params.reason, - }, - }); - - if (params.status !== "DEPLOYED") { - span.setStatus({ code: SpanStatusCode.ERROR, message: params.reason }); - } - - span.end(); - } catch (error) { - logger.debug("recordDeploymentOutcome failed", { - deploymentFriendlyId: params.deploymentFriendlyId, - error: error instanceof Error ? error.message : String(error), - }); - } -} diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index 5e417a786..63d81ba19 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -5,7 +5,7 @@ import { commonWorker } from "../commonWorker.server"; import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { type PrismaClientOrTransaction } from "~/db.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export class TimeoutDeploymentService extends BaseService { public async call(id: string, fromStatus: string, errorMessage: string) { @@ -38,25 +38,49 @@ export class TimeoutDeploymentService extends BaseService { return; } - const timedOutDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + const errorData = { message: errorMessage, name: "TimeoutError" }; + + // Guarded: keeps the fromStatus check atomic with the write + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: deployment.status, }, data: { status: "TIMED_OUT", - failedAt: new Date(), - errorData: { message: errorMessage, name: "TimeoutError" }, + failedAt, + errorData, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Deployment moved out of the expected state concurrently, skipping timeout", { + id: deployment.id, + fromStatus, + }); + return; + } + + const timedOutDeployment = { + ...deployment, + status: "TIMED_OUT" as const, + failedAt, + errorData, + buildEnvVars: null, + }; + + recordDeploymentFinished({ status: "TIMED_OUT", - deploymentFriendlyId: deployment.friendlyId, - organizationId: deployment.environment.project.organizationId, - projectId: deployment.environment.projectId, - environmentId: deployment.environmentId, - environmentType: deployment.environment.type, + deployment: timedOutDeployment, + environment: { + organizationId: deployment.environment.project.organizationId, + projectId: deployment.environment.projectId, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environmentId, + environmentType: deployment.environment.type, + }, reason: errorMessage, }); diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index cbf9a937c..7047d65d2 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -36,7 +36,9 @@ import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto"; import { BatchSpanProcessor, ParentBasedSampler, + type ReadableSpan, type Sampler, + type Span as SdkTraceSpan, SamplingDecision, type SamplingResult, SimpleSpanProcessor, @@ -69,7 +71,7 @@ import { metricsRegister } from "~/metrics.server"; import { collectDatabaseClientMetrics } from "~/utils/databaseMetrics.server"; import { performance } from "node:perf_hooks"; -const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; +export const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; export const DATASOURCE_CONTEXT_KEY = createContextKey("trigger.db.datasource"); @@ -89,6 +91,29 @@ class DatasourceAttributeSpanProcessor implements SpanProcessor { } } +// Mirrors name-prefixed spans into a second exporter; they still flow to the main one +class SpanNamePrefixMirrorProcessor implements SpanProcessor { + constructor( + private readonly _inner: SpanProcessor, + private readonly _prefix: string + ) {} + + onStart(span: SdkTraceSpan, parentContext: Context): void { + this._inner.onStart(span, parentContext); + } + onEnd(span: ReadableSpan): void { + if (span.name.startsWith(this._prefix)) { + this._inner.onEnd(span); + } + } + shutdown(): Promise { + return this._inner.shutdown(); + } + forceFlush(): Promise { + return this._inner.forceFlush(); + } +} + class CustomWebappSampler implements Sampler { constructor(private readonly _baseSampler: Sampler) {} @@ -270,6 +295,30 @@ function setupTelemetry() { } } + if (env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL) { + const deploymentEventExporter = new OTLPTraceExporter({ + url: env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL, + timeoutMillis: 15_000, + headers: parseInternalDeploymentEventHeaders() ?? {}, + }); + + spanProcessors.push( + new SpanNamePrefixMirrorProcessor( + new BatchSpanProcessor(deploymentEventExporter, { + maxExportBatchSize: 64, + scheduledDelayMillis: 1000, + exportTimeoutMillis: 30000, + maxQueueSize: 2048, + }), + "deployment." + ) + ); + + console.log( + `🔦 Tracer: deployment-event exporter enabled to ${env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL}` + ); + } + const ratioSampler = new TraceIdRatioBasedSampler(samplingRate); const provider = new NodeTracerProvider({ @@ -341,6 +390,13 @@ function setupTelemetry() { instrumentations, }); + // Without this flush every shutdown drops the last batch of spans + const flushOnShutdown = () => { + provider.forceFlush().catch(() => {}); + }; + process.once("SIGTERM", flushOnShutdown); + process.once("SIGINT", flushOnShutdown); + return { tracer: provider.getTracer("trigger.dev", "3.3.12"), logger: logs.getLogger("trigger.dev", "3.3.12"), @@ -874,6 +930,19 @@ function parseInternalTraceHeaders(): Record | undefined { } } +function parseInternalDeploymentEventHeaders(): Record | undefined { + try { + return env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS + ? (JSON.parse(env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS) as Record< + string, + string + >) + : undefined; + } catch { + return; + } +} + function parseInternalMetricsHeaders(): Record | undefined { try { return env.INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS diff --git a/apps/webapp/test/deploymentTelemetry.test.ts b/apps/webapp/test/deploymentTelemetry.test.ts new file mode 100644 index 000000000..9020f0c71 --- /dev/null +++ b/apps/webapp/test/deploymentTelemetry.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { deriveBuildPath, deriveDeploymentDurations } from "~/v3/deploymentTelemetry"; + +describe("deriveBuildPath", () => { + it("classifies fromBundle native builds as native_local_bundle", () => { + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: true })).toBe("native_local_bundle"); + }); + + it("classifies native builds without fromBundle as native", () => { + expect(deriveBuildPath({ isNativeBuild: true })).toBe("native"); + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: false })).toBe("native"); + }); + + it("classifies everything else as depot", () => { + expect(deriveBuildPath(null)).toBe("depot"); + expect(deriveBuildPath(undefined)).toBe("depot"); + expect(deriveBuildPath({})).toBe("depot"); + expect(deriveBuildPath({ buildId: "depot-build-id" })).toBe("depot"); + expect(deriveBuildPath({ isNativeBuild: false })).toBe("depot"); + // fromBundle alone (skewed writer) must not count as native_local_bundle + expect(deriveBuildPath({ fromBundle: true })).toBe("depot"); + expect(deriveBuildPath("garbage")).toBe("depot"); + }); +}); + +describe("deriveDeploymentDurations", () => { + const t = (seconds: number) => new Date(1_700_000_000_000 + seconds * 1000); + + it("derives all phases for the full build-server chain", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(10), installedAt: t(40), builtAt: t(100) }, + t(130) + ); + + expect(durations).toEqual({ + totalMs: 130_000, + queueMs: 10_000, + installMs: 30_000, + buildingMs: 60_000, + deployingMs: 30_000, + }); + }); + + it("omits install and measures building from startedAt when installedAt is missing (depot)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(0), installedAt: null, builtAt: t(90) }, + t(120) + ); + + expect(durations).toEqual({ + totalMs: 120_000, + queueMs: 0, + installMs: undefined, + buildingMs: 90_000, + deployingMs: 30_000, + }); + }); + + it("omits phases whose boundaries are missing (failed before building)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(5), installedAt: null, builtAt: null }, + t(20) + ); + + expect(durations).toEqual({ + totalMs: 20_000, + queueMs: 5_000, + installMs: undefined, + buildingMs: undefined, + deployingMs: undefined, + }); + }); + + it("never returns negative durations on clock skew", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(10), startedAt: t(5), installedAt: null, builtAt: null }, + t(3) + ); + + expect(durations.totalMs).toBe(0); + expect(durations.queueMs).toBeUndefined(); + }); +}); diff --git a/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql new file mode 100644 index 000000000..931ea9479 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "public"."WorkerDeployment" ADD COLUMN IF NOT EXISTS "cliVersion" TEXT; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index a77890930..a1423f340 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2265,6 +2265,8 @@ model WorkerDeployment { runtime String? runtimeVersion String? + /// CLI version that initiated the deploy, stamped at initialization + cliVersion String? imageReference String? imagePlatform String @default("linux/amd64") diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 8b9fd56eb..a7a07cf40 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -1016,6 +1016,7 @@ export class CliApiClient { Authorization: `Bearer ${this.accessToken}`, "Content-Type": "application/json", "x-trigger-source": this.source, + "x-trigger-cli-version": VERSION, ...this.getBranchHeader(), }; }