feat(webapp): link Sentry events to OTel traces via trace_id (#3531)

## Summary Stamps the active OpenTelemetry `trace_id` and `span_id` onto every Sentry event captured from the webapp, so engineers can copy a `trace_id` from a Sentry issue and search for the corresponding trace in any OTel-aware backend. Also adds an `otel_sampled` tag to indicate whether the trace was head-sampled — a cheap signal for whether the link will resolve to span data or hit a missing trace. ## Why Sentry and OTel were OTel-disconnected: `apps/webapp/sentry.server.ts` initialised Sentry with `skipOpenTelemetrySetup: true`, and no error-capture site (`logger.server.ts`, the Remix-wrapped `handleError`, the root `ErrorBoundary`) attached OTel context to the event. With many spans/sec across services, getting from a Sentry issue to its trace was guesswork. ## Approach Single global Sentry event processor, registered immediately after `Sentry.init`. On each event it reads `trace.getActiveSpan()?.spanContext()` via `@opentelemetry/api`, then writes: - `event.contexts.trace.trace_id` and `event.contexts.trace.span_id` (Sentry's native trace context fields) - `event.tags.otel_sampled` = `"true"` | `"false"` (derived from `traceFlags`) If no active span (module-load errors, scheduled timers without a context, primary cluster process), the processor returns the event unmodified — Sentry's default propagation context fills in. Implementation is co-located in `apps/webapp/sentry.server.ts` (no separate helper module — `sentry.server.ts` is built standalone by esbuild and a separate import would have required a new bundling step). Helper functions are exported so the unit tests can reach them without re-running `Sentry.init`. ## Non-goals (deliberate) - No sample rate change. ~95% of Sentry events will carry a `trace_id` that returns no spans in the tracing backend (head-sampled out). The `otel_sampled` tag makes that obvious at a glance. Raising find-rate is a separate conversation with cost trade-offs. - No user/org tags or `Sentry.setUser` (would need auth-helper + per-request scope wiring across multiple worker entrypoints — separate ticket). - Webapp image only. No changes to supervisor or CLI workers. ## Test plan - [x] Unit tests in `apps/webapp/test/sentryTraceContext.server.test.ts` — 9 tests covering: helper returns \`undefined\` with no active span; returns \`traceId\`/\`spanId\`/\`sampled=true\` for a recording span; returns \`sampled=false\` for a non-recording span; processor leaves the event unchanged with no active span; processor stamps \`trace_id\`/\`span_id\` onto \`contexts.trace\`; preserves existing \`contexts.trace\` fields; tags \`otel_sampled\` correctly for both sampled and non-sampled cases; never throws if \`@opentelemetry/api\` access throws. - [x] \`pnpm run typecheck --filter webapp\` passes. - [x] Manually verified end-to-end against a sandboxed Sentry project: confirmed both sampled and non-sampled traces correctly populate \`contexts.trace.trace_id\` matching the OTel ids logged from the loader, and the \`otel_sampled\` tag appears with the expected value. Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-08 11:47:24 +01:00
parent 3e6458f9b5
commit 749dc467f1
5 changed files with 189 additions and 1 deletions
@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Stamp the active OpenTelemetry trace_id and span_id onto every Sentry event so issues can be cross-referenced with traces in any OTel backend.
@@ -0,0 +1,52 @@
+import { type Span, TraceFlags, trace } from "@opentelemetry/api";
+import type { Event, EventHint } from "@sentry/remix";
+
+export type GetActiveSpan = () => Span | undefined;
+
+const defaultGetActiveSpan: GetActiveSpan = () => trace.getActiveSpan();
+
+export function getActiveTraceIds(
+  getActiveSpan: GetActiveSpan = defaultGetActiveSpan
+): { traceId: string; spanId: string; sampled: boolean } | undefined {
+  try {
+    const span = getActiveSpan();
+    if (!span) return undefined;
+    const ctx = span.spanContext();
+    return {
+      traceId: ctx.traceId,
+      spanId: ctx.spanId,
+      sampled: (ctx.traceFlags & TraceFlags.SAMPLED) !== 0,
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+export function addOtelTraceContextToEvent(
+  event: Event,
+  _hint: EventHint,
+  getActiveSpan: GetActiveSpan = defaultGetActiveSpan
+): Event {
+  const ids = getActiveTraceIds(getActiveSpan);
+  if (!ids) return event;
+  // We intentionally overwrite Sentry's own trace_id/span_id on contexts.trace.
+  // With skipOpenTelemetrySetup: true, Sentry generates an internal trace_id
+  // unrelated to OTel; replacing it with the active OTel ids is the whole
+  // point of this processor — it makes Sentry issues navigable to the
+  // corresponding OTel trace in any backend.
+  return {
+    ...event,
+    contexts: {
+      ...event.contexts,
+      trace: {
+        ...event.contexts?.trace,
+        trace_id: ids.traceId,
+        span_id: ids.spanId,
+      },
+    },
+    tags: {
+      ...event.tags,
+      otel_sampled: ids.sampled ? "true" : "false",
+    },
+  };
+}
@@ -7,7 +7,7 @@
    "build": "run-s build:** && pnpm run upload:sourcemaps",
    "build:remix": "remix build --sourcemap",
    "build:server": "esbuild --platform=node --format=cjs ./server.ts --outdir=build --sourcemap",
-    "build:sentry": "esbuild --platform=node --format=cjs ./sentry.server.ts --outdir=build --sourcemap",
+    "build:sentry": "esbuild --platform=node --format=cjs --outbase=. ./sentry.server.ts ./app/utils/sentryTraceContext.server.ts --outdir=build --sourcemap",
    "dev": "cross-env PORT=3030 remix dev -c \"node ./build/server.js\"",
    "dev:worker": "cross-env NODE_PATH=../../node_modules/.pnpm/node_modules node ./build/server.js",
    "format": "prettier --write .",
@@ -1,4 +1,5 @@
 import * as Sentry from "@sentry/remix";
+import { addOtelTraceContextToEvent } from "./app/utils/sentryTraceContext.server";

 if (process.env.SENTRY_DSN) {
  console.log("🔭 Initializing Sentry");
@@ -29,4 +30,6 @@ if (process.env.SENTRY_DSN) {
    ignoreErrors: ["queryRoute() call aborted", /^ServiceValidationError(?::|$)/],
    includeLocalVariables: false,
  });
+
+  Sentry.addEventProcessor(addOtelTraceContextToEvent);
 }
@@ -0,0 +1,127 @@
+import { ROOT_CONTEXT, TraceFlags, context, trace } from "@opentelemetry/api";
+import { describe, expect, it } from "vitest";
+import {
+  addOtelTraceContextToEvent,
+  getActiveTraceIds,
+} from "../app/utils/sentryTraceContext.server";
+import { createInMemoryTracing } from "./utils/tracing";
+
+describe("getActiveTraceIds", () => {
+  it("returns undefined when no OTel span is active", () => {
+    expect(getActiveTraceIds()).toBeUndefined();
+  });
+
+  it("returns the trace_id, span_id, and sampled=true for an active recording span", () => {
+    const { tracer } = createInMemoryTracing();
+
+    tracer.startActiveSpan("test-span", (span) => {
+      const ids = getActiveTraceIds();
+      expect(ids).toEqual({
+        traceId: span.spanContext().traceId,
+        spanId: span.spanContext().spanId,
+        sampled: true,
+      });
+      span.end();
+    });
+  });
+
+  it("returns sampled=false when the active span is non-recording", () => {
+    // Initialise the global context manager (createInMemoryTracing does this
+    // as a side effect of NodeTracerProvider.register()).
+    createInMemoryTracing();
+
+    const nonSampledSpan = trace.wrapSpanContext({
+      traceId: "0123456789abcdef0123456789abcdef",
+      spanId: "0123456789abcdef",
+      traceFlags: TraceFlags.NONE,
+    });
+
+    context.with(trace.setSpan(ROOT_CONTEXT, nonSampledSpan), () => {
+      expect(getActiveTraceIds()).toEqual({
+        traceId: "0123456789abcdef0123456789abcdef",
+        spanId: "0123456789abcdef",
+        sampled: false,
+      });
+    });
+  });
+});
+
+describe("addOtelTraceContextToEvent", () => {
+  it("returns the event unchanged when no OTel span is active", () => {
+    const event = { message: "boom" };
+    const result = addOtelTraceContextToEvent(event, {});
+    expect(result).toBe(event);
+    expect(result).toEqual({ message: "boom" });
+  });
+
+  it("stamps trace_id and span_id from the active span onto event.contexts.trace", () => {
+    const { tracer } = createInMemoryTracing();
+
+    tracer.startActiveSpan("test-span", (span) => {
+      const event = { message: "boom" };
+      const result = addOtelTraceContextToEvent(event, {});
+      expect(result.contexts?.trace?.trace_id).toBe(span.spanContext().traceId);
+      expect(result.contexts?.trace?.span_id).toBe(span.spanContext().spanId);
+      span.end();
+    });
+  });
+
+  it("tags the event with otel_sampled=true when the active span is recording", () => {
+    const { tracer } = createInMemoryTracing();
+
+    tracer.startActiveSpan("test-span", (span) => {
+      const event = { message: "boom" };
+      const result = addOtelTraceContextToEvent(event, {});
+      expect(result.tags?.otel_sampled).toBe("true");
+      span.end();
+    });
+  });
+
+  it("tags the event with otel_sampled=false when the active span is non-recording", () => {
+    createInMemoryTracing();
+
+    const nonSampledSpan = trace.wrapSpanContext({
+      traceId: "0123456789abcdef0123456789abcdef",
+      spanId: "0123456789abcdef",
+      traceFlags: TraceFlags.NONE,
+    });
+
+    context.with(trace.setSpan(ROOT_CONTEXT, nonSampledSpan), () => {
+      const event = { message: "boom" };
+      const result = addOtelTraceContextToEvent(event, {});
+      expect(result.tags?.otel_sampled).toBe("false");
+    });
+  });
+
+  it("preserves existing event.contexts.trace fields", () => {
+    const { tracer } = createInMemoryTracing();
+
+    tracer.startActiveSpan("test-span", (span) => {
+      const event = {
+        message: "boom",
+        contexts: {
+          trace: { op: "http.server", description: "GET /things" },
+          runtime: { name: "node" },
+        },
+      };
+      const result = addOtelTraceContextToEvent(event, {});
+      expect(result.contexts?.trace).toMatchObject({
+        op: "http.server",
+        description: "GET /things",
+        trace_id: span.spanContext().traceId,
+        span_id: span.spanContext().spanId,
+      });
+      expect(result.contexts?.runtime).toEqual({ name: "node" });
+      span.end();
+    });
+  });
+
+  it("returns the event unchanged if reading the OTel context throws", () => {
+    const throwingAccessor = () => {
+      throw new Error("otel api blew up");
+    };
+    const event = { message: "boom" };
+    const result = addOtelTraceContextToEvent(event, {}, throwingAccessor);
+    expect(result).toBe(event);
+  });
+});