latest @opentelemetry packages and correlate external traces (#2334)

.changeset/five-nails-whisper.md

@@ -0,0 +1,92 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+External Trace Correlation & OpenTelemetry Package Updates.
+
+| Package | Previous Version | New Version | Change Type |
+|---------|------------------|-------------|-------------|
+| `@opentelemetry/api` | 1.9.0 | 1.9.0 | No change (stable API) |
+| `@opentelemetry/api-logs` | 0.52.1 | 0.203.0 | Major update |
+| `@opentelemetry/core` | - | 2.0.1 | New dependency |
+| `@opentelemetry/exporter-logs-otlp-http` | 0.52.1 | 0.203.0 | Major update |
+| `@opentelemetry/exporter-trace-otlp-http` | 0.52.1 | 0.203.0 | Major update |
+| `@opentelemetry/instrumentation` | 0.52.1 | 0.203.0 | Major update |
+| `@opentelemetry/instrumentation-fetch` | 0.52.1 | 0.203.0 | Major update |
+| `@opentelemetry/resources` | 1.25.1 | 2.0.1 | Major update |
+| `@opentelemetry/sdk-logs` | 0.52.1 | 0.203.0 | Major update |
+| `@opentelemetry/sdk-node` | 0.52.1 | - | Removed (functionality consolidated) |
+| `@opentelemetry/sdk-trace-base` | 1.25.1 | 2.0.1 | Major update |
+| `@opentelemetry/sdk-trace-node` | 1.25.1 | 2.0.1 | Major update |
+| `@opentelemetry/semantic-conventions` | 1.25.1 | 1.36.0 | Minor update |
+
+### External trace correlation and propagation
+
+We will now correlate your external traces with trigger.dev traces and logs when using our external exporters:
+
+```ts
+import { defineConfig } from "@trigger.dev/sdk";
+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+
+export default defineConfig({
+  project: process.env.TRIGGER_PROJECT_REF,
+  dirs: ["./src/trigger"],
+  telemetry: {
+    logExporters: [
+      new OTLPLogExporter({
+        url: "https://api.axiom.co/v1/logs",
+        headers: {
+          Authorization: `Bearer ${process.env.AXIOM_TOKEN}`,
+          "X-Axiom-Dataset": "test",
+        },
+      }),
+    ],
+    exporters: [
+      new OTLPTraceExporter({
+        url: "https://api.axiom.co/v1/traces",
+        headers: {
+          Authorization: `Bearer ${process.env.AXIOM_TOKEN}`,
+          "X-Axiom-Dataset": "test",
+        },
+      }),
+    ],
+  },
+  maxDuration: 3600,
+});
+```
+
+You can also now propagate your external trace context when calling back into your own backend infra from inside a trigger.dev task:
+
+```ts
+import { otel, task } from "@trigger.dev/sdk";
+import { context, propagation } from "@opentelemetry/api";
+
+async function callNextjsApp() {
+  return await otel.withExternalTrace(async () => {
+    const headersObject = {};
+
+    // Now context.active() refers to your external trace context
+    propagation.inject(context.active(), headersObject);
+
+    const result = await fetch("http://localhost:3000/api/demo-call-from-trigger", {
+      headers: new Headers(headersObject),
+      method: "POST",
+      body: JSON.stringify({
+        message: "Hello from Trigger.dev",
+      }),
+    });
+
+    return result.json();
+  });
+}
+
+export const myTask = task({
+  id: "my-task",
+  run: async (payload: any) => {
+    await callNextjsApp()
+  }
+})
+```
+
+

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -4,9 +4,10 @@ import {
   SemanticInternalAttributes,
   TaskRunContext,
   TaskRunError,
+  TriggerTraceContext,
   V3TaskRunContext,
 } from "@trigger.dev/core/v3";
-import { AttemptId, getMaxDuration } from "@trigger.dev/core/v3/isomorphic";
+import { AttemptId, getMaxDuration, parseTraceparent } from "@trigger.dev/core/v3/isomorphic";
 import { RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { logger } from "~/services/logger.server";
 import { eventRepository, rehydrateAttribute } from "~/v3/eventRepository.server";
@@ -173,6 +174,8 @@ export class SpanPresenter extends BasePresenter {
 
     const context = await this.#getTaskRunContext({ run, machine: machine ?? undefined });
 
+    const externalTraceId = this.#getExternalTraceId(run.traceContext);
+
     return {
       id: run.id,
       friendlyId: run.friendlyId,
@@ -234,6 +237,7 @@ export class SpanPresenter extends BasePresenter {
       spanId: run.spanId,
       isCached: !!span.originalRun,
       machinePreset: machine?.name,
+      externalTraceId,
     };
   }
 
@@ -272,6 +276,7 @@ export class SpanPresenter extends BasePresenter {
         id: true,
         spanId: true,
         traceId: true,
+        traceContext: true,
         //metadata
         number: true,
         taskIdentifier: true,
@@ -574,4 +579,26 @@ export class SpanPresenter extends BasePresenter {
   async #getV4TaskRunContext({ run }: { run: FindRunResult }): Promise<TaskRunContext> {
     return engine.resolveTaskRunContext(run.id);
   }
+
+  #getExternalTraceId(traceContext: unknown) {
+    if (!traceContext) {
+      return;
+    }
+
+    const parsedTraceContext = TriggerTraceContext.safeParse(traceContext);
+
+    if (!parsedTraceContext.success) {
+      return;
+    }
+
+    const externalTraceparent = parsedTraceContext.data.external?.traceparent;
+
+    if (!externalTraceparent) {
+      return;
+    }
+
+    const parsedTraceparent = parseTraceparent(externalTraceparent);
+
+    return parsedTraceparent?.traceId;
+  }
 }

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -93,10 +93,9 @@ const { action, loader } = createActionApiRoute(
     const service = new TriggerTaskService();
 
     try {
-      const traceContext =
-        traceparent && isFromWorker /// If the request is from a worker, we should pass the trace context
-          ? { traceparent, tracestate }
-          : undefined;
+      const traceContext = isFromWorker
+        ? { traceparent, tracestate }
+        : { external: { traceparent, tracestate } };
 
       const oneTimeUseToken = await getOneTimeUseToken(authentication);
 
@@ -111,6 +110,14 @@ const { action, loader } = createActionApiRoute(
         traceContext,
       });
 
+      logger.debug("[otelContext]", {
+        taskId: params.taskId,
+        headers,
+        options: body.options,
+        isFromWorker,
+        traceContext,
+      });
+
       const idempotencyKeyExpiresAt = resolveIdempotencyKeyTTL(idempotencyKeyTTL);
 
       const result = await service.call(

apps/webapp/app/routes/api.v2.tasks.batch.ts

@@ -103,10 +103,9 @@ const { action, loader } = createActionApiRoute(
       return cachedResponse;
     }
 
-    const traceContext =
-      traceparent && isFromWorker // If the request is from a worker, we should pass the trace context
-        ? { traceparent, tracestate }
-        : undefined;
+    const traceContext = isFromWorker
+      ? { traceparent, tracestate }
+      : { external: { traceparent, tracestate } };
 
     const service = new RunEngineBatchTriggerService(batchProcessingStrategy ?? undefined);
 

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -743,6 +743,12 @@ function RunBody({
                   <Property.Label>Run Engine</Property.Label>
                   <Property.Value>{run.engine}</Property.Value>
                 </Property.Item>
+                {run.externalTraceId && (
+                  <Property.Item>
+                    <Property.Label>External Trace ID</Property.Label>
+                    <Property.Value>{run.externalTraceId}</Property.Value>
+                  </Property.Item>
+                )}
                 {isAdmin && (
                   <div className="border-t border-yellow-500/50 pt-2">
                     <Paragraph spacing variant="small" className="text-yellow-500">

apps/webapp/app/runEngine/services/batchTrigger.server.ts

@@ -42,7 +42,7 @@ export type BatchProcessingOptions = z.infer<typeof BatchProcessingOptions>;
 
 export type BatchTriggerTaskServiceOptions = {
   triggerVersion?: string;
-  traceContext?: Record<string, string | undefined>;
+  traceContext?: Record<string, string | undefined | Record<string, string | undefined>>;
   spanParentAsLink?: boolean;
   oneTimeUseToken?: string;
 };

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -10,8 +10,14 @@ import {
   taskRunErrorEnhancer,
   taskRunErrorToString,
   TriggerTaskRequestBody,
+  TriggerTraceContext,
 } from "@trigger.dev/core/v3";
-import { RunId, stringifyDuration } from "@trigger.dev/core/v3/isomorphic";
+import {
+  parseTraceparent,
+  RunId,
+  serializeTraceparent,
+  stringifyDuration,
+} from "@trigger.dev/core/v3/isomorphic";
 import type { PrismaClientOrTransaction } from "@trigger.dev/database";
 import { createTags } from "~/models/taskRunTag.server";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
@@ -253,7 +259,11 @@ export class RunEngineTriggerTaskService {
                   payload: payloadPacket.data ?? "",
                   payloadType: payloadPacket.dataType,
                   context: body.context,
-                  traceContext: event.traceContext,
+                  traceContext: this.#propagateExternalTraceContext(
+                    event.traceContext,
+                    parentRun?.traceContext,
+                    event.traceparent?.spanId
+                  ),
                   traceId: event.traceId,
                   spanId: event.spanId,
                   parentSpanId:
@@ -341,4 +351,49 @@ export class RunEngineTriggerTaskService {
       }
     });
   }
+
+  #propagateExternalTraceContext(
+    traceContext: Record<string, unknown>,
+    parentRunTraceContext: unknown,
+    parentSpanId: string | undefined
+  ): TriggerTraceContext {
+    if (!parentRunTraceContext) {
+      return traceContext;
+    }
+
+    const parsedParentRunTraceContext = TriggerTraceContext.safeParse(parentRunTraceContext);
+
+    if (!parsedParentRunTraceContext.success) {
+      return traceContext;
+    }
+
+    const { external } = parsedParentRunTraceContext.data;
+
+    if (!external) {
+      return traceContext;
+    }
+
+    if (!external.traceparent) {
+      return traceContext;
+    }
+
+    const parsedTraceparent = parseTraceparent(external.traceparent);
+
+    if (!parsedTraceparent) {
+      return traceContext;
+    }
+
+    const newExternalTraceparent = serializeTraceparent(
+      parsedTraceparent.traceId,
+      parentSpanId ?? parsedTraceparent.spanId
+    );
+
+    return {
+      ...traceContext,
+      external: {
+        ...external,
+        traceparent: newExternalTraceparent,
+      },
+    };
+  }
 }

apps/webapp/app/runEngine/types.ts

@@ -12,7 +12,7 @@ export type TriggerTaskServiceOptions = {
   idempotencyKey?: string;
   idempotencyKeyExpiresAt?: Date;
   triggerVersion?: string;
-  traceContext?: Record<string, string | undefined>;
+  traceContext?: Record<string, unknown>;
   spanParentAsLink?: boolean;
   parentAsLinkType?: "replay" | "trigger";
   batchId?: string;
@@ -119,7 +119,7 @@ export interface TriggerTaskValidator {
 export type TracedEventSpan = {
   traceId: string;
   spanId: string;
-  traceContext: Record<string, string | undefined>;
+  traceContext: Record<string, unknown>;
   traceparent?: {
     traceId: string;
     spanId: string;

apps/webapp/app/v3/environmentVariableRules.server.ts

@@ -8,8 +8,6 @@ type VariableRule =
 const blacklistedVariables: VariableRule[] = [
   { type: "exact", key: "TRIGGER_SECRET_KEY" },
   { type: "exact", key: "TRIGGER_API_URL" },
-  { type: "prefix", prefix: "OTEL_" },
-  { type: "whitelist", key: "OTEL_LOG_LEVEL" },
 ];
 
 export function removeBlacklistedVariables(

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -17,6 +17,7 @@ import {
 } from "./repository";
 import { removeBlacklistedVariables } from "../environmentVariableRules.server";
 import { deduplicateVariableArray } from "../deduplicateVariableArray.server";
+import { logger } from "~/services/logger.server";
 
 function secretKeyProjectPrefix(projectId: string) {
   return `environmentvariable:${projectId}:`;
@@ -837,11 +838,23 @@ export async function resolveVariablesForEnvironment(
       ? await resolveBuiltInDevVariables(runtimeEnvironment)
       : await resolveBuiltInProdVariables(runtimeEnvironment, parentEnvironment);
 
-  return deduplicateVariableArray([
+  const overridableOtelVariables =
+    runtimeEnvironment.type === "DEVELOPMENT"
+      ? await resolveOverridableOtelDevVariables(runtimeEnvironment)
+      : [];
+
+  const result = deduplicateVariableArray([
     ...overridableTriggerVariables,
+    ...overridableOtelVariables,
     ...projectSecrets,
     ...builtInVariables,
   ]);
+
+  logger.debug("Resolved variables", {
+    result,
+  });
+
+  return result;
 }
 
 async function resolveOverridableTriggerVariables(
@@ -860,7 +873,7 @@ async function resolveOverridableTriggerVariables(
 async function resolveBuiltInDevVariables(runtimeEnvironment: RuntimeEnvironmentForEnvRepo) {
   let result: Array<EnvironmentVariable> = [
     {
-      key: "OTEL_EXPORTER_OTLP_ENDPOINT",
+      key: "TRIGGER_OTEL_EXPORTER_OTLP_ENDPOINT",
       value: env.DEV_OTEL_EXPORTER_OTLP_ENDPOINT ?? `${env.APP_ORIGIN.replace(/\/$/, "")}/otel`,
     },
     {
@@ -875,6 +888,42 @@ async function resolveBuiltInDevVariables(runtimeEnvironment: RuntimeEnvironment
 
   if (env.DEV_OTEL_BATCH_PROCESSING_ENABLED === "1") {
     result = result.concat([
+      {
+        key: "TRIGGER_OTEL_BATCH_PROCESSING_ENABLED",
+        value: "1",
+      },
+      {
+        key: "TRIGGER_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE",
+        value: env.DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE,
+      },
+      {
+        key: "TRIGGER_OTEL_SPAN_SCHEDULED_DELAY_MILLIS",
+        value: env.DEV_OTEL_SPAN_SCHEDULED_DELAY_MILLIS,
+      },
+      {
+        key: "TRIGGER_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS",
+        value: env.DEV_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS,
+      },
+      {
+        key: "TRIGGER_OTEL_SPAN_MAX_QUEUE_SIZE",
+        value: env.DEV_OTEL_SPAN_MAX_QUEUE_SIZE,
+      },
+      {
+        key: "TRIGGER_OTEL_LOG_MAX_EXPORT_BATCH_SIZE",
+        value: env.DEV_OTEL_LOG_MAX_EXPORT_BATCH_SIZE,
+      },
+      {
+        key: "TRIGGER_OTEL_LOG_SCHEDULED_DELAY_MILLIS",
+        value: env.DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS,
+      },
+      {
+        key: "TRIGGER_OTEL_LOG_EXPORT_TIMEOUT_MILLIS",
+        value: env.DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS,
+      },
+      {
+        key: "TRIGGER_OTEL_LOG_MAX_QUEUE_SIZE",
+        value: env.DEV_OTEL_LOG_MAX_QUEUE_SIZE,
+      },
       {
         key: "OTEL_BATCH_PROCESSING_ENABLED",
         value: "1",
@@ -919,6 +968,19 @@ async function resolveBuiltInDevVariables(runtimeEnvironment: RuntimeEnvironment
   return [...result, ...commonVariables];
 }
 
+async function resolveOverridableOtelDevVariables(
+  runtimeEnvironment: RuntimeEnvironmentForEnvRepo
+) {
+  let result: Array<EnvironmentVariable> = [
+    {
+      key: "OTEL_EXPORTER_OTLP_ENDPOINT",
+      value: env.DEV_OTEL_EXPORTER_OTLP_ENDPOINT ?? `${env.APP_ORIGIN.replace(/\/$/, "")}/otel`,
+    },
+  ];
+
+  return result;
+}
+
 async function resolveBuiltInProdVariables(
   runtimeEnvironment: RuntimeEnvironmentForEnvRepo,
   parentEnvironment?: RuntimeEnvironmentForEnvRepo
@@ -957,6 +1019,42 @@ async function resolveBuiltInProdVariables(
 
   if (env.PROD_OTEL_BATCH_PROCESSING_ENABLED === "1") {
     result = result.concat([
+      {
+        key: "TRIGGER_OTEL_BATCH_PROCESSING_ENABLED",
+        value: "1",
+      },
+      {
+        key: "TRIGGER_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE",
+        value: env.PROD_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE,
+      },
+      {
+        key: "TRIGGER_OTEL_SPAN_SCHEDULED_DELAY_MILLIS",
+        value: env.PROD_OTEL_SPAN_SCHEDULED_DELAY_MILLIS,
+      },
+      {
+        key: "TRIGGER_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS",
+        value: env.PROD_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS,
+      },
+      {
+        key: "TRIGGER_OTEL_SPAN_MAX_QUEUE_SIZE",
+        value: env.PROD_OTEL_SPAN_MAX_QUEUE_SIZE,
+      },
+      {
+        key: "TRIGGER_OTEL_LOG_MAX_EXPORT_BATCH_SIZE",
+        value: env.PROD_OTEL_LOG_MAX_EXPORT_BATCH_SIZE,
+      },
+      {
+        key: "TRIGGER_OTEL_LOG_SCHEDULED_DELAY_MILLIS",
+        value: env.PROD_OTEL_LOG_SCHEDULED_DELAY_MILLIS,
+      },
+      {
+        key: "TRIGGER_OTEL_LOG_EXPORT_TIMEOUT_MILLIS",
+        value: env.PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS,
+      },
+      {
+        key: "TRIGGER_OTEL_LOG_MAX_QUEUE_SIZE",
+        value: env.PROD_OTEL_LOG_MAX_QUEUE_SIZE,
+      },
       {
         key: "OTEL_BATCH_PROCESSING_ENABLED",
         value: "1",

apps/webapp/app/v3/eventRepository.server.ts

@@ -2,9 +2,14 @@ import { Attributes, AttributeValue, Link, trace, TraceFlags, Tracer } from "@op
 import { RandomIdGenerator } from "@opentelemetry/sdk-trace-base";
 import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
 import {
+  correctErrorStackTrace,
+  createPacketAttributesAsJson,
   ExceptionEventProperties,
   ExceptionSpanEvent,
+  flattenAttributes,
+  isExceptionSpanEvent,
   NULL_SENTINEL,
+  omit,
   PRIMARY_VARIANT,
   SemanticInternalAttributes,
   SpanEvent,
@@ -13,28 +18,24 @@ import {
   TaskEventEnvironment,
   TaskEventStyle,
   TaskRunError,
-  correctErrorStackTrace,
-  createPacketAttributesAsJson,
-  flattenAttributes,
-  isExceptionSpanEvent,
-  omit,
   unflattenAttributes,
 } from "@trigger.dev/core/v3";
+import { parseTraceparent, serializeTraceparent } from "@trigger.dev/core/v3/isomorphic";
 import { Prisma, TaskEvent, TaskEventKind, TaskEventStatus } from "@trigger.dev/database";
+import { nanoid } from "nanoid";
 import { createHash } from "node:crypto";
 import { EventEmitter } from "node:stream";
 import { Gauge } from "prom-client";
-import { $replica, PrismaClient, PrismaReplicaClient, prisma } from "~/db.server";
+import { $replica, prisma, PrismaClient, PrismaReplicaClient } from "~/db.server";
 import { env } from "~/env.server";
 import { metricsRegister } from "~/metrics.server";
+import { createRedisClient, RedisClient, RedisWithClusterOptions } from "~/redis.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "~/utils/singleton";
 import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
-import { startActiveSpan } from "./tracer.server";
-import { createRedisClient, RedisClient, RedisWithClusterOptions } from "~/redis.server";
-import { startSpan } from "./tracing.server";
-import { nanoid } from "nanoid";
 import { TaskEventStore, TaskEventStoreTable } from "./taskEventStore.server";
+import { startActiveSpan } from "./tracer.server";
+import { startSpan } from "./tracing.server";
 
 const MAX_FLUSH_DEPTH = 5;
 
@@ -80,7 +81,7 @@ export type SetAttribute<T extends TraceAttributes> = (key: keyof T, value: T[ke
 
 export type TraceEventOptions = {
   kind?: CreatableEventKind;
-  context?: Record<string, string | undefined>;
+  context?: Record<string, unknown>;
   spanParentAsLink?: boolean;
   parentAsLinkType?: "trigger" | "replay";
   spanIdSeed?: string;
@@ -932,7 +933,7 @@ export class EventRepository {
       traceId,
       spanId,
       parentId,
-      tracestate,
+      tracestate: typeof tracestate === "string" ? tracestate : undefined,
       message: message,
       serviceName: "api server",
       serviceNamespace: "trigger.dev",
@@ -989,6 +990,11 @@ export class EventRepository {
   ): Promise<TResult> {
     const propagatedContext = extractContextFromCarrier(options.context ?? {});
 
+    logger.debug("[otelContext]", {
+      propagatedContext,
+      options,
+    });
+
     const start = process.hrtime.bigint();
     const startTime = options.startTime ?? getNowInNanoseconds();
 
@@ -1002,7 +1008,8 @@ export class EventRepository {
       : this.generateSpanId();
 
     const traceContext = {
-      traceparent: `00-${traceId}-${spanId}-01`,
+      ...options.context,
+      traceparent: serializeTraceparent(traceId, spanId),
     };
 
     const links: Link[] =
@@ -1087,7 +1094,7 @@ export class EventRepository {
       traceId,
       spanId,
       parentId,
-      tracestate,
+      tracestate: typeof tracestate === "string" ? tracestate : undefined,
       duration: options.incomplete ? 0 : duration,
       isPartial: failedWithError ? false : options.incomplete,
       isError: !!failedWithError,
@@ -1486,36 +1493,21 @@ function excludePartialEventsWithCorrespondingFullEvent(batch: CreatableEvent[])
   );
 }
 
-export function extractContextFromCarrier(carrier: Record<string, string | undefined>) {
+export function extractContextFromCarrier(carrier: Record<string, unknown>) {
   const traceparent = carrier["traceparent"];
   const tracestate = carrier["tracestate"];
 
+  if (typeof traceparent !== "string") {
+    return undefined;
+  }
+
   return {
+    ...carrier,
     traceparent: parseTraceparent(traceparent),
     tracestate,
   };
 }
 
-function parseTraceparent(traceparent?: string): { traceId: string; spanId: string } | undefined {
-  if (!traceparent) {
-    return undefined;
-  }
-
-  const parts = traceparent.split("-");
-
-  if (parts.length !== 4) {
-    return undefined;
-  }
-
-  const [version, traceId, spanId, flags] = parts;
-
-  if (version !== "00") {
-    return undefined;
-  }
-
-  return { traceId, spanId };
-}
-
 function prepareEvent(event: QueriedEvent): PreparedEvent {
   return {
     ...event,

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -19,7 +19,7 @@ export type TriggerTaskServiceOptions = {
   idempotencyKey?: string;
   idempotencyKeyExpiresAt?: Date;
   triggerVersion?: string;
-  traceContext?: Record<string, string | undefined>;
+  traceContext?: Record<string, unknown>;
   spanParentAsLink?: boolean;
   parentAsLinkType?: "replay" | "trigger";
   batchId?: string;

apps/webapp/test/environmentVariableRules.test.ts

@@ -15,33 +15,6 @@ describe("removeBlacklistedVariables", () => {
     expect(result).toEqual([{ key: "NORMAL_VAR", value: "normal" }]);
   });
 
-  it("should remove variables with blacklisted prefixes", () => {
-    const variables: EnvironmentVariable[] = [
-      { key: "OTEL_SERVICE_NAME", value: "my-service" },
-      { key: "OTEL_TRACE_SAMPLER", value: "always_on" },
-      { key: "NORMAL_VAR", value: "normal" },
-    ];
-
-    const result = removeBlacklistedVariables(variables);
-
-    expect(result).toEqual([{ key: "NORMAL_VAR", value: "normal" }]);
-  });
-
-  it("should keep whitelisted variables even if they match a blacklisted prefix", () => {
-    const variables: EnvironmentVariable[] = [
-      { key: "OTEL_LOG_LEVEL", value: "debug" },
-      { key: "OTEL_SERVICE_NAME", value: "my-service" },
-      { key: "NORMAL_VAR", value: "normal" },
-    ];
-
-    const result = removeBlacklistedVariables(variables);
-
-    expect(result).toEqual([
-      { key: "OTEL_LOG_LEVEL", value: "debug" },
-      { key: "NORMAL_VAR", value: "normal" },
-    ]);
-  });
-
   it("should handle empty input array", () => {
     const variables: EnvironmentVariable[] = [];
 
@@ -53,8 +26,6 @@ describe("removeBlacklistedVariables", () => {
   it("should handle mixed case variables", () => {
     const variables: EnvironmentVariable[] = [
       { key: "trigger_secret_key", value: "secret123" }, // Different case
-      { key: "OTEL_LOG_LEVEL", value: "debug" },
-      { key: "otel_service_name", value: "my-service" }, // Different case
       { key: "NORMAL_VAR", value: "normal" },
     ];
 
@@ -64,8 +35,6 @@ describe("removeBlacklistedVariables", () => {
     // Note: The function is case-sensitive, so different case variables should pass through
     expect(result).toEqual([
       { key: "trigger_secret_key", value: "secret123" },
-      { key: "OTEL_LOG_LEVEL", value: "debug" },
-      { key: "otel_service_name", value: "my-service" },
       { key: "NORMAL_VAR", value: "normal" },
     ]);
   });
@@ -73,17 +42,12 @@ describe("removeBlacklistedVariables", () => {
   it("should handle variables with empty values", () => {
     const variables: EnvironmentVariable[] = [
       { key: "TRIGGER_SECRET_KEY", value: "" },
-      { key: "OTEL_SERVICE_NAME", value: "" },
-      { key: "OTEL_LOG_LEVEL", value: "" },
       { key: "NORMAL_VAR", value: "" },
     ];
 
     const result = removeBlacklistedVariables(variables);
 
-    expect(result).toEqual([
-      { key: "OTEL_LOG_LEVEL", value: "" },
-      { key: "NORMAL_VAR", value: "" },
-    ]);
+    expect(result).toEqual([{ key: "NORMAL_VAR", value: "" }]);
   });
 
   it("should handle all types of rules in a single array", () => {
@@ -91,11 +55,6 @@ describe("removeBlacklistedVariables", () => {
       // Exact matches (should be removed)
       { key: "TRIGGER_SECRET_KEY", value: "secret123" },
       { key: "TRIGGER_API_URL", value: "https://api.example.com" },
-      // Prefix matches (should be removed)
-      { key: "OTEL_SERVICE_NAME", value: "my-service" },
-      { key: "OTEL_TRACE_SAMPLER", value: "always_on" },
-      // Whitelist exception (should be kept)
-      { key: "OTEL_LOG_LEVEL", value: "debug" },
       // Normal variables (should be kept)
       { key: "NORMAL_VAR", value: "normal" },
       { key: "DATABASE_URL", value: "postgres://..." },
@@ -104,7 +63,6 @@ describe("removeBlacklistedVariables", () => {
     const result = removeBlacklistedVariables(variables);
 
     expect(result).toEqual([
-      { key: "OTEL_LOG_LEVEL", value: "debug" },
       { key: "NORMAL_VAR", value: "normal" },
       { key: "DATABASE_URL", value: "postgres://..." },
     ]);

internal-packages/run-engine/src/engine/types.ts

@@ -6,6 +6,7 @@ import {
   MachinePresetName,
   RetryOptions,
   RunChainState,
+  TriggerTraceContext,
 } from "@trigger.dev/core/v3";
 import { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database";
 import { Worker, type WorkerConcurrencyOptions } from "@trigger.dev/redis-worker";
@@ -89,7 +90,7 @@ export type TriggerParams = {
   payload: string;
   payloadType: string;
   context: any;
-  traceContext: Record<string, string | undefined>;
+  traceContext: TriggerTraceContext;
   traceId: string;
   spanId: string;
   parentSpanId?: string;

packages/cli-v3/package.json

@@ -82,17 +82,13 @@
     "@depot/cli": "0.0.1-cli.2.80.0",
     "@modelcontextprotocol/sdk": "^1.6.1",
     "@opentelemetry/api": "1.9.0",
-    "@opentelemetry/api-logs": "0.52.1",
-    "@opentelemetry/exporter-logs-otlp-http": "0.52.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.52.1",
-    "@opentelemetry/instrumentation": "0.52.1",
-    "@opentelemetry/instrumentation-fetch": "0.52.1",
-    "@opentelemetry/resources": "1.25.1",
-    "@opentelemetry/sdk-logs": "0.52.1",
-    "@opentelemetry/sdk-node": "0.52.1",
-    "@opentelemetry/sdk-trace-base": "1.25.1",
-    "@opentelemetry/sdk-trace-node": "1.25.1",
-    "@opentelemetry/semantic-conventions": "1.25.1",
+    "@opentelemetry/api-logs": "0.203.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
+    "@opentelemetry/instrumentation": "0.203.0",
+    "@opentelemetry/instrumentation-fetch": "0.203.0",
+    "@opentelemetry/resources": "2.0.1",
+    "@opentelemetry/sdk-trace-node": "2.0.1",
+    "@opentelemetry/semantic-conventions": "1.36.0",
     "@trigger.dev/build": "workspace:4.0.0-v4-beta.26",
     "@trigger.dev/core": "workspace:4.0.0-v4-beta.26",
     "ansi-escapes": "^7.0.0",

packages/cli-v3/src/cli/common.ts

@@ -1,15 +1,13 @@
-import { flattenAttributes } from "@trigger.dev/core/v3";
-import { recordSpanException } from "@trigger.dev/core/v3/workers";
+import { outro } from "@clack/prompts";
 import { Command } from "commander";
 import { z } from "zod";
-import { getTracer, provider } from "../telemetry/tracing.js";
 import { fromZodError } from "zod-validation-error";
-import { logger } from "../utilities/logger.js";
-import { outro } from "@clack/prompts";
-import { chalkError } from "../utilities/cliOutput.js";
-import { CLOUD_API_URL } from "../consts.js";
-import { readAuthConfigCurrentProfileName } from "../utilities/configFiles.js";
 import { BundleError } from "../build/bundle.js";
+import { CLOUD_API_URL } from "../consts.js";
+import { chalkError } from "../utilities/cliOutput.js";
+import { readAuthConfigCurrentProfileName } from "../utilities/configFiles.js";
+import { logger } from "../utilities/logger.js";
+import { trace } from "@opentelemetry/api";
 
 export const CommonCommandOptions = z.object({
   apiUrl: z.string().optional(),
@@ -39,69 +37,52 @@ export class OutroCommandError extends SkipCommandError {}
 export async function handleTelemetry(action: () => Promise<void>) {
   try {
     await action();
-
-    await provider?.forceFlush();
   } catch (e) {
-    await provider?.forceFlush();
-
     process.exitCode = 1;
   }
 }
 
-export const tracer = getTracer();
-
 export async function wrapCommandAction<T extends z.AnyZodObject, TResult>(
   name: string,
   schema: T,
   options: unknown,
   action: (opts: z.output<T>) => Promise<TResult>
 ): Promise<TResult | undefined> {
-  return await tracer.startActiveSpan(name, async (span) => {
-    try {
-      const parsedOptions = schema.safeParse(options);
+  try {
+    const parsedOptions = schema.safeParse(options);
 
-      if (!parsedOptions.success) {
-        throw new Error(fromZodError(parsedOptions.error).toString());
-      }
-
-      span.setAttributes({
-        ...flattenAttributes(parsedOptions.data, "cli.options"),
-      });
-
-      logger.loggerLevel = parsedOptions.data.logLevel;
-
-      logger.debug(`Running "${name}" with the following options`, {
-        options: options,
-        spanContext: span?.spanContext(),
-      });
-
-      const result = await action(parsedOptions.data);
-
-      span.end();
-
-      return result;
-    } catch (e) {
-      if (e instanceof SkipLoggingError) {
-        recordSpanException(span, e);
-      } else if (e instanceof OutroCommandError) {
-        outro("Operation cancelled");
-      } else if (e instanceof SkipCommandError) {
-        // do nothing
-      } else if (e instanceof BundleError) {
-        process.exit(1);
-      } else {
-        recordSpanException(span, e);
-
-        logger.log(`${chalkError("X Error:")} ${e instanceof Error ? e.message : String(e)}`);
-      }
-
-      span.end();
-
-      throw e;
+    if (!parsedOptions.success) {
+      throw new Error(fromZodError(parsedOptions.error).toString());
     }
-  });
+
+    logger.loggerLevel = parsedOptions.data.logLevel;
+
+    logger.debug(`Running "${name}" with the following options`, {
+      options: options,
+    });
+
+    const result = await action(parsedOptions.data);
+
+    return result;
+  } catch (e) {
+    if (e instanceof SkipLoggingError) {
+      // do nothing
+    } else if (e instanceof OutroCommandError) {
+      outro("Operation cancelled");
+    } else if (e instanceof SkipCommandError) {
+      // do nothing
+    } else if (e instanceof BundleError) {
+      process.exit(1);
+    } else {
+      logger.log(`${chalkError("X Error:")} ${e instanceof Error ? e.message : String(e)}`);
+    }
+
+    throw e;
+  }
 }
 
+export const tracer = trace.getTracer("trigger.dev/cli");
+
 export function installExitHandler() {
   process.on("SIGINT", () => {
     process.exit(0);

packages/cli-v3/src/entryPoints/dev-run-worker.ts

@@ -21,6 +21,7 @@ import {
   runtime,
   runTimelineMetrics,
   taskContext,
+  TaskRunContext,
   TaskRunErrorCodes,
   TaskRunExecution,
   timeout,
@@ -29,6 +30,7 @@ import {
   waitUntil,
   WorkerManifest,
   WorkerToExecutorMessageCatalog,
+  traceContext,
 } from "@trigger.dev/core/v3";
 import { TriggerTracer } from "@trigger.dev/core/v3/tracer";
 import {
@@ -52,6 +54,7 @@ import {
   TracingSDK,
   usage,
   UsageTimeoutManager,
+  StandardTraceContextManager,
 } from "@trigger.dev/core/v3/workers";
 import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
 import { readFile } from "node:fs/promises";
@@ -126,6 +129,9 @@ timeout.setGlobalManager(usageTimeoutManager);
 const standardResourceCatalog = new StandardResourceCatalog();
 resourceCatalog.setGlobalResourceCatalog(standardResourceCatalog);
 
+const standardTraceContextManager = new StandardTraceContextManager();
+traceContext.setGlobalManager(standardTraceContextManager);
+
 const durableClock = new DurableClock();
 clock.setGlobalClock(durableClock);
 const runMetadataManager = new StandardMetadataManager(
@@ -175,11 +181,11 @@ async function doBootstrap() {
     const { config, handleError } = await importConfig(workerManifest.configPath);
 
     const tracingSDK = new TracingSDK({
-      url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
+      url: env.TRIGGER_OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
       instrumentations: config.telemetry?.instrumentations ?? config.instrumentations ?? [],
       exporters: config.telemetry?.exporters ?? [],
       logExporters: config.telemetry?.logExporters ?? [],
-      diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
+      diagLogLevel: (env.TRIGGER_OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
       forceFlushTimeoutMillis: 30_000,
     });
 
@@ -301,6 +307,7 @@ function resetExecutionEnvironment() {
   _sharedWorkerRuntime?.reset();
   durableClock.reset();
   taskContext.disable();
+  standardTraceContextManager.reset();
 
   log(`[${new Date().toISOString()}] Reset execution environment`);
 }
@@ -337,6 +344,7 @@ const zodIpc = new ZodIpcConnection({
 
       resetExecutionEnvironment();
 
+      standardTraceContextManager.traceContext = traceContext;
       standardRunTimelineMetricsManager.registerMetricsFromExecution(metrics, isWarmStart);
 
       if (_isRunning) {
@@ -361,6 +369,14 @@ const zodIpc = new ZodIpcConnection({
         return;
       }
 
+      const ctx = TaskRunContext.parse(execution);
+
+      taskContext.setGlobalTaskContext({
+        ctx,
+        worker: metadata,
+        isWarmStart: isWarmStart ?? false,
+      });
+
       try {
         const { tracer, tracingSDK, consoleInterceptor, config, workerManifest } =
           await bootstrap();
@@ -516,7 +532,7 @@ const zodIpc = new ZodIpcConnection({
 
           const signal = AbortSignal.any([_cancelController.signal, timeoutController.signal]);
 
-          const { result } = await executor.execute(execution, metadata, traceContext, signal);
+          const { result } = await executor.execute(execution, ctx, signal);
 
           if (_isRunning && !_isCancelled) {
             const usageSample = usage.stop(_executionMeasurement);

packages/cli-v3/src/entryPoints/managed-run-worker.ts

@@ -20,6 +20,7 @@ import {
   runtime,
   runTimelineMetrics,
   taskContext,
+  TaskRunContext,
   TaskRunErrorCodes,
   TaskRunExecution,
   timeout,
@@ -28,6 +29,7 @@ import {
   waitUntil,
   WorkerManifest,
   WorkerToExecutorMessageCatalog,
+  traceContext,
 } from "@trigger.dev/core/v3";
 import { TriggerTracer } from "@trigger.dev/core/v3/tracer";
 import {
@@ -52,6 +54,7 @@ import {
   TracingSDK,
   usage,
   UsageTimeoutManager,
+  StandardTraceContextManager,
 } from "@trigger.dev/core/v3/workers";
 import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
 import { readFile } from "node:fs/promises";
@@ -119,6 +122,9 @@ resourceCatalog.setGlobalResourceCatalog(standardResourceCatalog);
 const durableClock = new DurableClock();
 clock.setGlobalClock(durableClock);
 
+const standardTraceContextManager = new StandardTraceContextManager();
+traceContext.setGlobalManager(standardTraceContextManager);
+
 const runMetadataManager = new StandardMetadataManager(
   apiClientManager.clientOrThrow(),
   getEnvVar("TRIGGER_STREAM_URL", getEnvVar("TRIGGER_API_URL")) ?? "https://api.trigger.dev"
@@ -166,9 +172,9 @@ async function doBootstrap() {
     );
 
     const tracingSDK = new TracingSDK({
-      url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
+      url: env.TRIGGER_OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
       instrumentations: config.instrumentations ?? [],
-      diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
+      diagLogLevel: (env.TRIGGER_OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
       forceFlushTimeoutMillis: 30_000,
       exporters: config.telemetry?.exporters ?? [],
       logExporters: config.telemetry?.logExporters ?? [],
@@ -287,6 +293,7 @@ function resetExecutionEnvironment() {
   _sharedWorkerRuntime?.reset();
   durableClock.reset();
   taskContext.disable();
+  standardTraceContextManager.reset();
 
   console.log(`[${new Date().toISOString()}] Reset execution environment`);
 }
@@ -328,6 +335,8 @@ const zodIpc = new ZodIpcConnection({
 
       resetExecutionEnvironment();
 
+      standardTraceContextManager.traceContext = traceContext;
+
       const prodManager = initializeUsageManager({
         usageIntervalMs: getEnvVar("USAGE_HEARTBEAT_INTERVAL_MS"),
         usageEventUrl: getEnvVar("USAGE_EVENT_URL"),
@@ -365,6 +374,14 @@ const zodIpc = new ZodIpcConnection({
         return;
       }
 
+      const ctx = TaskRunContext.parse(execution);
+
+      taskContext.setGlobalTaskContext({
+        ctx,
+        worker: metadata,
+        isWarmStart: isWarmStart ?? false,
+      });
+
       try {
         const { tracer, tracingSDK, consoleInterceptor, config, workerManifest } =
           await bootstrap();
@@ -514,7 +531,7 @@ const zodIpc = new ZodIpcConnection({
 
           const signal = AbortSignal.any([_cancelController.signal, timeoutController.signal]);
 
-          const { result } = await executor.execute(execution, metadata, traceContext, signal);
+          const { result } = await executor.execute(execution, ctx, signal);
 
           if (_isRunning && !_isCancelled) {
             const usageSample = usage.stop(_executionMeasurement);

packages/cli-v3/src/telemetry/tracing.ts

@@ -1,75 +0,0 @@
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
-import { registerInstrumentations } from "@opentelemetry/instrumentation";
-import { Resource, detectResourcesSync, processDetectorSync } from "@opentelemetry/resources";
-import { NodeTracerProvider, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-node";
-import { FetchInstrumentation } from "@opentelemetry/instrumentation-fetch";
-import { DiagConsoleLogger, DiagLogLevel, diag, trace } from "@opentelemetry/api";
-import {
-  SEMRESATTRS_SERVICE_NAME,
-  SEMRESATTRS_SERVICE_VERSION,
-} from "@opentelemetry/semantic-conventions";
-import { logger } from "../utilities/logger.js";
-import { VERSION } from "../version.js";
-import { env } from "std-env";
-
-function initializeTracing(): NodeTracerProvider | undefined {
-  if (
-    process.argv.includes("--skip-telemetry") ||
-    env.TRIGGER_DEV_SKIP_TELEMETRY || // only for backwards compat
-    env.TRIGGER_TELEMETRY_DISABLED
-  ) {
-    logger.debug("📉 Telemetry disabled");
-    return;
-  }
-
-  if (env.OTEL_INTERNAL_DIAG_DEBUG) {
-    diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);
-  }
-
-  const resource = detectResourcesSync({
-    detectors: [processDetectorSync],
-  }).merge(
-    new Resource({
-      [SEMRESATTRS_SERVICE_NAME]: "trigger.dev cli v3",
-      [SEMRESATTRS_SERVICE_VERSION]: VERSION,
-    })
-  );
-
-  const traceProvider = new NodeTracerProvider({
-    forceFlushTimeoutMillis: 30_000,
-    resource,
-    spanLimits: {
-      attributeCountLimit: 1000,
-      attributeValueLengthLimit: 2048,
-      eventCountLimit: 100,
-      attributePerEventCountLimit: 100,
-      linkCountLimit: 10,
-      attributePerLinkCountLimit: 100,
-    },
-  });
-
-  const spanExporter = new OTLPTraceExporter({
-    url: "https://otel.baselime.io/v1",
-    timeoutMillis: 5000,
-    headers: {
-      "x-api-key": "b6e0fbbaf8dc2524773d2152ae2e9eb5c7fbaa52",
-    },
-  });
-
-  const spanProcessor = new SimpleSpanProcessor(spanExporter);
-
-  traceProvider.addSpanProcessor(spanProcessor);
-  traceProvider.register();
-
-  registerInstrumentations({
-    instrumentations: [new FetchInstrumentation()],
-  });
-
-  return traceProvider;
-}
-
-export const provider = initializeTracing();
-
-export function getTracer() {
-  return trace.getTracer("trigger.dev cli v3", VERSION);
-}

packages/cli-v3/src/utilities/session.ts

@@ -1,12 +1,9 @@
 import { recordSpanException } from "@trigger.dev/core/v3/workers";
 import { CliApiClient } from "../apiClient.js";
 import { readAuthConfigProfile } from "./configFiles.js";
-import { getTracer } from "../telemetry/tracing.js";
 import { logger } from "./logger.js";
 import { GitMeta } from "@trigger.dev/core/v3";
 
-const tracer = getTracer();
-
 export type LoginResultOk = {
   ok: true;
   profile: string;
@@ -31,63 +28,44 @@ export type LoginResult =
     };
 
 export async function isLoggedIn(profile: string = "default"): Promise<LoginResult> {
-  return await tracer.startActiveSpan("isLoggedIn", async (span) => {
-    try {
-      const config = readAuthConfigProfile(profile);
+  try {
+    const config = readAuthConfigProfile(profile);
 
-      if (!config?.accessToken || !config?.apiUrl) {
-        span.recordException(new Error("You must login first"));
-        span.end();
-        return { ok: false as const, error: "You must login first" };
-      }
+    if (!config?.accessToken || !config?.apiUrl) {
+      return { ok: false as const, error: "You must login first" };
+    }
 
-      const apiClient = new CliApiClient(config.apiUrl, config.accessToken);
-      const userData = await apiClient.whoAmI();
-
-      if (!userData.success) {
-        recordSpanException(span, userData.error);
-        span.end();
-
-        return {
-          ok: false as const,
-          error: userData.error,
-          auth: {
-            apiUrl: config.apiUrl,
-            accessToken: config.accessToken,
-          },
-        };
-      }
-
-      span.setAttributes({
-        "login.userId": userData.data.userId,
-        "login.email": userData.data.email,
-        "login.dashboardUrl": userData.data.dashboardUrl,
-        "login.profile": profile,
-      });
-
-      span.end();
+    const apiClient = new CliApiClient(config.apiUrl, config.accessToken);
+    const userData = await apiClient.whoAmI();
 
+    if (!userData.success) {
       return {
-        ok: true as const,
-        profile,
-        userId: userData.data.userId,
-        email: userData.data.email,
-        dashboardUrl: userData.data.dashboardUrl,
+        ok: false as const,
+        error: userData.error,
         auth: {
           apiUrl: config.apiUrl,
           accessToken: config.accessToken,
         },
       };
-    } catch (e) {
-      recordSpanException(span, e);
-      span.end();
-
-      return {
-        ok: false as const,
-        error: e instanceof Error ? e.message : "Unknown error",
-      };
     }
-  });
+
+    return {
+      ok: true as const,
+      profile,
+      userId: userData.data.userId,
+      email: userData.data.email,
+      dashboardUrl: userData.data.dashboardUrl,
+      auth: {
+        apiUrl: config.apiUrl,
+        accessToken: config.accessToken,
+      },
+    };
+  } catch (e) {
+    return {
+      ok: false as const,
+      error: e instanceof Error ? e.message : "Unknown error",
+    };
+  }
 }
 
 export type GetEnvOptions = {

packages/core/package.json

@@ -171,17 +171,16 @@
     "@google-cloud/precise-date": "^4.0.0",
     "@jsonhero/path": "^1.0.21",
     "@opentelemetry/api": "1.9.0",
-    "@opentelemetry/api-logs": "0.52.1",
-    "@opentelemetry/core": "^1.30.1",
-    "@opentelemetry/exporter-logs-otlp-http": "0.52.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.52.1",
-    "@opentelemetry/instrumentation": "0.52.1",
-    "@opentelemetry/resources": "1.25.1",
-    "@opentelemetry/sdk-logs": "0.52.1",
-    "@opentelemetry/sdk-node": "0.52.1",
-    "@opentelemetry/sdk-trace-base": "1.25.1",
-    "@opentelemetry/sdk-trace-node": "1.25.1",
-    "@opentelemetry/semantic-conventions": "1.25.1",
+    "@opentelemetry/api-logs": "0.203.0",
+    "@opentelemetry/core": "2.0.1",
+    "@opentelemetry/exporter-logs-otlp-http": "0.203.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
+    "@opentelemetry/instrumentation": "0.203.0",
+    "@opentelemetry/resources": "2.0.1",
+    "@opentelemetry/sdk-logs": "0.203.0",
+    "@opentelemetry/sdk-trace-base": "2.0.1",
+    "@opentelemetry/sdk-trace-node": "2.0.1",
+    "@opentelemetry/semantic-conventions": "1.36.0",
     "dequal": "^2.0.3",
     "eventsource": "^3.0.5",
     "eventsource-parser": "^3.0.0",

packages/core/src/v3/apiClient/core.ts

@@ -4,7 +4,7 @@ import { RetryOptions } from "../schemas/index.js";
 import { calculateNextRetryDelay } from "../utils/retries.js";
 import { ApiConnectionError, ApiError, ApiSchemaValidationError } from "./errors.js";
 
-import { Attributes, context, propagation, Span } from "@opentelemetry/api";
+import { Attributes, context, propagation, Span, trace } from "@opentelemetry/api";
 import { suppressTracing } from "@opentelemetry/core";
 import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
 import type { TriggerTracer } from "../tracer.js";
@@ -617,10 +617,6 @@ export function hasOwn(obj: Object, key: string): boolean {
 function injectPropagationHeadersIfInWorker(requestInit?: RequestInit): RequestInit | undefined {
   const headers = new Headers(requestInit?.headers);
 
-  if (headers.get("x-trigger-worker") !== "true") {
-    return requestInit;
-  }
-
   const headersObject = Object.fromEntries(headers.entries());
 
   propagation.inject(context.active(), headersObject);

packages/core/src/v3/index.ts

@@ -9,6 +9,7 @@ export * from "./limits.js";
 export * from "./logger-api.js";
 export * from "./runtime-api.js";
 export * from "./task-context-api.js";
+export * from "./trace-context-api.js";
 export * from "./apiClientManager-api.js";
 export * from "./usage-api.js";
 export * from "./run-metadata-api.js";

packages/core/src/v3/isomorphic/index.ts

@@ -3,3 +3,4 @@ export * from "./duration.js";
 export * from "./maxDuration.js";
 export * from "./queueName.js";
 export * from "./consts.js";
+export * from "./traceContext.js";

packages/core/src/v3/isomorphic/traceContext.ts

@@ -0,0 +1,29 @@
+export function parseTraceparent(
+  traceparent?: string
+): { traceId: string; spanId: string } | undefined {
+  if (!traceparent) {
+    return undefined;
+  }
+
+  const parts = traceparent.split("-");
+
+  if (parts.length !== 4) {
+    return undefined;
+  }
+
+  const [version, traceId, spanId] = parts;
+
+  if (version !== "00") {
+    return undefined;
+  }
+
+  if (!traceId || !spanId) {
+    return undefined;
+  }
+
+  return { traceId, spanId };
+}
+
+export function serializeTraceparent(traceId: string, spanId: string) {
+  return `00-${traceId}-${spanId}-01`;
+}

packages/core/src/v3/otel/tracingSDK.ts

@@ -1,25 +1,19 @@
 import { DiagConsoleLogger, DiagLogLevel, TracerProvider, diag } from "@opentelemetry/api";
-import { RandomIdGenerator } from "@opentelemetry/sdk-trace-base";
 import { logs } from "@opentelemetry/api-logs";
+import { TraceState } from "@opentelemetry/core";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { registerInstrumentations, type Instrumentation } from "@opentelemetry/instrumentation";
-import {
-  DetectorSync,
-  IResource,
-  Resource,
-  ResourceAttributes,
-  ResourceDetectionConfig,
-  detectResourcesSync,
-  processDetectorSync,
-} from "@opentelemetry/resources";
+import { detectResources, processDetector, resourceFromAttributes } from "@opentelemetry/resources";
 import {
   BatchLogRecordProcessor,
-  LoggerProvider,
   LogRecordExporter,
+  LogRecordProcessor,
+  LoggerProvider,
   ReadableLogRecord,
   SimpleLogRecordProcessor,
 } from "@opentelemetry/sdk-logs";
+import { RandomIdGenerator, SpanProcessor } from "@opentelemetry/sdk-trace-base";
 import {
   BatchSpanProcessor,
   NodeTracerProvider,
@@ -40,45 +34,14 @@ import {
   OTEL_SPAN_EVENT_COUNT_LIMIT,
 } from "../limits.js";
 import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
+import { taskContext } from "../task-context-api.js";
 import {
   TaskContextLogProcessor,
   TaskContextSpanProcessor,
 } from "../taskContext/otelProcessors.js";
+import { traceContext } from "../trace-context-api.js";
 import { getEnvVar } from "../utils/getEnv.js";
 
-class AsyncResourceDetector implements DetectorSync {
-  private _promise: Promise<ResourceAttributes>;
-  private _resolver?: (value: ResourceAttributes) => void;
-  private _resolved: boolean = false;
-
-  constructor() {
-    this._promise = new Promise((resolver) => {
-      this._resolver = resolver;
-    });
-  }
-
-  get isResolved() {
-    return this._resolved;
-  }
-
-  detect(_config?: ResourceDetectionConfig): Resource {
-    return new Resource({}, this._promise);
-  }
-
-  resolveWithAttributes(attributes: ResourceAttributes) {
-    if (!this._resolver) {
-      throw new Error("Resolver not available");
-    }
-
-    if (this._resolved) {
-      return;
-    }
-
-    this._resolved = true;
-    this._resolver(attributes);
-  }
-}
-
 export type TracingDiagnosticLogLevel =
   | "none"
   | "error"
@@ -91,7 +54,6 @@ export type TracingDiagnosticLogLevel =
 export type TracingSDKConfig = {
   url: string;
   forceFlushTimeoutMillis?: number;
-  resource?: IResource;
   instrumentations?: Instrumentation[];
   exporters?: SpanExporter[];
   logExporters?: LogRecordExporter[];
@@ -101,7 +63,6 @@ export type TracingSDKConfig = {
 const idGenerator = new RandomIdGenerator();
 
 export class TracingSDK {
-  public readonly asyncResourceDetector = new AsyncResourceDetector();
   private readonly _logProvider: LoggerProvider;
   private readonly _spanExporter: SpanExporter;
   private readonly _traceProvider: NodeTracerProvider;
@@ -112,27 +73,81 @@ export class TracingSDK {
   constructor(private readonly config: TracingSDKConfig) {
     setLogLevel(config.diagLogLevel ?? "none");
 
-    const envResourceAttributesSerialized = getEnvVar("OTEL_RESOURCE_ATTRIBUTES");
+    const envResourceAttributesSerialized = getEnvVar("TRIGGER_OTEL_RESOURCE_ATTRIBUTES");
     const envResourceAttributes = envResourceAttributesSerialized
       ? JSON.parse(envResourceAttributesSerialized)
       : {};
 
-    const commonResources = detectResourcesSync({
-      detectors: [this.asyncResourceDetector, processDetectorSync],
+    const commonResources = detectResources({
+      detectors: [processDetector],
     })
       .merge(
-        new Resource({
+        resourceFromAttributes({
           [SemanticResourceAttributes.CLOUD_PROVIDER]: "trigger.dev",
           [SemanticResourceAttributes.SERVICE_NAME]:
-            getEnvVar("OTEL_SERVICE_NAME") ?? "trigger.dev",
+            getEnvVar("TRIGGER_OTEL_SERVICE_NAME") ?? "trigger.dev",
           [SemanticInternalAttributes.TRIGGER]: true,
           [SemanticInternalAttributes.CLI_VERSION]: VERSION,
           [SemanticInternalAttributes.SDK_VERSION]: VERSION,
           [SemanticInternalAttributes.SDK_LANGUAGE]: "typescript",
         })
       )
-      .merge(config.resource ?? new Resource({}))
-      .merge(new Resource(envResourceAttributes));
+      .merge(resourceFromAttributes(envResourceAttributes))
+      .merge(resourceFromAttributes(taskContext.resourceAttributes));
+
+    const spanExporter = new OTLPTraceExporter({
+      url: `${config.url}/v1/traces`,
+      timeoutMillis: config.forceFlushTimeoutMillis,
+    });
+
+    const spanProcessors: Array<SpanProcessor> = [];
+
+    spanProcessors.push(
+      new TaskContextSpanProcessor(
+        VERSION,
+        getEnvVar("TRIGGER_OTEL_BATCH_PROCESSING_ENABLED") === "1"
+          ? new BatchSpanProcessor(spanExporter, {
+              maxExportBatchSize: parseInt(
+                getEnvVar("TRIGGER_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE") ?? "64"
+              ),
+              scheduledDelayMillis: parseInt(
+                getEnvVar("TRIGGER_OTEL_SPAN_SCHEDULED_DELAY_MILLIS") ?? "200"
+              ),
+              exportTimeoutMillis: parseInt(
+                getEnvVar("TRIGGER_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS") ?? "30000"
+              ),
+              maxQueueSize: parseInt(getEnvVar("TRIGGER_OTEL_SPAN_MAX_QUEUE_SIZE") ?? "512"),
+            })
+          : new SimpleSpanProcessor(spanExporter)
+      )
+    );
+
+    const externalTraceId = idGenerator.generateTraceId();
+    const externalTraceContext = traceContext.getExternalTraceContext();
+
+    for (const exporter of config.exporters ?? []) {
+      spanProcessors.push(
+        getEnvVar("TRIGGER_OTEL_BATCH_PROCESSING_ENABLED") === "1"
+          ? new BatchSpanProcessor(
+              new ExternalSpanExporterWrapper(exporter, externalTraceId, externalTraceContext),
+              {
+                maxExportBatchSize: parseInt(
+                  getEnvVar("TRIGGER_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE") ?? "64"
+                ),
+                scheduledDelayMillis: parseInt(
+                  getEnvVar("TRIGGER_OTEL_SPAN_SCHEDULED_DELAY_MILLIS") ?? "200"
+                ),
+                exportTimeoutMillis: parseInt(
+                  getEnvVar("TRIGGER_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS") ?? "30000"
+                ),
+                maxQueueSize: parseInt(getEnvVar("TRIGGER_OTEL_SPAN_MAX_QUEUE_SIZE") ?? "512"),
+              }
+            )
+          : new SimpleSpanProcessor(
+              new ExternalSpanExporterWrapper(exporter, externalTraceId, externalTraceContext)
+            )
+      );
+    }
 
     const traceProvider = new NodeTracerProvider({
       forceFlushTimeoutMillis: config.forceFlushTimeoutMillis,
@@ -145,50 +160,9 @@ export class TracingSDK {
         linkCountLimit: OTEL_LINK_COUNT_LIMIT,
         attributePerLinkCountLimit: OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT,
       },
+      spanProcessors,
     });
 
-    const spanExporter = new OTLPTraceExporter({
-      url: `${config.url}/v1/traces`,
-      timeoutMillis: config.forceFlushTimeoutMillis,
-    });
-
-    traceProvider.addSpanProcessor(
-      new TaskContextSpanProcessor(
-        traceProvider.getTracer("trigger-dev-worker", VERSION),
-        getEnvVar("OTEL_BATCH_PROCESSING_ENABLED") === "1"
-          ? new BatchSpanProcessor(spanExporter, {
-              maxExportBatchSize: parseInt(getEnvVar("OTEL_SPAN_MAX_EXPORT_BATCH_SIZE") ?? "64"),
-              scheduledDelayMillis: parseInt(
-                getEnvVar("OTEL_SPAN_SCHEDULED_DELAY_MILLIS") ?? "200"
-              ),
-              exportTimeoutMillis: parseInt(
-                getEnvVar("OTEL_SPAN_EXPORT_TIMEOUT_MILLIS") ?? "30000"
-              ),
-              maxQueueSize: parseInt(getEnvVar("OTEL_SPAN_MAX_QUEUE_SIZE") ?? "512"),
-            })
-          : new SimpleSpanProcessor(spanExporter)
-      )
-    );
-
-    const externalTraceId = idGenerator.generateTraceId();
-
-    for (const exporter of config.exporters ?? []) {
-      traceProvider.addSpanProcessor(
-        getEnvVar("OTEL_BATCH_PROCESSING_ENABLED") === "1"
-          ? new BatchSpanProcessor(new ExternalSpanExporterWrapper(exporter, externalTraceId), {
-              maxExportBatchSize: parseInt(getEnvVar("OTEL_SPAN_MAX_EXPORT_BATCH_SIZE") ?? "64"),
-              scheduledDelayMillis: parseInt(
-                getEnvVar("OTEL_SPAN_SCHEDULED_DELAY_MILLIS") ?? "200"
-              ),
-              exportTimeoutMillis: parseInt(
-                getEnvVar("OTEL_SPAN_EXPORT_TIMEOUT_MILLIS") ?? "30000"
-              ),
-              maxQueueSize: parseInt(getEnvVar("OTEL_SPAN_MAX_QUEUE_SIZE") ?? "512"),
-            })
-          : new SimpleSpanProcessor(new ExternalSpanExporterWrapper(exporter, externalTraceId))
-      );
-    }
-
     traceProvider.register();
 
     registerInstrumentations({
@@ -200,6 +174,57 @@ export class TracingSDK {
       url: `${config.url}/v1/logs`,
     });
 
+    const logProcessors: Array<LogRecordProcessor> = [
+      new TaskContextLogProcessor(
+        getEnvVar("TRIGGER_OTEL_BATCH_PROCESSING_ENABLED") === "1"
+          ? new BatchLogRecordProcessor(logExporter, {
+              maxExportBatchSize: parseInt(
+                getEnvVar("TRIGGER_OTEL_LOG_MAX_EXPORT_BATCH_SIZE") ?? "64"
+              ),
+              scheduledDelayMillis: parseInt(
+                getEnvVar("TRIGGER_OTEL_LOG_SCHEDULED_DELAY_MILLIS") ?? "200"
+              ),
+              exportTimeoutMillis: parseInt(
+                getEnvVar("TRIGGER_OTEL_LOG_EXPORT_TIMEOUT_MILLIS") ?? "30000"
+              ),
+              maxQueueSize: parseInt(getEnvVar("TRIGGER_OTEL_LOG_MAX_QUEUE_SIZE") ?? "512"),
+            })
+          : new SimpleLogRecordProcessor(logExporter)
+      ),
+    ];
+
+    for (const externalLogExporter of config.logExporters ?? []) {
+      logProcessors.push(
+        getEnvVar("TRIGGER_OTEL_BATCH_PROCESSING_ENABLED") === "1"
+          ? new BatchLogRecordProcessor(
+              new ExternalLogRecordExporterWrapper(
+                externalLogExporter,
+                externalTraceId,
+                externalTraceContext
+              ),
+              {
+                maxExportBatchSize: parseInt(
+                  getEnvVar("TRIGGER_OTEL_LOG_MAX_EXPORT_BATCH_SIZE") ?? "64"
+                ),
+                scheduledDelayMillis: parseInt(
+                  getEnvVar("TRIGGER_OTEL_LOG_SCHEDULED_DELAY_MILLIS") ?? "200"
+                ),
+                exportTimeoutMillis: parseInt(
+                  getEnvVar("TRIGGER_OTEL_LOG_EXPORT_TIMEOUT_MILLIS") ?? "30000"
+                ),
+                maxQueueSize: parseInt(getEnvVar("TRIGGER_OTEL_LOG_MAX_QUEUE_SIZE") ?? "512"),
+              }
+            )
+          : new SimpleLogRecordProcessor(
+              new ExternalLogRecordExporterWrapper(
+                externalLogExporter,
+                externalTraceId,
+                externalTraceContext
+              )
+            )
+      );
+    }
+
     // To start a logger, you first need to initialize the Logger provider.
     const loggerProvider = new LoggerProvider({
       resource: commonResources,
@@ -207,43 +232,9 @@ export class TracingSDK {
         attributeCountLimit: OTEL_LOG_ATTRIBUTE_COUNT_LIMIT,
         attributeValueLengthLimit: OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT,
       },
+      processors: logProcessors,
     });
 
-    loggerProvider.addLogRecordProcessor(
-      new TaskContextLogProcessor(
-        getEnvVar("OTEL_BATCH_PROCESSING_ENABLED") === "1"
-          ? new BatchLogRecordProcessor(logExporter, {
-              maxExportBatchSize: parseInt(getEnvVar("OTEL_LOG_MAX_EXPORT_BATCH_SIZE") ?? "64"),
-              scheduledDelayMillis: parseInt(getEnvVar("OTEL_LOG_SCHEDULED_DELAY_MILLIS") ?? "200"),
-              exportTimeoutMillis: parseInt(getEnvVar("OTEL_LOG_EXPORT_TIMEOUT_MILLIS") ?? "30000"),
-              maxQueueSize: parseInt(getEnvVar("OTEL_LOG_MAX_QUEUE_SIZE") ?? "512"),
-            })
-          : new SimpleLogRecordProcessor(logExporter)
-      )
-    );
-
-    for (const externalLogExporter of config.logExporters ?? []) {
-      loggerProvider.addLogRecordProcessor(
-        getEnvVar("OTEL_BATCH_PROCESSING_ENABLED") === "1"
-          ? new BatchLogRecordProcessor(
-              new ExternalLogRecordExporterWrapper(externalLogExporter, externalTraceId),
-              {
-                maxExportBatchSize: parseInt(getEnvVar("OTEL_LOG_MAX_EXPORT_BATCH_SIZE") ?? "64"),
-                scheduledDelayMillis: parseInt(
-                  getEnvVar("OTEL_LOG_SCHEDULED_DELAY_MILLIS") ?? "200"
-                ),
-                exportTimeoutMillis: parseInt(
-                  getEnvVar("OTEL_LOG_EXPORT_TIMEOUT_MILLIS") ?? "30000"
-                ),
-                maxQueueSize: parseInt(getEnvVar("OTEL_LOG_MAX_QUEUE_SIZE") ?? "512"),
-              }
-            )
-          : new SimpleLogRecordProcessor(
-              new ExternalLogRecordExporterWrapper(externalLogExporter, externalTraceId)
-            )
-      );
-    }
-
     this._logProvider = loggerProvider;
     this._spanExporter = spanExporter;
     this._traceProvider = traceProvider;
@@ -298,7 +289,10 @@ function setLogLevel(level: TracingDiagnosticLogLevel) {
 class ExternalSpanExporterWrapper {
   constructor(
     private underlyingExporter: SpanExporter,
-    private externalTraceId: string
+    private externalTraceId: string,
+    private externalTraceContext:
+      | { traceId: string; spanId: string; tracestate?: string }
+      | undefined
   ) {}
 
   private transformSpan(span: ReadableSpan): ReadableSpan | undefined {
@@ -307,14 +301,38 @@ class ExternalSpanExporterWrapper {
       return;
     }
 
+    const externalTraceId = this.externalTraceContext
+      ? this.externalTraceContext.traceId
+      : this.externalTraceId;
+
+    const isAttemptSpan = span.attributes[SemanticInternalAttributes.SPAN_ATTEMPT];
+
     const spanContext = span.spanContext();
+    let parentSpanContext = span.parentSpanContext;
+
+    if (parentSpanContext) {
+      parentSpanContext = {
+        ...parentSpanContext,
+        traceId: externalTraceId,
+      };
+    }
+
+    if (isAttemptSpan && this.externalTraceContext) {
+      parentSpanContext = {
+        ...parentSpanContext,
+        traceId: externalTraceId,
+        spanId: this.externalTraceContext.spanId,
+        traceState: this.externalTraceContext.tracestate
+          ? new TraceState(this.externalTraceContext.tracestate)
+          : undefined,
+        traceFlags: parentSpanContext?.traceFlags ?? 0,
+      };
+    }
 
     return {
       ...span,
-      spanContext: () => ({ ...spanContext, traceId: this.externalTraceId }),
-      parentSpanId: span.attributes[SemanticInternalAttributes.SPAN_ATTEMPT]
-        ? undefined
-        : span.parentSpanId,
+      spanContext: () => ({ ...spanContext, traceId: externalTraceId }),
+      parentSpanContext,
     };
   }
 
@@ -344,7 +362,10 @@ class ExternalSpanExporterWrapper {
 class ExternalLogRecordExporterWrapper {
   constructor(
     private underlyingExporter: LogRecordExporter,
-    private externalTraceId: string
+    private externalTraceId: string,
+    private externalTraceContext:
+      | { traceId: string; spanId: string; tracestate?: string }
+      | undefined
   ) {}
 
   export(logs: any[], resultCallback: (result: any) => void): void {
@@ -359,12 +380,14 @@ class ExternalLogRecordExporterWrapper {
 
   transformLogRecord(logRecord: ReadableLogRecord): ReadableLogRecord {
     // If there's no spanContext, or if the externalTraceId is not set, return the original logRecord.
-    if (!logRecord.spanContext || !this.externalTraceId) {
+    if (!logRecord.spanContext || !this.externalTraceId || !this.externalTraceContext) {
       return logRecord;
     }
 
     // Capture externalTraceId for use within the proxy's scope.
-    const { externalTraceId } = this;
+    const externalTraceId = this.externalTraceContext
+      ? this.externalTraceContext.traceId
+      : this.externalTraceId;
 
     return new Proxy(logRecord, {
       get(target, prop, receiver) {

packages/core/src/v3/schemas/schemas.ts

@@ -296,3 +296,16 @@ export const RunChainState = z.object({
 });
 
 export type RunChainState = z.infer<typeof RunChainState>;
+
+export const TriggerTraceContext = z.object({
+  traceparent: z.string().optional(),
+  tracestate: z.string().optional(),
+  external: z
+    .object({
+      traceparent: z.string().optional(),
+      tracestate: z.string().optional(),
+    })
+    .optional(),
+});
+
+export type TriggerTraceContext = z.infer<typeof TriggerTraceContext>;

packages/core/src/v3/taskContext/otelProcessors.ts

@@ -1,17 +1,16 @@
-import { LogRecord, LogRecordProcessor } from "@opentelemetry/sdk-logs";
+import { Context, trace, Tracer } from "@opentelemetry/api";
+import { LogRecordProcessor, SdkLogRecord } from "@opentelemetry/sdk-logs";
 import { Span, SpanProcessor } from "@opentelemetry/sdk-trace-base";
 import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
-import { Context } from "@opentelemetry/api";
-import { flattenAttributes } from "../utils/flattenAttributes.js";
 import { taskContext } from "../task-context-api.js";
-import { Tracer } from "@opentelemetry/api";
+import { flattenAttributes } from "../utils/flattenAttributes.js";
 
 export class TaskContextSpanProcessor implements SpanProcessor {
   private _innerProcessor: SpanProcessor;
   private _tracer: Tracer;
 
-  constructor(tracer: Tracer, innerProcessor: SpanProcessor) {
-    this._tracer = tracer;
+  constructor(version: string, innerProcessor: SpanProcessor) {
+    this._tracer = trace.getTracer("trigger-dev-worker", version);
     this._innerProcessor = innerProcessor;
   }
 
@@ -91,7 +90,7 @@ export class TaskContextLogProcessor implements LogRecordProcessor {
   forceFlush(): Promise<void> {
     return this._innerProcessor.forceFlush();
   }
-  onEmit(logRecord: LogRecord, context?: Context | undefined): void {
+  onEmit(logRecord: SdkLogRecord, context?: Context | undefined): void {
     // Adds in the context attributes to the log record
     if (taskContext.ctx) {
       logRecord.setAttributes(

packages/core/src/v3/trace-context-api.ts

@@ -0,0 +1,5 @@
+// Split module-level variable definition into separate files to allow
+// tree-shaking on each api instance.
+import { TraceContextAPI } from "./traceContext/api.js";
+/** Entrypoint for trace context API */
+export const traceContext = TraceContextAPI.getInstance();

packages/core/src/v3/traceContext/api.ts

@@ -0,0 +1,74 @@
+import { context, Context } from "@opentelemetry/api";
+import { getGlobal, registerGlobal, unregisterGlobal } from "../utils/globals.js";
+import { TraceContextManager } from "./types.js";
+
+const API_NAME = "trace-context";
+
+class NoopTraceContextManager implements TraceContextManager {
+  getTraceContext() {
+    return {};
+  }
+
+  reset() {}
+
+  getExternalTraceContext() {
+    return undefined;
+  }
+
+  extractContext(): Context {
+    return context.active();
+  }
+
+  withExternalTrace<T>(fn: () => T): T {
+    return fn();
+  }
+}
+
+const NOOP_TRACE_CONTEXT_MANAGER = new NoopTraceContextManager();
+
+export class TraceContextAPI implements TraceContextManager {
+  private static _instance?: TraceContextAPI;
+
+  private constructor() {}
+
+  public static getInstance(): TraceContextAPI {
+    if (!this._instance) {
+      this._instance = new TraceContextAPI();
+    }
+
+    return this._instance;
+  }
+
+  public setGlobalManager(manager: TraceContextManager): boolean {
+    return registerGlobal(API_NAME, manager);
+  }
+
+  public disable() {
+    unregisterGlobal(API_NAME);
+  }
+
+  public reset() {
+    this.#getManager().reset();
+    this.disable();
+  }
+
+  public getTraceContext() {
+    return this.#getManager().getTraceContext();
+  }
+
+  public getExternalTraceContext() {
+    return this.#getManager().getExternalTraceContext();
+  }
+
+  public extractContext() {
+    return this.#getManager().extractContext();
+  }
+
+  public withExternalTrace<T>(fn: () => T): T {
+    return this.#getManager().withExternalTrace(fn);
+  }
+
+  #getManager(): TraceContextManager {
+    return getGlobal(API_NAME) ?? NOOP_TRACE_CONTEXT_MANAGER;
+  }
+}

packages/core/src/v3/traceContext/manager.ts

@@ -0,0 +1,77 @@
+import { Context, context, propagation, trace, TraceFlags } from "@opentelemetry/api";
+import { TraceContextManager } from "./types.js";
+
+export class StandardTraceContextManager implements TraceContextManager {
+  public traceContext: Record<string, unknown> = {};
+
+  getTraceContext() {
+    return this.traceContext;
+  }
+
+  reset() {
+    this.traceContext = {};
+  }
+
+  getExternalTraceContext() {
+    return extractExternalTraceContext(this.traceContext?.external);
+  }
+
+  extractContext(): Context {
+    return propagation.extract(context.active(), this.traceContext ?? {});
+  }
+
+  withExternalTrace<T>(fn: () => T): T {
+    const externalTraceContext = this.getExternalTraceContext();
+
+    if (!externalTraceContext) {
+      return fn();
+    }
+
+    // Get the current active span context to extract the span ID
+    const currentSpanContext = trace.getActiveSpan()?.spanContext();
+
+    if (!currentSpanContext) {
+      throw new Error(
+        "No active span found. withExternalSpan must be called within an active span context."
+      );
+    }
+
+    const spanContext = {
+      traceId: externalTraceContext.traceId,
+      spanId: currentSpanContext.spanId,
+      traceFlags: TraceFlags.SAMPLED,
+      isRemote: true,
+    };
+
+    const contextWithSpan = trace.setSpanContext(context.active(), spanContext);
+
+    return context.with(contextWithSpan, fn);
+  }
+}
+
+function extractExternalTraceContext(traceContext: unknown) {
+  if (typeof traceContext !== "object" || traceContext === null) {
+    return undefined;
+  }
+
+  const tracestate =
+    "tracestate" in traceContext && typeof traceContext.tracestate === "string"
+      ? traceContext.tracestate
+      : undefined;
+
+  if ("traceparent" in traceContext && typeof traceContext.traceparent === "string") {
+    const [version, traceId, spanId] = traceContext.traceparent.split("-");
+
+    if (!traceId || !spanId) {
+      return undefined;
+    }
+
+    return {
+      traceId,
+      spanId,
+      tracestate: tracestate,
+    };
+  }
+
+  return undefined;
+}

packages/core/src/v3/traceContext/types.ts

@@ -0,0 +1,15 @@
+import { Context } from "@opentelemetry/api";
+
+export interface TraceContextManager {
+  getTraceContext(): Record<string, unknown>;
+  extractContext(): Context;
+  reset(): void;
+  getExternalTraceContext():
+    | {
+        traceId: string;
+        spanId: string;
+        tracestate?: string;
+      }
+    | undefined;
+  withExternalTrace<T>(fn: () => T): T;
+}

packages/core/src/v3/tracer.ts

@@ -63,10 +63,6 @@ export class TriggerTracer {
     return this._logger;
   }
 
-  extractContext(traceContext?: Record<string, unknown>) {
-    return propagation.extract(context.active(), traceContext ?? {});
-  }
-
   startActiveSpan<T>(
     name: string,
     fn: (span: Span) => Promise<T>,

packages/core/src/v3/utils/globals.ts

@@ -8,6 +8,7 @@ import type { RuntimeManager } from "../runtime/manager.js";
 import { RunTimelineMetricsManager } from "../runTimelineMetrics/types.js";
 import { TaskContext } from "../taskContext/types.js";
 import { TimeoutManager } from "../timeout/types.js";
+import { TraceContextManager } from "../traceContext/types.js";
 import { UsageManager } from "../usage/types.js";
 import { WaitUntilManager } from "../waitUntil/types.js";
 import { _globalThis } from "./platform.js";
@@ -66,4 +67,5 @@ type TriggerDotDevGlobalAPI = {
   ["run-timeline-metrics"]?: RunTimelineMetricsManager;
   ["lifecycle-hooks"]?: LifecycleHooksManager;
   ["locals"]?: LocalsManager;
+  ["trace-context"]?: TraceContextManager;
 };

packages/core/src/v3/workers/index.ts

@@ -28,3 +28,4 @@ export { WarmStartClient, type WarmStartClientOptions } from "../workers/warmSta
 export { StandardLifecycleHooksManager } from "../lifecycleHooks/manager.js";
 export { StandardLocalsManager } from "../locals/manager.js";
 export { populateEnv } from "./populateEnv.js";
+export { StandardTraceContextManager } from "../traceContext/manager.js";

packages/core/src/v3/workers/taskExecutor.ts

@@ -1,5 +1,5 @@
-import { Context, context, SpanKind, trace } from "@opentelemetry/api";
-import { VERSION } from "../../version.js";
+import { Context, context, SpanKind } from "@opentelemetry/api";
+import { promiseWithResolvers } from "../../utils.js";
 import { ApiError, RateLimitError } from "../apiClient/errors.js";
 import { ConsoleInterceptor } from "../consoleInterceptor.js";
 import {
@@ -17,6 +17,7 @@ import {
   lifecycleHooks,
   OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT,
   runMetadata,
+  traceContext,
   waitUntil,
 } from "../index.js";
 import {
@@ -31,7 +32,6 @@ import { runTimelineMetrics } from "../run-timeline-metrics-api.js";
 import {
   COLD_VARIANT,
   RetryOptions,
-  ServerBackgroundWorker,
   TaskRunContext,
   TaskRunErrorCodes,
   TaskRunExecution,
@@ -40,7 +40,6 @@ import {
   WARM_VARIANT,
 } from "../schemas/index.js";
 import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
-import { taskContext } from "../task-context-api.js";
 import { TriggerTracer } from "../tracer.js";
 import { tryCatch } from "../tryCatch.js";
 import { HandleErrorModificationOptions, TaskMetadataWithFunctions } from "../types/index.js";
@@ -52,7 +51,6 @@ import {
   stringifyIO,
 } from "../utils/ioSerialization.js";
 import { calculateNextRetryDelay } from "../utils/retries.js";
-import { promiseWithResolvers } from "../../utils.js";
 
 export type TaskExecutorOptions = {
   tracingSDK: TracingSDK;
@@ -93,12 +91,9 @@ export class TaskExecutor {
 
   async execute(
     execution: TaskRunExecution,
-    worker: ServerBackgroundWorker,
-    traceContext: Record<string, unknown>,
-    signal: AbortSignal,
-    isWarmStart?: boolean
+    ctx: TaskRunContext,
+    signal: AbortSignal
   ): Promise<{ result: TaskRunExecutionResult }> {
-    const ctx = TaskRunContext.parse(execution);
     const attemptMessage = `Attempt ${execution.attempt.number}`;
 
     const originalPacket = {
@@ -106,22 +101,10 @@ export class TaskExecutor {
       dataType: execution.run.payloadType,
     };
 
-    taskContext.setGlobalTaskContext({
-      ctx,
-      worker,
-      isWarmStart: isWarmStart ?? this._isWarmStart,
-    });
-
     if (execution.run.metadata) {
       runMetadata.enterWithMetadata(execution.run.metadata);
     }
 
-    if (!this._tracingSDK.asyncResourceDetector.isResolved) {
-      this._tracingSDK.asyncResourceDetector.resolveWithAttributes({
-        ...taskContext.resourceAttributes,
-      });
-    }
-
     const result = await this._tracer.startActiveSpan(
       attemptMessage,
       async (span) => {
@@ -369,7 +352,7 @@ export class TaskExecutor {
             ? runTimelineMetrics.convertMetricsToSpanEvents()
             : undefined,
       },
-      this._tracer.extractContext(traceContext),
+      traceContext.extractContext(),
       signal
     );
 

packages/core/test/taskExecutor.test.ts

@@ -1942,5 +1942,5 @@ function executeTask(
 
   const $signal = signal ? signal : new AbortController().signal;
 
-  return executor.execute(execution, worker, {}, $signal);
+  return executor.execute(execution, execution, $signal);
 }

packages/trigger-sdk/package.json

@@ -50,8 +50,7 @@
   },
   "dependencies": {
     "@opentelemetry/api": "1.9.0",
-    "@opentelemetry/api-logs": "0.52.1",
-    "@opentelemetry/semantic-conventions": "1.25.1",
+    "@opentelemetry/semantic-conventions": "1.36.0",
     "@trigger.dev/core": "workspace:4.0.0-v4-beta.26",
     "chalk": "^5.2.0",
     "cronstrue": "^2.21.0",

packages/trigger-sdk/src/v3/index.ts

@@ -13,6 +13,7 @@ export * from "./metadata.js";
 export * from "./timeout.js";
 export * from "./webhooks.js";
 export * from "./locals.js";
+export * from "./otel.js";
 export type { Context };
 
 import type { Context } from "./shared.js";

packages/trigger-sdk/src/v3/otel.ts

@@ -0,0 +1,7 @@
+import { traceContext } from "@trigger.dev/core/v3";
+
+export const otel = {
+  withExternalTrace: <T>(fn: () => T): T => {
+    return traceContext.withExternalTrace(fn);
+  },
+};

references/d3-chat/package.json

@@ -22,13 +22,18 @@
     "@ai-sdk/anthropic": "^1.2.4",
     "@ai-sdk/openai": "1.3.3",
     "@e2b/code-interpreter": "^1.1.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/api-logs": "^0.203.0",
+    "@opentelemetry/exporter-logs-otlp-http": "0.203.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
+    "@opentelemetry/instrumentation": "^0.203.0",
+    "@opentelemetry/sdk-logs": "^0.203.0",
     "@radix-ui/react-avatar": "^1.1.3",
     "@slack/web-api": "7.9.1",
     "@trigger.dev/python": "workspace:*",
     "@trigger.dev/react-hooks": "workspace:*",
     "@trigger.dev/sdk": "workspace:*",
-    "@opentelemetry/exporter-logs-otlp-http": "0.52.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.52.1",
+    "@vercel/otel": "^1.13.0",
     "@vercel/postgres": "^0.10.0",
     "ai": "4.2.5",
     "class-variance-authority": "^0.7.1",

references/d3-chat/src/app/api/demo-batch-trigger/route.ts

@@ -0,0 +1,18 @@
+import { NextResponse } from "next/server";
+import { tasks } from "@trigger.dev/sdk";
+import type { todoChat } from "@/trigger/chat";
+
+export async function POST(request: Request) {
+  const body = await request.json();
+
+  const handle = await tasks.batchTrigger<typeof todoChat>("todo-chat", [
+    {
+      payload: {
+        input: body.input,
+        userId: "123",
+      },
+    },
+  ]);
+
+  return NextResponse.json({ handle });
+}

references/d3-chat/src/app/api/demo-call-from-trigger/route.ts

@@ -0,0 +1,5 @@
+import { NextResponse } from "next/server";
+
+export async function POST(request: Request) {
+  return NextResponse.json({ body: "Hello, world!" });
+}

references/d3-chat/src/app/api/demo-trigger/route.ts

@@ -0,0 +1,14 @@
+import { NextResponse } from "next/server";
+import { tasks } from "@trigger.dev/sdk";
+import type { todoChat } from "@/trigger/chat";
+
+export async function POST(request: Request) {
+  const body = await request.json();
+
+  const handle = await tasks.trigger<typeof todoChat>("todo-chat", {
+    input: body.input,
+    userId: "123",
+  });
+
+  return NextResponse.json({ handle });
+}

references/d3-chat/src/instrumentation.ts

@@ -0,0 +1,5 @@
+import { registerOTel } from "@vercel/otel";
+
+export function register() {
+  registerOTel({ serviceName: "d3-chat" });
+}

references/d3-chat/src/trigger/chat.ts

@@ -1,7 +1,7 @@
 import { anthropic } from "@ai-sdk/anthropic";
 import { openai } from "@ai-sdk/openai";
 import { ai } from "@trigger.dev/sdk/ai";
-import { logger, metadata, runs, schemaTask, tasks, wait } from "@trigger.dev/sdk/v3";
+import { logger, metadata, runs, schemaTask, tasks, wait, otel } from "@trigger.dev/sdk/v3";
 import { sql } from "@vercel/postgres";
 import {
   CoreMessage,
@@ -17,6 +17,25 @@ import { sendSQLApprovalMessage } from "../lib/slack";
 import { crawler } from "./crawler";
 import { chartTool } from "./sandbox";
 import { QueryApproval } from "./schemas";
+import { context, propagation } from "@opentelemetry/api";
+
+async function callNextjsApp() {
+  return await otel.withExternalTrace(async () => {
+    const headersObject = {};
+
+    propagation.inject(context.active(), headersObject);
+
+    const result = await fetch("http://localhost:3000/api/demo-call-from-trigger", {
+      headers: new Headers(headersObject),
+      method: "POST",
+      body: JSON.stringify({
+        message: "Hello from Trigger.dev",
+      }),
+    });
+
+    return result.json();
+  });
+}
 
 const queryApprovalTask = schemaTask({
   id: "query-approval",
@@ -29,6 +48,8 @@ const queryApprovalTask = schemaTask({
   run: async ({ userId, input, query }) => {
     logger.info("queryApproval: starting", { projectRef: process.env.TRIGGER_PROJECT_REF });
 
+    await callNextjsApp();
+
     const token = await wait.createToken({
       tags: [`user:${userId}`, "approval"],
       timeout: "5m", // timeout in 5 minutes
@@ -129,6 +150,8 @@ export const todoChat = schemaTask({
   run: async ({ input, userId }, { signal }) => {
     metadata.set("user_id", userId);
 
+    logger.info("todoChat: starting", { input, userId });
+
     const system = `
       You are a SQL (postgres) expert who can turn natural language descriptions for a todo app 
       into a SQL query which can then be executed against a SQL database. Here is the schema: