diff --git a/.env.example b/.env.example index ab1d09d7b..a19ff46f2 100644 --- a/.env.example +++ b/.env.example @@ -64,4 +64,12 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com" # OBJECT_STORE_ACCESS_KEY_ID= # OBJECT_STORE_SECRET_ACCESS_KEY= -# RUNTIME_WAIT_THRESHOLD_IN_MS=10000 \ No newline at end of file +# RUNTIME_WAIT_THRESHOLD_IN_MS=10000 + +# These control the server-side internal telemetry +# INTERNAL_OTEL_TRACE_EXPORTER_URL= +# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME=
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE= +# INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1 +# INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded) +# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0, \ No newline at end of file diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 5599bc331..69c1c334b 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -85,8 +85,6 @@ const EnvironmentSchema = z.object({ //v3 V3_ENABLED: z.string().default("false"), - OTLP_EXPORTER_TRACES_URL: z.string().optional(), - LOG_TELEMETRY: z.string().default("true"), IMAGE_REGISTRY: z.string().default("docker.io"), IMAGE_REPO: z.string().default("task"), PROVIDER_SECRET: z.string().default("provider-secret"), @@ -117,6 +115,15 @@ const EnvironmentSchema = z.object({ DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), DEV_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"), RUNTIME_WAIT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000), + + // Internal OTEL environment variables + INTERNAL_OTEL_TRACE_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME: z.string().optional(), + INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE: z.string().optional(), + INTERNAL_OTEL_TRACE_LOGGING_ENABLED: z.string().default("1"), + // this means 1/20 traces or 5% of traces will be sampled (sampled = recorded) + INTERNAL_OTEL_TRACE_SAMPING_RATE: z.string().default("20"), + INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"), }); export type Environment = z.infer; diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index bdbe16549..e872aba27 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -9,7 +9,7 @@ import { trace, } from "@opentelemetry/api"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; -import { registerInstrumentations } from "@opentelemetry/instrumentation"; +import { InstrumentationOption, registerInstrumentations } from "@opentelemetry/instrumentation"; import { ExpressInstrumentation } from "@opentelemetry/instrumentation-express"; import { HttpInstrumentation } from "@opentelemetry/instrumentation-http"; import { Resource } from "@opentelemetry/resources"; @@ -23,12 +23,13 @@ import { TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; -import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions"; +import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; import { PrismaInstrumentation } from "@prisma/instrumentation"; import { env } from "~/env.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { singleton } from "~/utils/singleton"; import { LoggerSpanExporter } from "./telemetry/loggerExporter.server"; + class CustomWebappSampler implements Sampler { constructor(private readonly _baseSampler: Sampler) {} @@ -44,10 +45,7 @@ class CustomWebappSampler implements Sampler { const parentContext = trace.getSpanContext(context); // Exclude Prisma spans (adjust this logic as needed for your use case) - if ( - !parentContext && - ((attributes && attributes["model"] && attributes["method"]) || name.includes("prisma")) - ) { + if (!parentContext && name.includes("prisma")) { return { decision: SamplingDecision.NOT_RECORD }; } @@ -65,29 +63,46 @@ export const tracer = singleton("tracer", getTracer); function getTracer() { diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR); + const samplingRate = 1.0 / Math.max(parseInt(env.INTERNAL_OTEL_TRACE_SAMPING_RATE, 10), 1); + const provider = new NodeTracerProvider({ forceFlushTimeoutMillis: 500, resource: new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: "trigger.dev", + [SEMRESATTRS_SERVICE_NAME]: "trigger.dev", }), sampler: new ParentBasedSampler({ - root: new CustomWebappSampler( - new TraceIdRatioBasedSampler(env.APP_ENV === "development" ? 1.0 : 0.05) - ), // 5% sampling + root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)), // 5% sampling }), // 5% sampling }); - if (env.OTLP_EXPORTER_TRACES_URL) { + if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) { const exporter = new OTLPTraceExporter({ - url: env.OTLP_EXPORTER_TRACES_URL, + url: env.INTERNAL_OTEL_TRACE_EXPORTER_URL, timeoutMillis: 1000, + headers: + env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME && + env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE + ? { + [env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME]: + env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE, + } + : undefined, }); - provider.addSpanProcessor(new BatchSpanProcessor(exporter)); + provider.addSpanProcessor( + new BatchSpanProcessor(exporter, { + maxExportBatchSize: 512, + scheduledDelayMillis: 200, + exportTimeoutMillis: 30000, + maxQueueSize: 2048, + }) + ); - console.log(`⚡ Tracer: OTLP exporter enabled to ${env.OTLP_EXPORTER_TRACES_URL}`); + console.log(`🔦 Tracer: OTLP exporter enabled to ${env.INTERNAL_OTEL_TRACE_EXPORTER_URL}`); } else { - if (env.LOG_TELEMETRY === "true") { + if (env.INTERNAL_OTEL_TRACE_LOGGING_ENABLED === "1") { + console.log(`🔦 Tracer: Logger exporter enabled`); + const loggerExporter = new LoggerSpanExporter(); provider.addSpanProcessor(new SimpleSpanProcessor(loggerExporter)); @@ -96,13 +111,18 @@ function getTracer() { provider.register(); + let instrumentations: InstrumentationOption[] = [ + new HttpInstrumentation(), + new ExpressInstrumentation(), + ]; + + if (env.INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED === "1") { + instrumentations.push(new PrismaInstrumentation()); + } + registerInstrumentations({ tracerProvider: provider, - instrumentations: [ - new HttpInstrumentation(), - new ExpressInstrumentation(), - new PrismaInstrumentation(), - ], + instrumentations, }); return provider.getTracer("trigger.dev", "3.0.0.dp.1"); diff --git a/packages/core-backend/src/logger.ts b/packages/core-backend/src/logger.ts index d86c20e27..3c469552e 100644 --- a/packages/core-backend/src/logger.ts +++ b/packages/core-backend/src/logger.ts @@ -94,20 +94,22 @@ export class Logger { ...args: Array | undefined> ) { // Get the current context from trace if it exists - const currentContext = trace.getSpan(context.active()); + const currentSpan = trace.getSpan(context.active()); - const structuredLog = { - ...structureArgs(safeJsonClone(args) as Record[], this.#filteredKeys), - ...this.#additionalFields(), - timestamp: new Date(), - name: this.#name, - message, - level, - traceId: currentContext?.spanContext().traceId, - parentSpanId: currentContext?.spanContext().spanId, - }; + if (!currentSpan || currentSpan.isRecording()) { + const structuredLog = { + ...structureArgs(safeJsonClone(args) as Record[], this.#filteredKeys), + ...this.#additionalFields(), + timestamp: new Date(), + name: this.#name, + message, + level, + traceId: currentSpan?.spanContext().traceId, + parentSpanId: currentSpan?.spanContext().spanId, + }; - loggerFunction(JSON.stringify(structuredLog, this.#jsonReplacer)); + loggerFunction(JSON.stringify(structuredLog, this.#jsonReplacer)); + } } }