From ca399565bab1a0cba7634b4cd3560a7db878814f Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Thu, 23 Apr 2026 22:41:24 +0100 Subject: [PATCH] feat(webapp): add per-worker Node.js heap metrics (#3437) ## Summary Adds direct V8 heap and process-memory gauges to the webapp's OpenTelemetry meter. The webapp already exports per-cluster-worker Node.js runtime metrics (event-loop lag / utilization, active handles, active requests, libuv threadpool size) via a custom meter under the `trigger.dev` scope. Heap and memory were missing; this PR adds them alongside, in the same observable-batch pattern. ## New gauges | Metric | Source | Unit | | --- | --- | --- | | `nodejs.memory.heap.used` | `process.memoryUsage().heapUsed` | bytes | | `nodejs.memory.heap.total` | `process.memoryUsage().heapTotal` | bytes | | `nodejs.memory.heap.limit` | `v8.getHeapStatistics().heap_size_limit` | bytes | | `nodejs.memory.external` | `process.memoryUsage().external` | bytes | | `nodejs.memory.array_buffers` | `process.memoryUsage().arrayBuffers` | bytes | | `nodejs.memory.rss` | `process.memoryUsage().rss` | bytes | Gated by the existing `INTERNAL_OTEL_NODEJS_METRICS_ENABLED` flag, same as the adjacent event-loop / handle gauges. Zero overhead when disabled. ## Why `@opentelemetry/host-metrics` publishes `process.memory.usage`, which is RSS only. RSS is the sum of V8 heap, external memory (Buffers, etc.), native code, and thread stacks. Without a direct heap metric it is not possible to size the V8 heap cap (`--max-old-space-size`) from metrics alone, because RSS overstates heap by the external + native footprint. A worker can have a 4 GB RSS with a 2.5 GB heap and 1.5 GB of buffers; the former constrains `--max-old-space-size`, the latter does not. `nodejs.memory.heap.limit` also surfaces the configured `--max-old-space-size` (read from `v8.getHeapStatistics().heap_size_limit`), so operators can see the current limit in the same dashboard as actual usage rather than cross-referencing container environment variables. ## Risk Minimal. Observable gauges are sampled at the configured metric-export interval. `v8.getHeapStatistics()` and `process.memoryUsage()` are each microsecond-level calls, and six gauges are added to the same batch callback that already reads ~20 other Node.js runtime values per sample. Same registration pattern as the existing event-loop metrics in the file. ## Test plan - [ ] Deploy and confirm the six new gauges appear at the configured exporter - [ ] In cluster mode, confirm per-worker granularity (one series per cluster worker, tagged by `process.executable.name` / `service.instance.id`) - [ ] Confirm `nodejs.memory.heap.limit` reports the configured `--max-old-space-size` value in bytes --- .server-changes/nodejs-heap-metrics.md | 6 +++ apps/webapp/app/v3/tracer.server.ts | 63 ++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 .server-changes/nodejs-heap-metrics.md diff --git a/.server-changes/nodejs-heap-metrics.md b/.server-changes/nodejs-heap-metrics.md new file mode 100644 index 000000000..bb82fcca9 --- /dev/null +++ b/.server-changes/nodejs-heap-metrics.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Add per-worker Node.js heap metrics to the OTel meter — `nodejs.memory.heap.used`, `nodejs.memory.heap.total`, `nodejs.memory.heap.limit`, `nodejs.memory.external`, `nodejs.memory.array_buffers`, `nodejs.memory.rss`. Host-metrics only publishes RSS, which overstates V8 heap by the external + native footprint; these give direct heap visibility per cluster worker so `NODE_MAX_OLD_SPACE_SIZE` can be sized against observed heap peaks rather than RSS. diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index 1115ab42d..3b924ff8a 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -38,6 +38,7 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; import { PrismaInstrumentation } from "@prisma/instrumentation"; import { HostMetrics } from "@opentelemetry/host-metrics"; import { AwsInstrumentation as AwsSdkInstrumentation } from "@opentelemetry/instrumentation-aws-sdk"; +import v8 from "node:v8"; import { awsEcsDetector, awsEc2Detector } from "@opentelemetry/resource-detector-aws"; import { detectResources, @@ -630,6 +631,39 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { unit: "1", // OpenTelemetry convention for ratios }); + // V8 heap + process memory. `NODE_MAX_OLD_SPACE_SIZE` caps V8 old space + // (reflected in `heap.limit`), but doesn't cap external/arrayBuffers/native + // memory — which is why RSS can exceed the heap total. Tracking all of these + // per-worker lets us size `NODE_MAX_OLD_SPACE_SIZE` against observed heap + // peaks rather than RSS (which overstates heap by the external + native + // footprint). `host-metrics` already publishes `process.memory.usage` + // (RSS), but we duplicate it under `nodejs.memory.rss` so all the memory + // numbers land in the same scope and are queryable together. + const heapUsedGauge = meter.createObservableGauge("nodejs.memory.heap.used", { + description: "V8 heap actively in use after the last GC", + unit: "By", + }); + const heapTotalGauge = meter.createObservableGauge("nodejs.memory.heap.total", { + description: "V8 heap reserved (young + old generations)", + unit: "By", + }); + const heapLimitGauge = meter.createObservableGauge("nodejs.memory.heap.limit", { + description: "V8 heap size limit (configured via --max-old-space-size)", + unit: "By", + }); + const externalMemoryGauge = meter.createObservableGauge("nodejs.memory.external", { + description: "Memory used by C++ objects bound to JS (Buffer, etc.)", + unit: "By", + }); + const arrayBuffersGauge = meter.createObservableGauge("nodejs.memory.array_buffers", { + description: "Memory allocated for ArrayBuffers and SharedArrayBuffers", + unit: "By", + }); + const rssGauge = meter.createObservableGauge("nodejs.memory.rss", { + description: "Resident set size — total physical memory held by the process", + unit: "By", + }); + // Get UV threadpool size (defaults to 4 if not set) const uvThreadpoolSize = parseInt(process.env.UV_THREADPOOL_SIZE || "4", 10); @@ -683,10 +717,16 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { currentEventLoopUtilization, lastEventLoopUtilization ); + // Rotate the baseline so the next collection reports per-interval + // utilization rather than the cumulative average from process start. + lastEventLoopUtilization = currentEventLoopUtilization; // diff.utilization is between 0 and 1 (fraction of time "active") const utilization = Number.isFinite(diff.utilization) ? diff.utilization : 0; + const mem = process.memoryUsage(); + const heapStats = v8.getHeapStatistics(); + return { threadpoolSize: uvThreadpoolSize, handlesByType, @@ -702,6 +742,14 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { p99: eventLoopLagP99?.values?.[0]?.value ?? 0, utilization, }, + memory: { + heapUsed: mem.heapUsed, + heapTotal: mem.heapTotal, + heapLimit: heapStats.heap_size_limit, + external: mem.external, + arrayBuffers: mem.arrayBuffers, + rss: mem.rss, + }, }; } @@ -714,6 +762,7 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { requestsByType, requestsTotal, eventLoop, + memory, } = await readNodeMetrics(); // Observe UV threadpool size @@ -739,6 +788,14 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { res.observe(eventLoopLagP90Gauge, eventLoop.p90); res.observe(eventLoopLagP99Gauge, eventLoop.p99); res.observe(eluGauge, eventLoop.utilization); + + // Observe memory metrics (bytes) + res.observe(heapUsedGauge, memory.heapUsed); + res.observe(heapTotalGauge, memory.heapTotal); + res.observe(heapLimitGauge, memory.heapLimit); + res.observe(externalMemoryGauge, memory.external); + res.observe(arrayBuffersGauge, memory.arrayBuffers); + res.observe(rssGauge, memory.rss); }, [ uvThreadpoolSizeGauge, @@ -753,6 +810,12 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { eventLoopLagP90Gauge, eventLoopLagP99Gauge, eluGauge, + heapUsedGauge, + heapTotalGauge, + heapLimitGauge, + externalMemoryGauge, + arrayBuffersGauge, + rssGauge, ] ); }