From 469b039090741ca20ad4d8c1af28dd5c4994e3af Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 20 Feb 2026 13:16:34 +0000 Subject: [PATCH] feat: OTEL metrics pipeline for task workers (#3061) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Adds an end-to-end OTEL metrics pipeline: task workers collect and export metrics via OpenTelemetry, the webapp ingests them into ClickHouse, and they're queryable through the existing dashboard query engine - Workers emit process CPU/memory metrics (via `@opentelemetry/host-metrics`) and Node.js runtime metrics (event loop utilization, event loop delay, heap usage) - Users can create custom metrics in their tasks via `otel.metrics.getMeter()` from `@trigger.dev/sdk` - Metrics are automatically tagged with run context (run ID, task slug, machine, worker version) so they can be sliced per-run, per-task, or per-machine - The TSQL query engine gains metrics table support with typed attribute columns, `prettyFormat()` for human-readable values, and per-schema time bucket thresholds - Includes reference tasks (`references/hello-world/src/trigger/metrics.ts`) demonstrating CPU-intensive, memory-ramp, bursty workload, and custom metrics patterns ## What changed ### Metrics collection (packages/core, packages/cli-v3) - **Metrics export pipeline** — `TracingSDK` now sets up a `MeterProvider` with a `PeriodicExportingMetricReader` that chains through `TaskContextMetricExporter` (adds run context attributes) and `BufferingMetricExporter` (batches exports to reduce overhead) - **Host metrics** — Enabled `@opentelemetry/host-metrics` for process CPU, memory, and system-level metrics - **Node.js runtime metrics** — New `nodejsRuntimeMetrics.ts` module using `performance.eventLoopUtilization()`, `monitorEventLoopDelay()`, and `process.memoryUsage()` to emit 6 observable gauges - File system and diskio metrics - **Custom metrics** — Exposed `otel.metrics` from `@trigger.dev/sdk` so users can create counters, histograms, and gauges in their tasks - **Machine ID** — Stable per-worker machine identifier for grouping metrics - **Dev worker** — Drops `system.*` metrics to reduce noise, keeps sending metrics between runs in warm workers ### Metrics ingestion (apps/webapp) - **OTEL endpoint** — `otel.v1.metrics.ts` accepts OTEL metric export requests (JSON and protobuf), converts to ClickHouse rows - **ClickHouse schema** — `017_create_metrics_v1.sql` with 10-second aggregation buckets, JSON attributes column, 60-day TTLs ### Query engine (internal-packages/tsql, apps/webapp) - **Metrics query schema** — Typed columns for metric attributes (`task_identifier`, `run_id`, `machine_name`, `worker_version`, etc.) extracted from the JSON attributes column - **`prettyFormat()`** — TSQL function that annotates columns with format hints (`bytes`, `percent`, `durationSeconds`) for frontend rendering without changing the underlying data - **Per-schema time buckets** — Different tables can define their own time bucket thresholds (metrics uses tighter intervals than runs) - **AI query integration** — The AI query service knows about the metrics table and can generate metric queries - **Chart improvements** — Better formatting for byte values, percentages, and durations in charts and tables ### Reference project - **`references/hello-world/src/trigger/metrics.ts`** — 6 example tasks: `cpu-intensive`, `memory-ramp`, `bursty-workload`, `sustained-workload`, `concurrent-load`, `custom-metrics` ## Test plan - [ ] Build all packages and webapp - [ ] Start dev worker with hello-world reference project - [ ] Run `cpu-intensive`, `memory-ramp`, and `custom-metrics` tasks - [ ] Verify metrics in ClickHouse: `SELECT DISTINCT metric_name FROM metrics_v1` - [ ] Query via dashboard AI: "show me CPU utilization over time" - [ ] Verify `prettyFormat` renders correctly in chart tooltips and table cells - [ ] Confirm dev worker drops `system.*` metrics but keeps `process.*` and `nodejs.*` --- .changeset/fix-coderabbit-review-items.md | 6 + .../app/components/code/QueryResultsChart.tsx | 77 ++++- .../app/components/code/TSQLResultsTable.tsx | 88 ++++- .../primitives/charts/BigNumberCard.tsx | 29 +- .../components/primitives/charts/Chart.tsx | 9 +- .../components/primitives/charts/ChartBar.tsx | 5 +- .../primitives/charts/ChartLegendCompound.tsx | 29 +- .../primitives/charts/ChartLine.tsx | 19 +- .../primitives/charts/ChartRoot.tsx | 7 + .../app/components/query/QueryEditor.tsx | 6 +- apps/webapp/app/env.server.ts | 10 + .../AITabContent.tsx | 2 + .../ExamplesContent.tsx | 67 +++- .../TableSchemaContent.tsx | 46 ++- .../routes/deployments.$deploymentParam.ts | 70 ++++ apps/webapp/app/routes/otel.v1.metrics.ts | 41 +++ .../app/services/queryService.server.ts | 23 +- apps/webapp/app/utils/columnFormat.ts | 70 ++++ .../environmentVariablesRepository.server.ts | 49 +++ .../clickhouseEventRepository.server.ts | 1 + .../eventRepository/eventRepository.types.ts | 1 + apps/webapp/app/v3/otlpExporter.server.ts | 245 +++++++++++++ apps/webapp/app/v3/querySchemas.ts | 166 ++++++++- .../app/v3/services/aiQueryService.server.ts | 70 +++- apps/webapp/package.json | 2 +- .../schema/017_create_metrics_v1.sql | 44 +++ .../018_add_machine_id_to_task_events_v2.sql | 7 + internal-packages/clickhouse/src/index.ts | 10 +- internal-packages/clickhouse/src/metrics.ts | 29 ++ .../clickhouse/src/taskEvents.ts | 2 + internal-packages/otlp-importer/src/index.ts | 39 +++ internal-packages/tsql/src/index.ts | 3 + .../tsql/src/query/printer.test.ts | 207 ++++++++++- internal-packages/tsql/src/query/printer.ts | 89 ++++- internal-packages/tsql/src/query/schema.ts | 51 +++ .../tsql/src/query/time_buckets.ts | 22 +- packages/cli-v3/src/dev/taskRunProcessPool.ts | 48 +++ .../src/entryPoints/dev-run-controller.ts | 2 +- .../cli-v3/src/entryPoints/dev-run-worker.ts | 11 +- .../src/entryPoints/managed-run-worker.ts | 14 +- .../managed/taskRunProcessProvider.ts | 6 +- .../cli-v3/src/executions/taskRunProcess.ts | 6 +- packages/core/package.json | 3 + packages/core/src/v3/config.ts | 15 + packages/core/src/v3/index.ts | 1 + packages/core/src/v3/otel/diskIoMetrics.ts | 92 +++++ .../core/src/v3/otel/filesystemMetrics.ts | 134 +++++++ packages/core/src/v3/otel/machineId.ts | 4 + .../core/src/v3/otel/nodejsRuntimeMetrics.ts | 81 +++++ packages/core/src/v3/otel/tracingSDK.ts | 127 ++++++- packages/core/src/v3/schemas/messages.ts | 1 + .../core/src/v3/semanticInternalAttributes.ts | 2 + packages/core/src/v3/taskContext/index.ts | 12 +- .../core/src/v3/taskContext/otelProcessors.ts | 203 ++++++++++- packages/core/src/v3/workers/index.ts | 1 + packages/trigger-sdk/src/v3/otel.ts | 2 + pnpm-lock.yaml | 23 +- references/bun-catalog/src/trigger/bun.ts | 2 + references/bun-catalog/trigger.config.ts | 2 +- references/hello-world/src/trigger/metrics.ts | 326 ++++++++++++++++++ 60 files changed, 2659 insertions(+), 100 deletions(-) create mode 100644 .changeset/fix-coderabbit-review-items.md create mode 100644 apps/webapp/app/routes/deployments.$deploymentParam.ts create mode 100644 apps/webapp/app/routes/otel.v1.metrics.ts create mode 100644 apps/webapp/app/utils/columnFormat.ts create mode 100644 internal-packages/clickhouse/schema/017_create_metrics_v1.sql create mode 100644 internal-packages/clickhouse/schema/018_add_machine_id_to_task_events_v2.sql create mode 100644 internal-packages/clickhouse/src/metrics.ts create mode 100644 packages/core/src/v3/otel/diskIoMetrics.ts create mode 100644 packages/core/src/v3/otel/filesystemMetrics.ts create mode 100644 packages/core/src/v3/otel/machineId.ts create mode 100644 packages/core/src/v3/otel/nodejsRuntimeMetrics.ts create mode 100644 references/hello-world/src/trigger/metrics.ts diff --git a/.changeset/fix-coderabbit-review-items.md b/.changeset/fix-coderabbit-review-items.md new file mode 100644 index 000000000..00bebbb1a --- /dev/null +++ b/.changeset/fix-coderabbit-review-items.md @@ -0,0 +1,6 @@ +--- +"trigger.dev": patch +"@trigger.dev/sdk": patch +--- + +Add OTEL metrics pipeline for task workers. Workers collect process CPU/memory, Node.js runtime metrics (event loop utilization, event loop delay, heap usage), and user-defined custom metrics via `otel.metrics.getMeter()`. Metrics are exported to ClickHouse with 10-second aggregation buckets and 1m/5m rollups, and are queryable through the dashboard query engine with typed attribute columns, `prettyFormat()` for human-readable values, and AI query support. diff --git a/apps/webapp/app/components/code/QueryResultsChart.tsx b/apps/webapp/app/components/code/QueryResultsChart.tsx index d2893cfb9..2da90c9e0 100644 --- a/apps/webapp/app/components/code/QueryResultsChart.tsx +++ b/apps/webapp/app/components/code/QueryResultsChart.tsx @@ -1,6 +1,9 @@ -import type { OutputColumnMetadata } from "@internal/clickhouse"; +import type { ColumnFormatType, OutputColumnMetadata } from "@internal/clickhouse"; +import { formatDurationMilliseconds } from "@trigger.dev/core/v3"; import { BarChart3, LineChart } from "lucide-react"; import { memo, useMemo } from "react"; +import { createValueFormatter } from "~/utils/columnFormat"; +import { formatCurrencyAccurate } from "~/utils/numberFormatter"; import type { ChartConfig } from "~/components/primitives/charts/Chart"; import { Chart } from "~/components/primitives/charts/ChartCompound"; import { ChartBlankState } from "../primitives/charts/ChartBlankState"; @@ -855,8 +858,24 @@ export const QueryResultsChart = memo(function QueryResultsChart({ }; }, [isDateBased, timeGranularity]); - // Create dynamic Y-axis formatter based on data range - const yAxisFormatter = useMemo(() => createYAxisFormatter(data, series), [data, series]); + // Resolve the Y-axis column format for formatting + const yAxisFormat = useMemo(() => { + if (yAxisColumns.length === 0) return undefined; + const col = columns.find((c) => c.name === yAxisColumns[0]); + return (col?.format ?? col?.customRenderType) as ColumnFormatType | undefined; + }, [yAxisColumns, columns]); + + // Create dynamic Y-axis formatter based on data range and format + const yAxisFormatter = useMemo( + () => createYAxisFormatter(data, series, yAxisFormat), + [data, series, yAxisFormat] + ); + + // Create value formatter for tooltips and legend based on column format + const tooltipValueFormatter = useMemo( + () => createValueFormatter(yAxisFormat), + [yAxisFormat] + ); // Check if the group-by column has a runStatus customRenderType const groupByIsRunStatus = useMemo(() => { @@ -1081,6 +1100,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({ showLegend={showLegend} maxLegendItems={fullLegend ? Infinity : 5} legendAggregation={config.aggregation} + legendValueFormatter={tooltipValueFormatter} minHeight="300px" fillContainer onViewAllLegendItems={onViewAllLegendItems} @@ -1093,6 +1113,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({ yAxisProps={yAxisProps} stackId={stacked ? "stack" : undefined} tooltipLabelFormatter={tooltipLabelFormatter} + tooltipValueFormatter={tooltipValueFormatter} /> ); @@ -1110,6 +1131,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({ showLegend={showLegend} maxLegendItems={fullLegend ? Infinity : 5} legendAggregation={config.aggregation} + legendValueFormatter={tooltipValueFormatter} minHeight="300px" fillContainer onViewAllLegendItems={onViewAllLegendItems} @@ -1122,6 +1144,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({ yAxisProps={yAxisProps} stacked={stacked && visibleSeries.length > 1} tooltipLabelFormatter={tooltipLabelFormatter} + tooltipValueFormatter={tooltipValueFormatter} lineType="linear" /> @@ -1129,9 +1152,13 @@ export const QueryResultsChart = memo(function QueryResultsChart({ }); /** - * Creates a Y-axis value formatter based on the data range + * Creates a Y-axis value formatter based on the data range and optional format hint */ -function createYAxisFormatter(data: Record[], series: string[]) { +function createYAxisFormatter( + data: Record[], + series: string[], + format?: ColumnFormatType +) { // Find min and max values across all series let minVal = Infinity; let maxVal = -Infinity; @@ -1148,6 +1175,46 @@ function createYAxisFormatter(data: Record[], series: string[]) const range = maxVal - minVal; + // Format-aware formatters + if (format === "bytes" || format === "decimalBytes") { + const divisor = format === "bytes" ? 1024 : 1000; + const units = + format === "bytes" + ? ["B", "KiB", "MiB", "GiB", "TiB"] + : ["B", "KB", "MB", "GB", "TB"]; + return (value: number): string => { + if (value === 0) return "0 B"; + // Use consistent unit for all ticks based on max value + const i = Math.min( + Math.max(0, Math.floor(Math.log(Math.abs(maxVal || 1)) / Math.log(divisor))), + units.length - 1 + ); + const scaled = value / Math.pow(divisor, i); + return `${scaled.toFixed(scaled < 10 ? 1 : 0)} ${units[i]}`; + }; + } + + if (format === "percent") { + return (value: number): string => `${value.toFixed(range < 1 ? 2 : 1)}%`; + } + + if (format === "duration") { + return (value: number): string => formatDurationMilliseconds(value, { style: "short" }); + } + + if (format === "durationSeconds") { + return (value: number): string => + formatDurationMilliseconds(value * 1000, { style: "short" }); + } + + if (format === "costInDollars" || format === "cost") { + return (value: number): string => { + const dollars = format === "cost" ? value / 100 : value; + return formatCurrencyAccurate(dollars); + }; + } + + // Default formatter return (value: number): string => { // Use abbreviations for large numbers if (Math.abs(value) >= 1_000_000) { diff --git a/apps/webapp/app/components/code/TSQLResultsTable.tsx b/apps/webapp/app/components/code/TSQLResultsTable.tsx index 36ae3a290..dae045bc4 100644 --- a/apps/webapp/app/components/code/TSQLResultsTable.tsx +++ b/apps/webapp/app/components/code/TSQLResultsTable.tsx @@ -35,6 +35,7 @@ import { useCopy } from "~/hooks/useCopy"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import { cn } from "~/utils/cn"; +import { formatBytes, formatDecimalBytes, formatQuantity } from "~/utils/columnFormat"; import { formatCurrencyAccurate, formatNumber } from "~/utils/numberFormatter"; import { v3ProjectPath, v3RunPathFromFriendlyId } from "~/utils/pathBuilder"; import { ChartBlankState } from "../primitives/charts/ChartBlankState"; @@ -66,9 +67,10 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string if (value === null) return "NULL"; if (value === undefined) return ""; - // Handle custom render types - if (column.customRenderType) { - switch (column.customRenderType) { + // Handle format hints (from prettyFormat() or auto-populated from customRenderType) + const formatType = column.format ?? column.customRenderType; + if (formatType) { + switch (formatType) { case "duration": if (typeof value === "number") { return formatDurationMilliseconds(value, { style: "short" }); @@ -95,6 +97,26 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string return value; } break; + case "bytes": + if (typeof value === "number") { + return formatBytes(value); + } + break; + case "decimalBytes": + if (typeof value === "number") { + return formatDecimalBytes(value); + } + break; + case "percent": + if (typeof value === "number") { + return `${value.toFixed(2)}%`; + } + break; + case "quantity": + if (typeof value === "number") { + return formatQuantity(value); + } + break; } } @@ -222,6 +244,21 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number if (value === null) return 4; // "NULL" if (value === undefined) return 9; // "UNDEFINED" + // Handle format hint types - estimate their rendered width + const fmt = column.format; + if (fmt === "bytes" || fmt === "decimalBytes") { + // e.g., "1.50 GiB" or "256.00 MB" + return 12; + } + if (fmt === "percent") { + // e.g., "45.23%" + return 8; + } + if (fmt === "quantity") { + // e.g., "1.50M" + return 8; + } + // Handle custom render types - estimate their rendered width if (column.customRenderType) { switch (column.customRenderType) { @@ -263,6 +300,8 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number return typeof value === "string" ? Math.min(value.length, 20) : 12; case "queue": return typeof value === "string" ? Math.min(value.length, 25) : 15; + case "deploymentId": + return typeof value === "string" ? Math.min(value.length, 25) : 20; } } @@ -394,6 +433,10 @@ function isRightAlignedColumn(column: OutputColumnMetadata): boolean { ) { return true; } + const fmt = column.format; + if (fmt === "bytes" || fmt === "decimalBytes" || fmt === "percent" || fmt === "quantity") { + return true; + } return isNumericType(column.type); } @@ -476,6 +519,32 @@ function CellValue({ return
UNDEFINED
; } + // Check format hint for new format types (from prettyFormat()) + if (column.format && !column.customRenderType) { + switch (column.format) { + case "bytes": + if (typeof value === "number") { + return {formatBytes(value)}; + } + break; + case "decimalBytes": + if (typeof value === "number") { + return {formatDecimalBytes(value)}; + } + break; + case "percent": + if (typeof value === "number") { + return {value.toFixed(2)}%; + } + break; + case "quantity": + if (typeof value === "number") { + return {formatQuantity(value)}; + } + break; + } + } + // First check customRenderType for special rendering if (column.customRenderType) { switch (column.customRenderType) { @@ -577,6 +646,19 @@ function CellValue({ } return {String(value)}; } + case "deploymentId": { + if (typeof value === "string" && value.startsWith("deployment_")) { + return ( +