Eric Allam
469b039090
- Adds an end-to-end OTEL metrics pipeline: task workers collect and export metrics via OpenTelemetry, the webapp ingests them into ClickHouse, and they're queryable through the existing dashboard query engine - Workers emit process CPU/memory metrics (via `@opentelemetry/host-metrics`) and Node.js runtime metrics (event loop utilization, event loop delay, heap usage) - Users can create custom metrics in their tasks via `otel.metrics.getMeter()` from `@trigger.dev/sdk` - Metrics are automatically tagged with run context (run ID, task slug, machine, worker version) so they can be sliced per-run, per-task, or per-machine - The TSQL query engine gains metrics table support with typed attribute columns, `prettyFormat()` for human-readable values, and per-schema time bucket thresholds - Includes reference tasks (`references/hello-world/src/trigger/metrics.ts`) demonstrating CPU-intensive, memory-ramp, bursty workload, and custom metrics patterns ## What changed ### Metrics collection (packages/core, packages/cli-v3) - **Metrics export pipeline** — `TracingSDK` now sets up a `MeterProvider` with a `PeriodicExportingMetricReader` that chains through `TaskContextMetricExporter` (adds run context attributes) and `BufferingMetricExporter` (batches exports to reduce overhead) - **Host metrics** — Enabled `@opentelemetry/host-metrics` for process CPU, memory, and system-level metrics - **Node.js runtime metrics** — New `nodejsRuntimeMetrics.ts` module using `performance.eventLoopUtilization()`, `monitorEventLoopDelay()`, and `process.memoryUsage()` to emit 6 observable gauges - File system and diskio metrics - **Custom metrics** — Exposed `otel.metrics` from `@trigger.dev/sdk` so users can create counters, histograms, and gauges in their tasks - **Machine ID** — Stable per-worker machine identifier for grouping metrics - **Dev worker** — Drops `system.*` metrics to reduce noise, keeps sending metrics between runs in warm workers ### Metrics ingestion (apps/webapp) - **OTEL endpoint** — `otel.v1.metrics.ts` accepts OTEL metric export requests (JSON and protobuf), converts to ClickHouse rows - **ClickHouse schema** — `017_create_metrics_v1.sql` with 10-second aggregation buckets, JSON attributes column, 60-day TTLs ### Query engine (internal-packages/tsql, apps/webapp) - **Metrics query schema** — Typed columns for metric attributes (`task_identifier`, `run_id`, `machine_name`, `worker_version`, etc.) extracted from the JSON attributes column - **`prettyFormat()`** — TSQL function that annotates columns with format hints (`bytes`, `percent`, `durationSeconds`) for frontend rendering without changing the underlying data - **Per-schema time buckets** — Different tables can define their own time bucket thresholds (metrics uses tighter intervals than runs) - **AI query integration** — The AI query service knows about the metrics table and can generate metric queries - **Chart improvements** — Better formatting for byte values, percentages, and durations in charts and tables ### Reference project - **`references/hello-world/src/trigger/metrics.ts`** — 6 example tasks: `cpu-intensive`, `memory-ramp`, `bursty-workload`, `sustained-workload`, `concurrent-load`, `custom-metrics` ## Test plan - [ ] Build all packages and webapp - [ ] Start dev worker with hello-world reference project - [ ] Run `cpu-intensive`, `memory-ramp`, and `custom-metrics` tasks - [ ] Verify metrics in ClickHouse: `SELECT DISTINCT metric_name FROM metrics_v1` - [ ] Query via dashboard AI: "show me CPU utilization over time" - [ ] Verify `prettyFormat` renders correctly in chart tooltips and table cells - [ ] Confirm dev worker drops `system.*` metrics but keeps `process.*` and `nodejs.*`
71 lines
2.3 KiB
TypeScript
71 lines
2.3 KiB
TypeScript
import type { ColumnFormatType } from "@internal/clickhouse";
|
|
import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
|
|
import { formatCurrencyAccurate } from "~/utils/numberFormatter";
|
|
|
|
/**
|
|
* Format a number as binary bytes (KiB, MiB, GiB, TiB)
|
|
*/
|
|
export function formatBytes(bytes: number): string {
|
|
if (bytes === 0) return "0 B";
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
const i = Math.min(
|
|
Math.max(0, Math.floor(Math.log(Math.abs(bytes)) / Math.log(1024))),
|
|
units.length - 1
|
|
);
|
|
return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 2)} ${units[i]}`;
|
|
}
|
|
|
|
/**
|
|
* Format a number as decimal bytes (KB, MB, GB, TB)
|
|
*/
|
|
export function formatDecimalBytes(bytes: number): string {
|
|
if (bytes === 0) return "0 B";
|
|
const units = ["B", "KB", "MB", "GB", "TB"];
|
|
const i = Math.min(
|
|
Math.max(0, Math.floor(Math.log(Math.abs(bytes)) / Math.log(1000))),
|
|
units.length - 1
|
|
);
|
|
return `${(bytes / Math.pow(1000, i)).toFixed(i === 0 ? 0 : 2)} ${units[i]}`;
|
|
}
|
|
|
|
/**
|
|
* Format a large number with human-readable suffix (K, M, B)
|
|
*/
|
|
export function formatQuantity(value: number): string {
|
|
const abs = Math.abs(value);
|
|
if (abs >= 1_000_000_000) return `${(value / 1_000_000_000).toFixed(2)}B`;
|
|
if (abs >= 1_000_000) return `${(value / 1_000_000).toFixed(2)}M`;
|
|
if (abs >= 1_000) return `${(value / 1_000).toFixed(2)}K`;
|
|
return value.toLocaleString();
|
|
}
|
|
|
|
/**
|
|
* Creates a value formatter function for a given column format type.
|
|
* Used by chart tooltips, legend values, and big number cards.
|
|
*/
|
|
export function createValueFormatter(
|
|
format?: ColumnFormatType
|
|
): ((value: number) => string) | undefined {
|
|
if (!format) return undefined;
|
|
switch (format) {
|
|
case "bytes":
|
|
return (v) => formatBytes(v);
|
|
case "decimalBytes":
|
|
return (v) => formatDecimalBytes(v);
|
|
case "percent":
|
|
return (v) => `${v.toFixed(2)}%`;
|
|
case "quantity":
|
|
return (v) => formatQuantity(v);
|
|
case "duration":
|
|
return (v) => formatDurationMilliseconds(v, { style: "short" });
|
|
case "durationSeconds":
|
|
return (v) => formatDurationMilliseconds(v * 1000, { style: "short" });
|
|
case "costInDollars":
|
|
return (v) => formatCurrencyAccurate(v);
|
|
case "cost":
|
|
return (v) => formatCurrencyAccurate(v / 100);
|
|
default:
|
|
return undefined;
|
|
}
|
|
}
|