feat: OTEL metrics pipeline for task workers (#3061)

- Adds an end-to-end OTEL metrics pipeline: task workers collect and
export metrics via OpenTelemetry, the webapp ingests them into
ClickHouse, and they're queryable through the existing dashboard query
engine
- Workers emit process CPU/memory metrics (via
`@opentelemetry/host-metrics`) and Node.js runtime metrics (event loop
utilization, event loop delay, heap usage)
- Users can create custom metrics in their tasks via
`otel.metrics.getMeter()` from `@trigger.dev/sdk`
- Metrics are automatically tagged with run context (run ID, task slug,
machine, worker version) so they can be sliced per-run, per-task, or
per-machine
- The TSQL query engine gains metrics table support with typed attribute
columns, `prettyFormat()` for human-readable values, and per-schema time
bucket thresholds
- Includes reference tasks
(`references/hello-world/src/trigger/metrics.ts`) demonstrating
CPU-intensive, memory-ramp, bursty workload, and custom metrics patterns

## What changed

### Metrics collection (packages/core, packages/cli-v3)
- **Metrics export pipeline** — `TracingSDK` now sets up a
`MeterProvider` with a `PeriodicExportingMetricReader` that chains
through `TaskContextMetricExporter` (adds run context attributes) and
`BufferingMetricExporter` (batches exports to reduce overhead)
- **Host metrics** — Enabled `@opentelemetry/host-metrics` for process
CPU, memory, and system-level metrics
- **Node.js runtime metrics** — New `nodejsRuntimeMetrics.ts` module
using `performance.eventLoopUtilization()`, `monitorEventLoopDelay()`,
and `process.memoryUsage()` to emit 6 observable gauges
- File system and diskio metrics
- **Custom metrics** — Exposed `otel.metrics` from `@trigger.dev/sdk` so
users can create counters, histograms, and gauges in their tasks
- **Machine ID** — Stable per-worker machine identifier for grouping
metrics
- **Dev worker** — Drops `system.*` metrics to reduce noise, keeps
sending metrics between runs in warm workers

### Metrics ingestion (apps/webapp)
- **OTEL endpoint** — `otel.v1.metrics.ts` accepts OTEL metric export
requests (JSON and protobuf), converts to ClickHouse rows
- **ClickHouse schema** — `017_create_metrics_v1.sql` with 10-second
aggregation buckets, JSON attributes column, 60-day TTLs

### Query engine (internal-packages/tsql, apps/webapp)
- **Metrics query schema** — Typed columns for metric attributes
(`task_identifier`, `run_id`, `machine_name`, `worker_version`, etc.)
extracted from the JSON attributes column
- **`prettyFormat()`** — TSQL function that annotates columns with
format hints (`bytes`, `percent`, `durationSeconds`) for frontend
rendering without changing the underlying data
- **Per-schema time buckets** — Different tables can define their own
time bucket thresholds (metrics uses tighter intervals than runs)
- **AI query integration** — The AI query service knows about the
metrics table and can generate metric queries
- **Chart improvements** — Better formatting for byte values,
percentages, and durations in charts and tables

### Reference project
- **`references/hello-world/src/trigger/metrics.ts`** — 6 example tasks:
`cpu-intensive`, `memory-ramp`, `bursty-workload`, `sustained-workload`,
`concurrent-load`, `custom-metrics`

## Test plan

- [ ] Build all packages and webapp
- [ ] Start dev worker with hello-world reference project
- [ ] Run `cpu-intensive`, `memory-ramp`, and `custom-metrics` tasks
- [ ] Verify metrics in ClickHouse: `SELECT DISTINCT metric_name FROM
metrics_v1`
- [ ] Query via dashboard AI: "show me CPU utilization over time"
- [ ] Verify `prettyFormat` renders correctly in chart tooltips and
table cells
- [ ] Confirm dev worker drops `system.*` metrics but keeps `process.*`
and `nodejs.*`

.changeset/fix-coderabbit-review-items.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/sdk": patch
+---
+
+Add OTEL metrics pipeline for task workers. Workers collect process CPU/memory, Node.js runtime metrics (event loop utilization, event loop delay, heap usage), and user-defined custom metrics via `otel.metrics.getMeter()`. Metrics are exported to ClickHouse with 10-second aggregation buckets and 1m/5m rollups, and are queryable through the dashboard query engine with typed attribute columns, `prettyFormat()` for human-readable values, and AI query support.

apps/webapp/app/components/code/QueryResultsChart.tsx

@@ -1,6 +1,9 @@
-import type { OutputColumnMetadata } from "@internal/clickhouse";
+import type { ColumnFormatType, OutputColumnMetadata } from "@internal/clickhouse";
+import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
 import { BarChart3, LineChart } from "lucide-react";
 import { memo, useMemo } from "react";
+import { createValueFormatter } from "~/utils/columnFormat";
+import { formatCurrencyAccurate } from "~/utils/numberFormatter";
 import type { ChartConfig } from "~/components/primitives/charts/Chart";
 import { Chart } from "~/components/primitives/charts/ChartCompound";
 import { ChartBlankState } from "../primitives/charts/ChartBlankState";
@@ -855,8 +858,24 @@ export const QueryResultsChart = memo(function QueryResultsChart({
     };
   }, [isDateBased, timeGranularity]);
 
-  // Create dynamic Y-axis formatter based on data range
-  const yAxisFormatter = useMemo(() => createYAxisFormatter(data, series), [data, series]);
+  // Resolve the Y-axis column format for formatting
+  const yAxisFormat = useMemo(() => {
+    if (yAxisColumns.length === 0) return undefined;
+    const col = columns.find((c) => c.name === yAxisColumns[0]);
+    return (col?.format ?? col?.customRenderType) as ColumnFormatType | undefined;
+  }, [yAxisColumns, columns]);
+
+  // Create dynamic Y-axis formatter based on data range and format
+  const yAxisFormatter = useMemo(
+    () => createYAxisFormatter(data, series, yAxisFormat),
+    [data, series, yAxisFormat]
+  );
+
+  // Create value formatter for tooltips and legend based on column format
+  const tooltipValueFormatter = useMemo(
+    () => createValueFormatter(yAxisFormat),
+    [yAxisFormat]
+  );
 
   // Check if the group-by column has a runStatus customRenderType
   const groupByIsRunStatus = useMemo(() => {
@@ -1081,6 +1100,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
         showLegend={showLegend}
         maxLegendItems={fullLegend ? Infinity : 5}
         legendAggregation={config.aggregation}
+        legendValueFormatter={tooltipValueFormatter}
         minHeight="300px"
         fillContainer
         onViewAllLegendItems={onViewAllLegendItems}
@@ -1093,6 +1113,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
           yAxisProps={yAxisProps}
           stackId={stacked ? "stack" : undefined}
           tooltipLabelFormatter={tooltipLabelFormatter}
+          tooltipValueFormatter={tooltipValueFormatter}
         />
       </Chart.Root>
     );
@@ -1110,6 +1131,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
       showLegend={showLegend}
       maxLegendItems={fullLegend ? Infinity : 5}
       legendAggregation={config.aggregation}
+      legendValueFormatter={tooltipValueFormatter}
       minHeight="300px"
       fillContainer
       onViewAllLegendItems={onViewAllLegendItems}
@@ -1122,6 +1144,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
         yAxisProps={yAxisProps}
         stacked={stacked && visibleSeries.length > 1}
         tooltipLabelFormatter={tooltipLabelFormatter}
+        tooltipValueFormatter={tooltipValueFormatter}
         lineType="linear"
       />
     </Chart.Root>
@@ -1129,9 +1152,13 @@ export const QueryResultsChart = memo(function QueryResultsChart({
 });
 
 /**
- * Creates a Y-axis value formatter based on the data range
+ * Creates a Y-axis value formatter based on the data range and optional format hint
  */
-function createYAxisFormatter(data: Record<string, unknown>[], series: string[]) {
+function createYAxisFormatter(
+  data: Record<string, unknown>[],
+  series: string[],
+  format?: ColumnFormatType
+) {
   // Find min and max values across all series
   let minVal = Infinity;
   let maxVal = -Infinity;
@@ -1148,6 +1175,46 @@ function createYAxisFormatter(data: Record<string, unknown>[], series: string[])
 
   const range = maxVal - minVal;
 
+  // Format-aware formatters
+  if (format === "bytes" || format === "decimalBytes") {
+    const divisor = format === "bytes" ? 1024 : 1000;
+    const units =
+      format === "bytes"
+        ? ["B", "KiB", "MiB", "GiB", "TiB"]
+        : ["B", "KB", "MB", "GB", "TB"];
+    return (value: number): string => {
+      if (value === 0) return "0 B";
+      // Use consistent unit for all ticks based on max value
+      const i = Math.min(
+        Math.max(0, Math.floor(Math.log(Math.abs(maxVal || 1)) / Math.log(divisor))),
+        units.length - 1
+      );
+      const scaled = value / Math.pow(divisor, i);
+      return `${scaled.toFixed(scaled < 10 ? 1 : 0)} ${units[i]}`;
+    };
+  }
+
+  if (format === "percent") {
+    return (value: number): string => `${value.toFixed(range < 1 ? 2 : 1)}%`;
+  }
+
+  if (format === "duration") {
+    return (value: number): string => formatDurationMilliseconds(value, { style: "short" });
+  }
+
+  if (format === "durationSeconds") {
+    return (value: number): string =>
+      formatDurationMilliseconds(value * 1000, { style: "short" });
+  }
+
+  if (format === "costInDollars" || format === "cost") {
+    return (value: number): string => {
+      const dollars = format === "cost" ? value / 100 : value;
+      return formatCurrencyAccurate(dollars);
+    };
+  }
+
+  // Default formatter
   return (value: number): string => {
     // Use abbreviations for large numbers
     if (Math.abs(value) >= 1_000_000) {

apps/webapp/app/components/code/TSQLResultsTable.tsx

@@ -35,6 +35,7 @@ import { useCopy } from "~/hooks/useCopy";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { cn } from "~/utils/cn";
+import { formatBytes, formatDecimalBytes, formatQuantity } from "~/utils/columnFormat";
 import { formatCurrencyAccurate, formatNumber } from "~/utils/numberFormatter";
 import { v3ProjectPath, v3RunPathFromFriendlyId } from "~/utils/pathBuilder";
 import { ChartBlankState } from "../primitives/charts/ChartBlankState";
@@ -66,9 +67,10 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string
   if (value === null) return "NULL";
   if (value === undefined) return "";
 
-  // Handle custom render types
-  if (column.customRenderType) {
-    switch (column.customRenderType) {
+  // Handle format hints (from prettyFormat() or auto-populated from customRenderType)
+  const formatType = column.format ?? column.customRenderType;
+  if (formatType) {
+    switch (formatType) {
       case "duration":
         if (typeof value === "number") {
           return formatDurationMilliseconds(value, { style: "short" });
@@ -95,6 +97,26 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string
           return value;
         }
         break;
+      case "bytes":
+        if (typeof value === "number") {
+          return formatBytes(value);
+        }
+        break;
+      case "decimalBytes":
+        if (typeof value === "number") {
+          return formatDecimalBytes(value);
+        }
+        break;
+      case "percent":
+        if (typeof value === "number") {
+          return `${value.toFixed(2)}%`;
+        }
+        break;
+      case "quantity":
+        if (typeof value === "number") {
+          return formatQuantity(value);
+        }
+        break;
     }
   }
 
@@ -222,6 +244,21 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number
   if (value === null) return 4; // "NULL"
   if (value === undefined) return 9; // "UNDEFINED"
 
+  // Handle format hint types - estimate their rendered width
+  const fmt = column.format;
+  if (fmt === "bytes" || fmt === "decimalBytes") {
+    // e.g., "1.50 GiB" or "256.00 MB"
+    return 12;
+  }
+  if (fmt === "percent") {
+    // e.g., "45.23%"
+    return 8;
+  }
+  if (fmt === "quantity") {
+    // e.g., "1.50M"
+    return 8;
+  }
+
   // Handle custom render types - estimate their rendered width
   if (column.customRenderType) {
     switch (column.customRenderType) {
@@ -263,6 +300,8 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number
         return typeof value === "string" ? Math.min(value.length, 20) : 12;
       case "queue":
         return typeof value === "string" ? Math.min(value.length, 25) : 15;
+      case "deploymentId":
+        return typeof value === "string" ? Math.min(value.length, 25) : 20;
     }
   }
 
@@ -394,6 +433,10 @@ function isRightAlignedColumn(column: OutputColumnMetadata): boolean {
   ) {
     return true;
   }
+  const fmt = column.format;
+  if (fmt === "bytes" || fmt === "decimalBytes" || fmt === "percent" || fmt === "quantity") {
+    return true;
+  }
   return isNumericType(column.type);
 }
 
@@ -476,6 +519,32 @@ function CellValue({
     return <pre className="text-text-dimmed">UNDEFINED</pre>;
   }
 
+  // Check format hint for new format types (from prettyFormat())
+  if (column.format && !column.customRenderType) {
+    switch (column.format) {
+      case "bytes":
+        if (typeof value === "number") {
+          return <span className="tabular-nums">{formatBytes(value)}</span>;
+        }
+        break;
+      case "decimalBytes":
+        if (typeof value === "number") {
+          return <span className="tabular-nums">{formatDecimalBytes(value)}</span>;
+        }
+        break;
+      case "percent":
+        if (typeof value === "number") {
+          return <span className="tabular-nums">{value.toFixed(2)}%</span>;
+        }
+        break;
+      case "quantity":
+        if (typeof value === "number") {
+          return <span className="tabular-nums">{formatQuantity(value)}</span>;
+        }
+        break;
+    }
+  }
+
   // First check customRenderType for special rendering
   if (column.customRenderType) {
     switch (column.customRenderType) {
@@ -577,6 +646,19 @@ function CellValue({
         }
         return <span>{String(value)}</span>;
       }
+      case "deploymentId": {
+        if (typeof value === "string" && value.startsWith("deployment_")) {
+          return (
+            <SimpleTooltip
+              content="Jump to deployment"
+              disableHoverableContent
+              hidden={!hovered}
+              button={<TextLink to={`/deployments/${value}`}>{value}</TextLink>}
+            />
+          );
+        }
+        return <span>{String(value)}</span>;
+      }
     }
   }
 

apps/webapp/app/components/primitives/charts/BigNumberCard.tsx

@@ -1,10 +1,11 @@
-import type { OutputColumnMetadata } from "@internal/tsql";
+import type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql";
 import { Hash } from "lucide-react";
 import { useMemo } from "react";
 import type {
   BigNumberAggregationType,
   BigNumberConfiguration,
 } from "~/components/metrics/QueryWidget";
+import { createValueFormatter } from "~/utils/columnFormat";
 import { AnimatedNumber } from "../AnimatedNumber";
 import { ChartBlankState } from "./ChartBlankState";
 import { Spinner } from "../Spinner";
@@ -130,6 +131,15 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
     return aggregateValues(values, aggregation);
   }, [rows, column, aggregation, sortDirection]);
 
+  // Look up column format for format-aware display
+  const columnValueFormatter = useMemo(() => {
+    const columnMeta = columns.find((c) => c.name === column);
+    const formatType = (columnMeta?.format ?? columnMeta?.customRenderType) as
+      | ColumnFormatType
+      | undefined;
+    return createValueFormatter(formatType);
+  }, [columns, column]);
+
   if (isLoading) {
     return (
       <div className="grid h-full place-items-center [container-type:size]">
@@ -142,6 +152,21 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
     return <ChartBlankState icon={Hash} message="No data to display" />;
   }
 
+  // Use format-aware formatter when available
+  if (columnValueFormatter) {
+    return (
+      <div className="h-full w-full [container-type:size]">
+        <div className="grid h-full w-full place-items-center">
+          <div className="flex items-baseline gap-[0.15em] whitespace-nowrap text-[clamp(24px,12cqw,96px)] font-normal tabular-nums leading-none text-text-bright">
+            {prefix && <span>{prefix}</span>}
+            <span>{columnValueFormatter(result)}</span>
+            {suffix && <span className="text-[0.4em] text-text-dimmed">{suffix}</span>}
+          </div>
+        </div>
+      </div>
+    );
+  }
+
   const { displayValue, unitSuffix, decimalPlaces } = abbreviate
     ? abbreviateValue(result)
     : { displayValue: result, unitSuffix: undefined, decimalPlaces: getDecimalPlaces(result) };
@@ -149,7 +174,7 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
   return (
     <div className="h-full w-full [container-type:size]">
       <div className="grid h-full w-full place-items-center">
-        <div className="flex items-baseline gap-[0.15em] whitespace-nowrap font-normal tabular-nums leading-none text-text-bright text-[clamp(24px,12cqw,96px)]">
+        <div className="flex items-baseline gap-[0.15em] whitespace-nowrap text-[clamp(24px,12cqw,96px)] font-normal tabular-nums leading-none text-text-bright">
           {prefix && <span>{prefix}</span>}
           <AnimatedNumber value={displayValue} decimalPlaces={decimalPlaces} />
           {(unitSuffix || suffix) && (

apps/webapp/app/components/primitives/charts/Chart.tsx

@@ -104,6 +104,8 @@ const ChartTooltipContent = React.forwardRef<
       indicator?: "line" | "dot" | "dashed";
       nameKey?: string;
       labelKey?: string;
+      /** Optional formatter for numeric values (e.g. bytes, duration) */
+      valueFormatter?: (value: number) => string;
     }
 >(
   (
@@ -121,6 +123,7 @@ const ChartTooltipContent = React.forwardRef<
       color,
       nameKey,
       labelKey,
+      valueFormatter,
     },
     ref
   ) => {
@@ -221,9 +224,11 @@ const ChartTooltipContent = React.forwardRef<
                           {itemConfig?.label || item.name}
                         </span>
                       </div>
-                      {item.value && (
+                      {item.value != null && (
                         <span className="text-foreground font-mono font-medium tabular-nums">
-                          {item.value.toLocaleString()}
+                          {valueFormatter && typeof item.value === "number"
+                            ? valueFormatter(item.value)
+                            : item.value.toLocaleString()}
                         </span>
                       )}
                     </div>

apps/webapp/app/components/primitives/charts/ChartBar.tsx

@@ -38,6 +38,8 @@ export type ChartBarRendererProps = {
   referenceLine?: ReferenceLineProps;
   /** Custom tooltip label formatter */
   tooltipLabelFormatter?: (label: string, payload: any[]) => string;
+  /** Optional formatter for numeric tooltip values (e.g. bytes, duration) */
+  tooltipValueFormatter?: (value: number) => string;
   /** Width injected by ResponsiveContainer */
   width?: number;
   /** Height injected by ResponsiveContainer */
@@ -62,6 +64,7 @@ export function ChartBarRenderer({
   yAxisProps: yAxisPropsProp,
   referenceLine,
   tooltipLabelFormatter,
+  tooltipValueFormatter,
   width,
   height,
 }: ChartBarRendererProps) {
@@ -159,7 +162,7 @@ export function ChartBarRenderer({
           showLegend ? (
             () => null
           ) : tooltipLabelFormatter ? (
-            <ChartTooltipContent />
+            <ChartTooltipContent valueFormatter={tooltipValueFormatter} />
           ) : (
             <ZoomTooltip
               isSelecting={zoom?.isSelecting}

apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx

@@ -26,6 +26,8 @@ export type ChartLegendCompoundProps = {
   totalLabel?: string;
   /** Aggregation method – controls the header label and how totals are computed */
   aggregation?: AggregationType;
+  /** Optional formatter for numeric values (e.g. bytes, duration) */
+  valueFormatter?: (value: number) => string;
   /** Callback when "View all" button is clicked */
   onViewAllLegendItems?: () => void;
   /** When true, constrains legend to max 50% height with scrolling */
@@ -50,6 +52,7 @@ export function ChartLegendCompound({
   className,
   totalLabel,
   aggregation,
+  valueFormatter,
   onViewAllLegendItems,
   scrollable = false,
 }: ChartLegendCompoundProps) {
@@ -180,7 +183,11 @@ export function ChartLegendCompound({
         <span className="font-medium">{currentTotalLabel}</span>
         <span className="font-medium tabular-nums">
           {currentTotal != null ? (
-            <AnimatedNumber value={currentTotal} duration={0.25} />
+            valueFormatter ? (
+              valueFormatter(currentTotal)
+            ) : (
+              <AnimatedNumber value={currentTotal} duration={0.25} />
+            )
           ) : (
             "\u2013"
           )}
@@ -252,7 +259,11 @@ export function ChartLegendCompound({
                   )}
                 >
                   {total != null ? (
-                    <AnimatedNumber value={total} duration={0.25} />
+                    valueFormatter ? (
+                      valueFormatter(total)
+                    ) : (
+                      <AnimatedNumber value={total} duration={0.25} />
+                    )
                   ) : (
                     "\u2013"
                   )}
@@ -269,6 +280,7 @@ export function ChartLegendCompound({
               item={legendItems.hoveredHiddenItem}
               value={currentData[legendItems.hoveredHiddenItem.dataKey] ?? null}
               remainingCount={legendItems.remaining - 1}
+              valueFormatter={valueFormatter}
             />
           ) : (
             <ViewAllDataRow
@@ -315,9 +327,10 @@ type HoveredHiddenItemRowProps = {
   item: { dataKey: string; color?: string; label: React.ReactNode };
   value: number | null;
   remainingCount: number;
+  valueFormatter?: (value: number) => string;
 };
 
-function HoveredHiddenItemRow({ item, value, remainingCount }: HoveredHiddenItemRowProps) {
+function HoveredHiddenItemRow({ item, value, remainingCount, valueFormatter }: HoveredHiddenItemRowProps) {
   return (
     <div className="relative flex w-full items-center justify-between gap-2 rounded px-2 py-1">
       {/* Active highlight background */}
@@ -339,7 +352,15 @@ function HoveredHiddenItemRow({ item, value, remainingCount }: HoveredHiddenItem
           {remainingCount > 0 && <span className="text-text-dimmed">+{remainingCount} more</span>}
         </div>
         <span className="tabular-nums text-text-bright">
-          {value != null ? <AnimatedNumber value={value} duration={0.25} /> : "\u2013"}
+          {value != null ? (
+            valueFormatter ? (
+              valueFormatter(value)
+            ) : (
+              <AnimatedNumber value={value} duration={0.25} />
+            )
+          ) : (
+            "\u2013"
+          )}
         </span>
       </div>
     </div>

apps/webapp/app/components/primitives/charts/ChartLine.tsx

@@ -51,6 +51,8 @@ export type ChartLineRendererProps = {
   stacked?: boolean;
   /** Custom tooltip label formatter */
   tooltipLabelFormatter?: (label: string, payload: any[]) => string;
+  /** Optional formatter for numeric tooltip values (e.g. bytes, duration) */
+  tooltipValueFormatter?: (value: number) => string;
   /** Width injected by ResponsiveContainer */
   width?: number;
   /** Height injected by ResponsiveContainer */
@@ -75,6 +77,7 @@ export function ChartLineRenderer({
   yAxisProps: yAxisPropsProp,
   stacked = false,
   tooltipLabelFormatter,
+  tooltipValueFormatter,
   width,
   height,
 }: ChartLineRendererProps) {
@@ -157,7 +160,13 @@ export function ChartLineRenderer({
         {/* When legend is shown below, render tooltip with cursor only (no content popup) */}
         <ChartTooltip
           cursor={{ stroke: "rgba(255, 255, 255, 0.1)", strokeWidth: 1 }}
-          content={showLegend ? () => null : <ChartTooltipContent indicator="line" />}
+          content={
+            showLegend ? (
+              () => null
+            ) : (
+              <ChartTooltipContent indicator="line" valueFormatter={tooltipValueFormatter} />
+            )
+          }
           labelFormatter={tooltipLabelFormatter}
         />
         {/* Note: Legend is now rendered by ChartRoot outside the chart container */}
@@ -205,7 +214,13 @@ export function ChartLineRenderer({
       {/* When legend is shown below, render tooltip with cursor only (no content popup) */}
       <ChartTooltip
         cursor={{ stroke: "rgba(255, 255, 255, 0.1)", strokeWidth: 1 }}
-        content={showLegend ? () => null : <ChartTooltipContent />}
+        content={
+          showLegend ? (
+            () => null
+          ) : (
+            <ChartTooltipContent valueFormatter={tooltipValueFormatter} />
+          )
+        }
         labelFormatter={tooltipLabelFormatter}
       />
       {/* Note: Legend is now rendered by ChartRoot outside the chart container */}

apps/webapp/app/components/primitives/charts/ChartRoot.tsx

@@ -34,6 +34,8 @@ export type ChartRootProps = {
   legendTotalLabel?: string;
   /** Aggregation method used by the legend to compute totals (defaults to sum behavior) */
   legendAggregation?: AggregationType;
+  /** Optional formatter for numeric legend values (e.g. bytes, duration) */
+  legendValueFormatter?: (value: number) => string;
   /** Callback when "View all" legend button is clicked */
   onViewAllLegendItems?: () => void;
   /** When true, constrains legend to max 50% height with scrolling */
@@ -82,6 +84,7 @@ export function ChartRoot({
   maxLegendItems = 5,
   legendTotalLabel,
   legendAggregation,
+  legendValueFormatter,
   onViewAllLegendItems,
   legendScrollable = false,
   fillContainer = false,
@@ -108,6 +111,7 @@ export function ChartRoot({
         maxLegendItems={maxLegendItems}
         legendTotalLabel={legendTotalLabel}
         legendAggregation={legendAggregation}
+        legendValueFormatter={legendValueFormatter}
         onViewAllLegendItems={onViewAllLegendItems}
         legendScrollable={legendScrollable}
         fillContainer={fillContainer}
@@ -126,6 +130,7 @@ type ChartRootInnerProps = {
   maxLegendItems?: number;
   legendTotalLabel?: string;
   legendAggregation?: AggregationType;
+  legendValueFormatter?: (value: number) => string;
   onViewAllLegendItems?: () => void;
   legendScrollable?: boolean;
   fillContainer?: boolean;
@@ -140,6 +145,7 @@ function ChartRootInner({
   maxLegendItems = 5,
   legendTotalLabel,
   legendAggregation,
+  legendValueFormatter,
   onViewAllLegendItems,
   legendScrollable = false,
   fillContainer = false,
@@ -184,6 +190,7 @@ function ChartRootInner({
           maxItems={maxLegendItems}
           totalLabel={legendTotalLabel}
           aggregation={legendAggregation}
+          valueFormatter={legendValueFormatter}
           onViewAllLegendItems={onViewAllLegendItems}
           scrollable={legendScrollable}
         />

apps/webapp/app/components/query/QueryEditor.tsx

@@ -1175,9 +1175,9 @@ function QueryResultsCallouts({
     <div className="flex flex-col gap-2 px-2 pt-2">
       {hiddenColumns && hiddenColumns.length > 0 && (
         <Callout variant="warning" className="shrink-0 text-sm">
-          <code>SELECT *</code> doesn't return all columns because it's slow. The following columns
-          are not shown: <span className="font-mono text-xs">{hiddenColumns.join(", ")}</span>.
-          Specify them explicitly to include them.
+          <code>SELECT *</code> returns core columns only. To include{" "}
+          <span className="font-mono text-xs">{hiddenColumns.join(", ")}</span>, add them to your
+          SELECT explicitly.
         </Callout>
       )}
       {periodClipped && (

apps/webapp/app/env.server.ts

@@ -372,6 +372,7 @@ const EnvironmentSchema = z
 
     // Development OTEL environment variables
     DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(),
+    DEV_OTEL_METRICS_ENDPOINT: z.string().optional(),
     // If this is set to 1, then the below variables are used to configure the batch processor for spans and logs
     DEV_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"),
     DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"),
@@ -382,6 +383,9 @@ const EnvironmentSchema = z
     DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
     DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
     DEV_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
+    DEV_OTEL_METRICS_EXPORT_INTERVAL_MILLIS: z.string().optional(),
+    DEV_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS: z.string().optional(),
+    DEV_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS: z.string().optional(),
 
     PROD_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"),
     PROD_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"),
@@ -392,6 +396,9 @@ const EnvironmentSchema = z
     PROD_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
     PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
     PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
+    PROD_OTEL_METRICS_EXPORT_INTERVAL_MILLIS: z.string().optional(),
+    PROD_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS: z.string().optional(),
+    PROD_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS: z.string().optional(),
 
     TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"),
     TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"),
@@ -1229,6 +1236,9 @@ const EnvironmentSchema = z
     EVENTS_CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"),
     EVENTS_CLICKHOUSE_BATCH_SIZE: z.coerce.number().int().default(1000),
     EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000),
+    METRICS_CLICKHOUSE_BATCH_SIZE: z.coerce.number().int().default(10000),
+    METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000),
+    METRICS_CLICKHOUSE_MAX_CONCURRENCY: z.coerce.number().int().default(3),
     EVENTS_CLICKHOUSE_INSERT_STRATEGY: z.enum(["insert", "insert_async"]).default("insert"),
     EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT: z.string().default("1"),
     EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: z.coerce.number().int().default(10485760),

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/AITabContent.tsx

@@ -30,6 +30,8 @@ export function AITabContent({
     "Top 50 most expensive runs this week",
     "Average execution duration by task this week",
     "Run counts by tag in the past 7 days",
+    "CPU utilization over time by task",
+    "Peak memory usage per run",
   ];
 
   return (

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx

@@ -1,6 +1,9 @@
+import { useState } from "react";
 import { Header3 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import SegmentedControl from "~/components/primitives/SegmentedControl";
 import type { QueryScope } from "~/services/queryService.server";
+import { querySchemas } from "~/v3/querySchemas";
 import { TryableCodeBlock } from "./TRQLGuideContent";
 
 // Example queries for the Examples tab
@@ -9,6 +12,7 @@ export const exampleQueries: Array<{
   description: string;
   query: string;
   scope: QueryScope;
+  table: string;
 }> = [
   {
     title: "Failed runs by task (past 7 days)",
@@ -23,6 +27,7 @@ GROUP BY task_identifier
 ORDER BY failed_count DESC
 LIMIT 20`,
     scope: "environment",
+    table: "runs",
   },
   {
     title: "Execution duration p50 by task (past 7d)",
@@ -37,6 +42,7 @@ GROUP BY task_identifier
 ORDER BY p50_duration_ms DESC
 LIMIT 20`,
     scope: "environment",
+    table: "runs",
   },
   {
     title: "Runs over time",
@@ -50,6 +56,7 @@ GROUP BY timeBucket
 ORDER BY timeBucket
 LIMIT 1000`,
     scope: "environment",
+    table: "runs",
   },
   {
     title: "Most expensive 100 runs (past 7d)",
@@ -67,17 +74,75 @@ WHERE triggered_at > now() - INTERVAL 7 DAY
 ORDER BY total_cost DESC
 LIMIT 100`,
     scope: "environment",
+    table: "runs",
+  },
+  {
+    title: "CPU utilization over time",
+    description: "Track process CPU utilization bucketed over time.",
+    query: `SELECT
+  timeBucket(),
+  avg(metric_value) AS avg_cpu
+FROM metrics
+WHERE metric_name = 'process.cpu.utilization'
+GROUP BY timeBucket
+ORDER BY timeBucket
+LIMIT 1000`,
+    scope: "environment",
+    table: "metrics",
+  },
+  {
+    title: "Memory usage by task (past 7d)",
+    description: "Average memory usage per task identifier over the last 7 days.",
+    query: `SELECT
+  task_identifier,
+  avg(metric_value) AS avg_memory
+FROM metrics
+WHERE metric_name = 'system.memory.usage'
+  AND bucket_start > now() - INTERVAL 7 DAY
+GROUP BY task_identifier
+ORDER BY avg_memory DESC
+LIMIT 20`,
+    scope: "environment",
+    table: "metrics",
+  },
+  {
+    title: "Available metric names",
+    description: "List all distinct metric names collected in your environment.",
+    query: `SELECT
+  metric_name,
+  count() AS sample_count
+FROM metrics
+GROUP BY metric_name
+ORDER BY sample_count DESC
+LIMIT 100`,
+    scope: "environment",
+    table: "metrics",
   },
 ];
 
+const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name }));
+
 export function ExamplesContent({
   onTryExample,
 }: {
   onTryExample: (query: string, scope: QueryScope) => void;
 }) {
+  const [selectedTable, setSelectedTable] = useState(querySchemas[0].name);
+  const filtered = exampleQueries.filter((e) => e.table === selectedTable);
+
   return (
     <div className="space-y-6">
-      {exampleQueries.map((example) => (
+      <div className="sticky top-0 z-10 bg-background-bright pb-3">
+        <SegmentedControl
+          name="examples-table-selector"
+          value={selectedTable}
+          options={tableOptions}
+          variant="secondary/small"
+          fullWidth
+          onChange={setSelectedTable}
+        />
+      </div>
+      {filtered.map((example) => (
         <div key={example.title}>
           <Header3 className="mb-1 text-text-bright">{example.title}</Header3>
           <Paragraph variant="small" className="mb-2 text-text-dimmed">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx

@@ -1,8 +1,10 @@
+import { useState } from "react";
 import type { ColumnSchema } from "@internal/tsql";
 import { Badge } from "~/components/primitives/Badge";
 import { CopyableText } from "~/components/primitives/CopyableText";
 import { Header3 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import SegmentedControl from "~/components/primitives/SegmentedControl";
 import { querySchemas } from "~/v3/querySchemas";
 
 function ColumnHelpItem({ col }: { col: ColumnSchema }) {
@@ -42,26 +44,36 @@ function ColumnHelpItem({ col }: { col: ColumnSchema }) {
   );
 }
 
+const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name }));
+
 export function TableSchemaContent() {
+  const [selectedTable, setSelectedTable] = useState(querySchemas[0].name);
+  const table = querySchemas.find((s) => s.name === selectedTable) ?? querySchemas[0];
+
   return (
     <div>
-      {querySchemas.map((table) => (
-        <div key={table.name} className="mb-6">
-          <div className="mb-2">
-            <Header3 className="font-mono text-text-bright">{table.name}</Header3>
-            {table.description && (
-              <Paragraph variant="small" className="mt-1 text-text-dimmed">
-                {table.description}
-              </Paragraph>
-            )}
-          </div>
-          <div className="flex flex-col gap-2 divide-y divide-grid-dimmed">
-            {Object.values(table.columns).map((col) => (
-              <ColumnHelpItem key={col.name} col={col} />
-            ))}
-          </div>
-        </div>
-      ))}
+      <div className="sticky top-0 z-10 bg-background-bright pb-3">
+        <SegmentedControl
+          name="table-schema-selector"
+          value={selectedTable}
+          options={tableOptions}
+          variant="secondary/small"
+          fullWidth
+          onChange={setSelectedTable}
+        />
+      </div>
+      <div className="mb-2">
+        {table.description && (
+          <Paragraph variant="small" className="text-text-dimmed">
+            {table.description}
+          </Paragraph>
+        )}
+      </div>
+      <div className="flex flex-col gap-2 divide-y divide-grid-dimmed">
+        {Object.values(table.columns).map((col) => (
+          <ColumnHelpItem key={col.name} col={col} />
+        ))}
+      </div>
     </div>
   );
 }

apps/webapp/app/routes/deployments.$deploymentParam.ts

@@ -0,0 +1,70 @@
+import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage } from "~/models/message.server";
+import { requireUser } from "~/services/session.server";
+import { rootPath, v3DeploymentPath } from "~/utils/pathBuilder";
+
+const ParamsSchema = z.object({
+  deploymentParam: z.string(),
+});
+
+export async function loader({ params, request }: LoaderFunctionArgs) {
+  const user = await requireUser(request);
+
+  const { deploymentParam } = ParamsSchema.parse(params);
+
+  const deployment = await prisma.workerDeployment.findFirst({
+    where: {
+      friendlyId: deploymentParam,
+      project: {
+        organization: {
+          members: {
+            some: {
+              userId: user.id,
+            },
+          },
+        },
+      },
+    },
+    select: {
+      shortCode: true,
+      environment: {
+        select: {
+          slug: true,
+        },
+      },
+      project: {
+        select: {
+          slug: true,
+          organization: {
+            select: {
+              slug: true,
+            },
+          },
+        },
+      },
+    },
+  });
+
+  if (!deployment) {
+    return redirectWithErrorMessage(
+      rootPath(),
+      request,
+      "Deployment either doesn't exist or you don't have permission to view it",
+      {
+        ephemeral: false,
+      }
+    );
+  }
+
+  return redirect(
+    v3DeploymentPath(
+      { slug: deployment.project.organization.slug },
+      { slug: deployment.project.slug },
+      { slug: deployment.environment.slug },
+      { shortCode: deployment.shortCode },
+      0
+    )
+  );
+}

apps/webapp/app/routes/otel.v1.metrics.ts

@@ -0,0 +1,41 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import {
+  ExportMetricsServiceRequest,
+  ExportMetricsServiceResponse,
+} from "@trigger.dev/otlp-importer";
+import { otlpExporter } from "~/v3/otlpExporter.server";
+
+export async function action({ request }: ActionFunctionArgs) {
+  try {
+    const contentType = request.headers.get("content-type")?.toLowerCase() ?? "";
+
+    if (contentType.startsWith("application/json")) {
+      const body = await request.json();
+
+      const exportResponse = await otlpExporter.exportMetrics(
+        body as ExportMetricsServiceRequest
+      );
+
+      return json(exportResponse, { status: 200 });
+    } else if (contentType.startsWith("application/x-protobuf")) {
+      const buffer = await request.arrayBuffer();
+
+      const exportRequest = ExportMetricsServiceRequest.decode(new Uint8Array(buffer));
+
+      const exportResponse = await otlpExporter.exportMetrics(exportRequest);
+
+      return new Response(ExportMetricsServiceResponse.encode(exportResponse).finish(), {
+        status: 200,
+      });
+    } else {
+      return new Response(
+        "Unsupported content type. Must be either application/x-protobuf or application/json",
+        { status: 400 }
+      );
+    }
+  } catch (error) {
+    console.error(error);
+
+    return new Response("Internal Server Error", { status: 500 });
+  }
+}

apps/webapp/app/services/queryService.server.ts

@@ -152,7 +152,14 @@ export async function executeQuery<TOut extends z.ZodSchema>(
     return { success: false, error: new QueryError(errorMessage, { query: options.query }) };
   }
 
-  // Build time filter fallback for triggered_at column
+  // Detect which table the query targets to determine the time column
+  // Each table schema declares its primary time column via timeConstraint
+  const matchedSchema = querySchemas.find((s) =>
+    new RegExp(`\\bFROM\\s+${s.name}\\b`, "i").test(options.query)
+  );
+  const timeColumn = matchedSchema?.timeConstraint ?? "triggered_at";
+
+  // Build time filter fallback for the table's time column
   const defaultPeriod = await getDefaultPeriod(organizationId);
   const timeFilter = timeFilters({
     period: period ?? undefined,
@@ -173,15 +180,15 @@ export async function executeQuery<TOut extends z.ZodSchema>(
   }
 
   // Build the fallback WHERE condition based on what the user specified
-  let triggeredAtFallback: WhereClauseCondition;
+  let timeFallback: WhereClauseCondition;
   if (timeFilter.from && timeFilter.to) {
-    triggeredAtFallback = { op: "between", low: timeFilter.from, high: timeFilter.to };
+    timeFallback = { op: "between", low: timeFilter.from, high: timeFilter.to };
   } else if (timeFilter.from) {
-    triggeredAtFallback = { op: "gte", value: timeFilter.from };
+    timeFallback = { op: "gte", value: timeFilter.from };
   } else if (timeFilter.to) {
-    triggeredAtFallback = { op: "lte", value: timeFilter.to };
+    timeFallback = { op: "lte", value: timeFilter.to };
   } else {
-    triggeredAtFallback = { op: "gte", value: requestedFromDate! };
+    timeFallback = { op: "gte", value: requestedFromDate! };
   }
 
   const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30);
@@ -196,7 +203,7 @@ export async function executeQuery<TOut extends z.ZodSchema>(
     project_id:
       scope === "project" || scope === "environment" ? { op: "eq", value: projectId } : undefined,
     environment_id: scope === "environment" ? { op: "eq", value: environmentId } : undefined,
-    triggered_at: { op: "gte", value: maxQueryPeriodDate },
+    [timeColumn]: { op: "gte", value: maxQueryPeriodDate },
     // Optional filters for tasks and queues
     task_identifier:
       taskIdentifiers && taskIdentifiers.length > 0
@@ -238,7 +245,7 @@ export async function executeQuery<TOut extends z.ZodSchema>(
       enforcedWhereClause,
       fieldMappings,
       whereClauseFallback: {
-        triggered_at: triggeredAtFallback,
+        [timeColumn]: timeFallback,
       },
       timeRange,
       clickhouseSettings: {

apps/webapp/app/utils/columnFormat.ts

@@ -0,0 +1,70 @@
+import type { ColumnFormatType } from "@internal/clickhouse";
+import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
+import { formatCurrencyAccurate } from "~/utils/numberFormatter";
+
+/**
+ * Format a number as binary bytes (KiB, MiB, GiB, TiB)
+ */
+export function formatBytes(bytes: number): string {
+  if (bytes === 0) return "0 B";
+  const units = ["B", "KiB", "MiB", "GiB", "TiB"];
+  const i = Math.min(
+    Math.max(0, Math.floor(Math.log(Math.abs(bytes)) / Math.log(1024))),
+    units.length - 1
+  );
+  return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 2)} ${units[i]}`;
+}
+
+/**
+ * Format a number as decimal bytes (KB, MB, GB, TB)
+ */
+export function formatDecimalBytes(bytes: number): string {
+  if (bytes === 0) return "0 B";
+  const units = ["B", "KB", "MB", "GB", "TB"];
+  const i = Math.min(
+    Math.max(0, Math.floor(Math.log(Math.abs(bytes)) / Math.log(1000))),
+    units.length - 1
+  );
+  return `${(bytes / Math.pow(1000, i)).toFixed(i === 0 ? 0 : 2)} ${units[i]}`;
+}
+
+/**
+ * Format a large number with human-readable suffix (K, M, B)
+ */
+export function formatQuantity(value: number): string {
+  const abs = Math.abs(value);
+  if (abs >= 1_000_000_000) return `${(value / 1_000_000_000).toFixed(2)}B`;
+  if (abs >= 1_000_000) return `${(value / 1_000_000).toFixed(2)}M`;
+  if (abs >= 1_000) return `${(value / 1_000).toFixed(2)}K`;
+  return value.toLocaleString();
+}
+
+/**
+ * Creates a value formatter function for a given column format type.
+ * Used by chart tooltips, legend values, and big number cards.
+ */
+export function createValueFormatter(
+  format?: ColumnFormatType
+): ((value: number) => string) | undefined {
+  if (!format) return undefined;
+  switch (format) {
+    case "bytes":
+      return (v) => formatBytes(v);
+    case "decimalBytes":
+      return (v) => formatDecimalBytes(v);
+    case "percent":
+      return (v) => `${v.toFixed(2)}%`;
+    case "quantity":
+      return (v) => formatQuantity(v);
+    case "duration":
+      return (v) => formatDurationMilliseconds(v, { style: "short" });
+    case "durationSeconds":
+      return (v) => formatDurationMilliseconds(v * 1000, { style: "short" });
+    case "costInDollars":
+      return (v) => formatCurrencyAccurate(v);
+    case "cost":
+      return (v) => formatCurrencyAccurate(v / 100);
+    default:
+      return undefined;
+  }
+}

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -956,6 +956,34 @@ async function resolveBuiltInDevVariables(runtimeEnvironment: RuntimeEnvironment
     },
   ];
 
+  if (env.DEV_OTEL_METRICS_ENDPOINT) {
+    result.push({
+      key: "TRIGGER_OTEL_METRICS_ENDPOINT",
+      value: env.DEV_OTEL_METRICS_ENDPOINT,
+    });
+  }
+
+  if (env.DEV_OTEL_METRICS_EXPORT_INTERVAL_MILLIS) {
+    result.push({
+      key: "TRIGGER_OTEL_METRICS_EXPORT_INTERVAL_MILLIS",
+      value: env.DEV_OTEL_METRICS_EXPORT_INTERVAL_MILLIS,
+    });
+  }
+
+  if (env.DEV_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS) {
+    result.push({
+      key: "TRIGGER_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS",
+      value: env.DEV_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS,
+    });
+  }
+
+  if (env.DEV_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS) {
+    result.push({
+      key: "TRIGGER_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS",
+      value: env.DEV_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS,
+    });
+  }
+
   if (env.DEV_OTEL_BATCH_PROCESSING_ENABLED === "1") {
     result = result.concat([
       {
@@ -1087,6 +1115,27 @@ async function resolveBuiltInProdVariables(
     ]);
   }
 
+  if (env.PROD_OTEL_METRICS_EXPORT_INTERVAL_MILLIS) {
+    result.push({
+      key: "TRIGGER_OTEL_METRICS_EXPORT_INTERVAL_MILLIS",
+      value: env.PROD_OTEL_METRICS_EXPORT_INTERVAL_MILLIS,
+    });
+  }
+
+  if (env.PROD_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS) {
+    result.push({
+      key: "TRIGGER_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS",
+      value: env.PROD_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS,
+    });
+  }
+
+  if (env.PROD_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS) {
+    result.push({
+      key: "TRIGGER_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS",
+      value: env.PROD_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS,
+    });
+  }
+
   if (env.PROD_OTEL_BATCH_PROCESSING_ENABLED === "1") {
     result = result.concat([
       {

apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts

@@ -266,6 +266,7 @@ export class ClickhouseEventRepository implements IEventRepository {
         expires_at: convertDateToClickhouseDateTime(
           new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) // 1 year
         ),
+        machine_id: event.machineId ?? "",
       },
       ...this.spanEventsToTaskEventV1Input(event),
     ];

apps/webapp/app/v3/eventRepository/eventRepository.types.ts

@@ -56,6 +56,7 @@ export type CreateEventInput = Omit<
   resourceProperties?: Attributes;
   metadata: Attributes | undefined;
   style: Attributes | undefined;
+  machineId?: string;
 };
 
 export type CreatableEventKind = TaskEventKind;

apps/webapp/app/v3/otlpExporter.server.ts

@@ -4,10 +4,13 @@ import {
   AnyValue,
   ExportLogsServiceRequest,
   ExportLogsServiceResponse,
+  ExportMetricsServiceRequest,
+  ExportMetricsServiceResponse,
   ExportTraceServiceRequest,
   ExportTraceServiceResponse,
   KeyValue,
   ResourceLogs,
+  ResourceMetrics,
   ResourceSpans,
   SeverityNumber,
   Span,
@@ -15,7 +18,10 @@ import {
   Span_SpanKind,
   Status_StatusCode,
 } from "@trigger.dev/otlp-importer";
+import type { MetricsV1Input } from "@internal/clickhouse";
 import { logger } from "~/services/logger.server";
+import { clickhouseClient } from "~/services/clickhouseInstance.server";
+import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
 import { ClickhouseEventRepository } from "./eventRepository/clickhouseEventRepository.server";
 import {
   clickhouseEventRepository,
@@ -42,6 +48,7 @@ class OTLPExporter {
     private readonly _eventRepository: EventRepository,
     private readonly _clickhouseEventRepository: ClickhouseEventRepository,
     private readonly _clickhouseEventRepositoryV2: ClickhouseEventRepository,
+    private readonly _metricsFlushScheduler: DynamicFlushScheduler<MetricsV1Input>,
     private readonly _verbose: boolean,
     private readonly _spanAttributeValueLengthLimit: number
   ) {
@@ -66,6 +73,29 @@ class OTLPExporter {
     });
   }
 
+  async exportMetrics(
+    request: ExportMetricsServiceRequest
+  ): Promise<ExportMetricsServiceResponse> {
+    return await startSpan(this._tracer, "exportMetrics", async (span) => {
+      const rows = this.#filterResourceMetrics(request.resourceMetrics).flatMap(
+        (resourceMetrics) => {
+          return convertMetricsToClickhouseRows(
+            resourceMetrics,
+            this._spanAttributeValueLengthLimit
+          );
+        }
+      );
+
+      span.setAttribute("metric_row_count", rows.length);
+
+      if (rows.length > 0) {
+        this._metricsFlushScheduler.addToBatch(rows);
+      }
+
+      return ExportMetricsServiceResponse.create();
+    });
+  }
+
   async exportLogs(request: ExportLogsServiceRequest): Promise<ExportLogsServiceResponse> {
     return await startSpan(this._tracer, "exportLogs", async (span) => {
       this.#logExportLogsVerbose(request);
@@ -202,6 +232,18 @@ class OTLPExporter {
       return isBoolValue(attribute.value) ? attribute.value.boolValue : false;
     });
   }
+
+  #filterResourceMetrics(resourceMetrics: ResourceMetrics[]): ResourceMetrics[] {
+    return resourceMetrics.filter((rm) => {
+      const triggerAttribute = rm.resource?.attributes.find(
+        (attribute) => attribute.key === SemanticInternalAttributes.TRIGGER
+      );
+
+      if (!triggerAttribute) return false;
+
+      return isBoolValue(triggerAttribute.value) ? triggerAttribute.value.boolValue : false;
+    });
+  }
 }
 
 function convertLogsToCreateableEvents(
@@ -289,6 +331,7 @@ function convertLogsToCreateableEvents(
           projectId: logProperties.projectId ?? resourceProperties.projectId ?? "unknown",
           runId: logProperties.runId ?? resourceProperties.runId ?? "unknown",
           taskSlug: logProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown",
+          machineId: logProperties.machineId ?? resourceProperties.machineId,
           attemptNumber:
             extractNumberAttribute(
               log.attributes ?? [],
@@ -395,6 +438,7 @@ function convertSpansToCreateableEvents(
           projectId: spanProperties.projectId ?? resourceProperties.projectId ?? "unknown",
           runId: spanProperties.runId ?? resourceProperties.runId ?? "unknown",
           taskSlug: spanProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown",
+          machineId: spanProperties.machineId ?? resourceProperties.machineId,
           attemptNumber:
             extractNumberAttribute(
               span.attributes ?? [],
@@ -410,6 +454,194 @@ function convertSpansToCreateableEvents(
   return { events, taskEventStore };
 }
 
+function floorToTenSecondBucket(timeUnixNano: bigint | number): string {
+  const epochMs = Number(BigInt(timeUnixNano) / BigInt(1_000_000));
+  const flooredMs = Math.floor(epochMs / 10_000) * 10_000;
+  const date = new Date(flooredMs);
+  // Format as ClickHouse DateTime: YYYY-MM-DD HH:MM:SS
+  return date.toISOString().replace("T", " ").replace(/\.\d{3}Z$/, "");
+}
+
+function convertMetricsToClickhouseRows(
+  resourceMetrics: ResourceMetrics,
+  spanAttributeValueLengthLimit: number
+): MetricsV1Input[] {
+  const resourceAttributes = resourceMetrics.resource?.attributes ?? [];
+  const resourceProperties = extractEventProperties(resourceAttributes);
+
+  const organizationId = resourceProperties.organizationId ?? "unknown";
+  const projectId = resourceProperties.projectId ?? "unknown";
+  const environmentId = resourceProperties.environmentId ?? "unknown";
+  const resourceCtx = {
+    taskSlug: resourceProperties.taskSlug,
+    runId: resourceProperties.runId,
+    attemptNumber: resourceProperties.attemptNumber,
+    machineId: extractStringAttribute(resourceAttributes, SemanticInternalAttributes.MACHINE_ID),
+    workerId: extractStringAttribute(resourceAttributes, SemanticInternalAttributes.WORKER_ID),
+    workerVersion: extractStringAttribute(
+      resourceAttributes,
+      SemanticInternalAttributes.WORKER_VERSION
+    ),
+  };
+
+  const rows: MetricsV1Input[] = [];
+
+  for (const scopeMetrics of resourceMetrics.scopeMetrics) {
+    for (const metric of scopeMetrics.metrics) {
+      const metricName = metric.name;
+
+      // Process gauge data points
+      if (metric.gauge) {
+        for (const dp of metric.gauge.dataPoints) {
+          const value: number =
+            dp.asDouble !== undefined ? dp.asDouble : dp.asInt !== undefined ? Number(dp.asInt) : 0;
+          const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
+
+          rows.push({
+            organization_id: organizationId,
+            project_id: projectId,
+            environment_id: environmentId,
+            metric_name: metricName,
+            metric_type: "gauge",
+            metric_subject: resolved.machineId ?? "unknown",
+            bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
+            value,
+            attributes: resolved.attributes,
+          });
+        }
+      }
+
+      // Process sum data points
+      if (metric.sum) {
+        for (const dp of metric.sum.dataPoints) {
+          const value: number =
+            dp.asDouble !== undefined ? dp.asDouble : dp.asInt !== undefined ? Number(dp.asInt) : 0;
+          const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
+
+          rows.push({
+            organization_id: organizationId,
+            project_id: projectId,
+            environment_id: environmentId,
+            metric_name: metricName,
+            metric_type: "sum",
+            metric_subject: resolved.machineId ?? "unknown",
+            bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
+            value,
+            attributes: resolved.attributes,
+          });
+        }
+      }
+
+      // Process histogram data points
+      if (metric.histogram) {
+        for (const dp of metric.histogram.dataPoints) {
+          const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
+          const count = Number(dp.count);
+          const sum = dp.sum ?? 0;
+
+          rows.push({
+            organization_id: organizationId,
+            project_id: projectId,
+            environment_id: environmentId,
+            metric_name: metricName,
+            metric_type: "histogram",
+            metric_subject: resolved.machineId ?? "unknown",
+            bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
+            value: count > 0 ? sum / count : 0,
+            attributes: resolved.attributes,
+          });
+        }
+      }
+    }
+  }
+
+  return rows;
+}
+
+// Prefixes injected by TaskContextMetricExporter — these are extracted into
+// the nested `trigger` key and should not appear as top-level user attributes.
+const INTERNAL_METRIC_ATTRIBUTE_PREFIXES = ["ctx.", "worker."];
+
+interface ResourceContext {
+  taskSlug: string | undefined;
+  runId: string | undefined;
+  attemptNumber: number | undefined;
+  machineId: string | undefined;
+  workerId: string | undefined;
+  workerVersion: string | undefined;
+}
+
+function resolveDataPointContext(
+  dpAttributes: KeyValue[],
+  resourceCtx: ResourceContext
+): {
+  machineId: string | undefined;
+  attributes: Record<string, unknown>;
+} {
+  const runId =
+    resourceCtx.runId ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.RUN_ID);
+  const taskSlug =
+    resourceCtx.taskSlug ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.TASK_SLUG);
+  const attemptNumber =
+    resourceCtx.attemptNumber ??
+    extractNumberAttribute(dpAttributes, SemanticInternalAttributes.ATTEMPT_NUMBER);
+  const machineId =
+    resourceCtx.machineId ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.MACHINE_ID);
+  const workerId =
+    resourceCtx.workerId ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.WORKER_ID);
+  const workerVersion =
+    resourceCtx.workerVersion ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.WORKER_VERSION);
+  const machineName = extractStringAttribute(
+    dpAttributes,
+    SemanticInternalAttributes.MACHINE_PRESET_NAME
+  );
+  const environmentType = extractStringAttribute(
+    dpAttributes,
+    SemanticInternalAttributes.ENVIRONMENT_TYPE
+  );
+
+  // Build the trigger context object with only defined values
+  const trigger: Record<string, string | number> = {};
+  if (runId) trigger.run_id = runId;
+  if (taskSlug) trigger.task_slug = taskSlug;
+  if (attemptNumber !== undefined) trigger.attempt_number = attemptNumber;
+  if (machineId) trigger.machine_id = machineId;
+  if (machineName) trigger.machine_name = machineName;
+  if (workerId) trigger.worker_id = workerId;
+  if (workerVersion) trigger.worker_version = workerVersion;
+  if (environmentType) trigger.environment_type = environmentType;
+
+  // Build user attributes, filtering out internal ctx/worker keys
+  const result: Record<string, unknown> = {};
+
+  if (Object.keys(trigger).length > 0) {
+    result.trigger = trigger;
+  }
+
+  for (const attr of dpAttributes) {
+    if (INTERNAL_METRIC_ATTRIBUTE_PREFIXES.some((prefix) => attr.key.startsWith(prefix))) {
+      continue;
+    }
+
+    if (isStringValue(attr.value)) {
+      result[attr.key] = attr.value.stringValue;
+    } else if (isIntValue(attr.value)) {
+      result[attr.key] = Number(attr.value.intValue);
+    } else if (isDoubleValue(attr.value)) {
+      result[attr.key] = attr.value.doubleValue;
+    } else if (isBoolValue(attr.value)) {
+      result[attr.key] = attr.value.boolValue;
+    }
+  }
+
+  return { machineId, attributes: result };
+}
+
 function extractEventProperties(attributes: KeyValue[], prefix?: string) {
   return {
     metadata: convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]),
@@ -428,6 +660,7 @@ function extractEventProperties(attributes: KeyValue[], prefix?: string) {
       SemanticInternalAttributes.ATTEMPT_NUMBER,
     ]),
     taskSlug: extractStringAttribute(attributes, [prefix, SemanticInternalAttributes.TASK_SLUG]),
+    machineId: extractStringAttribute(attributes, [prefix, SemanticInternalAttributes.MACHINE_ID]),
   };
 }
 
@@ -891,10 +1124,22 @@ function hasUnpairedSurrogateAtEnd(str: string): boolean {
 export const otlpExporter = singleton("otlpExporter", initializeOTLPExporter);
 
 function initializeOTLPExporter() {
+  const metricsFlushScheduler = new DynamicFlushScheduler<MetricsV1Input>({
+    batchSize: env.METRICS_CLICKHOUSE_BATCH_SIZE,
+    flushInterval: env.METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS,
+    callback: async (_flushId, batch) => {
+      await clickhouseClient.metrics.insert(batch);
+    },
+    minConcurrency: 1,
+    maxConcurrency: env.METRICS_CLICKHOUSE_MAX_CONCURRENCY,
+    loadSheddingEnabled: false,
+  });
+
   return new OTLPExporter(
     eventRepository,
     clickhouseEventRepository,
     clickhouseEventRepositoryV2,
+    metricsFlushScheduler,
     process.env.OTLP_EXPORTER_VERBOSE === "1",
     process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT
       ? parseInt(process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, 10)

apps/webapp/app/v3/querySchemas.ts

@@ -1,4 +1,4 @@
-import { column, type TableSchema } from "@internal/tsql";
+import { column, type BucketThreshold, type TableSchema } from "@internal/tsql";
 import { z } from "zod";
 import { autoFormatSQL } from "~/components/code/TSQLEditor";
 import { runFriendlyStatus, runStatusTitleFromStatus } from "~/components/runs/v3/TaskRunStatus";
@@ -33,6 +33,7 @@ export const runsSchema: TableSchema = {
   clickhouseName: "trigger_dev.task_runs_v2",
   description: "Task runs - stores all task execution records",
   timeConstraint: "triggered_at",
+  useFinal: true,
   tenantColumns: {
     organizationId: "organization_id",
     projectId: "project_id",
@@ -434,10 +435,171 @@ export const runsSchema: TableSchema = {
   },
 };
 
+/**
+ * Schema definition for the metrics table (trigger_dev.metrics_v1)
+ */
+export const metricsSchema: TableSchema = {
+  name: "metrics",
+  clickhouseName: "trigger_dev.metrics_v1",
+  description: "Host and runtime metrics collected during task execution",
+  timeConstraint: "bucket_start",
+  tenantColumns: {
+    organizationId: "organization_id",
+    projectId: "project_id",
+    environmentId: "environment_id",
+  },
+  columns: {
+    environment: {
+      name: "environment",
+      clickhouseName: "environment_id",
+      ...column("String", { description: "The environment slug", example: "prod" }),
+      fieldMapping: "environment",
+      customRenderType: "environment",
+    },
+    project: {
+      name: "project",
+      clickhouseName: "project_id",
+      ...column("String", {
+        description: "The project reference, they always start with `proj_`.",
+        example: "proj_howcnaxbfxdmwmxazktx",
+      }),
+      fieldMapping: "project",
+      customRenderType: "project",
+    },
+    metric_name: {
+      name: "metric_name",
+      ...column("LowCardinality(String)", {
+        description: "The name of the metric (e.g. process.cpu.utilization, system.memory.usage)",
+        example: "process.cpu.utilization",
+        coreColumn: true,
+      }),
+    },
+    metric_type: {
+      name: "metric_type",
+      ...column("LowCardinality(String)", {
+        description: "The type of metric",
+        allowedValues: ["gauge", "sum", "histogram"],
+        example: "gauge",
+      }),
+    },
+    machine_id: {
+      name: "machine_id",
+      clickhouseName: "metric_subject",
+      ...column("String", {
+        description: "The machine ID that produced this metric",
+        example: "machine-abc123",
+      }),
+    },
+    bucket_start: {
+      name: "bucket_start",
+      ...column("DateTime", {
+        description: "The start of the 10-second aggregation bucket",
+        example: "2024-01-15 09:30:00",
+        coreColumn: true,
+      }),
+    },
+    metric_value: {
+      name: "metric_value",
+      clickhouseName: "value",
+      ...column("Float64", {
+        description: "The metric value",
+        example: "0.75",
+        coreColumn: true,
+      }),
+    },
+
+    // Attributes (JSON column for user-defined and system attributes)
+    attributes: {
+      name: "attributes",
+      ...column("JSON", {
+        description: "JSON attributes attached to the metric data point.",
+        example: '{"region": "us-east-1"}',
+      }),
+    },
+
+    // Trigger context columns (from attributes.trigger.* JSON subpaths)
+    run_id: {
+      name: "run_id",
+      ...column("String", {
+        description: "The run ID associated with this metric",
+        customRenderType: "runId",
+        example: "run_cm1a2b3c4d5e6f7g8h9i",
+        coreColumn: true,
+      }),
+      expression: "attributes.trigger.run_id",
+    },
+    task_identifier: {
+      name: "task_identifier",
+      ...column("String", {
+        description: "Task identifier/slug",
+        example: "my-background-task",
+        coreColumn: true,
+      }),
+      expression: "attributes.trigger.task_slug",
+    },
+    attempt_number: {
+      name: "attempt_number",
+      ...column("UInt64", {
+        description: "The attempt number for this metric",
+        example: "1",
+      }),
+      expression: "attributes.trigger.attempt_number",
+    },
+    machine_name: {
+      name: "machine_name",
+      ...column("String", {
+        description: "The machine preset used for execution",
+        allowedValues: [...MACHINE_PRESETS],
+        example: "small-1x",
+      }),
+      expression: "attributes.trigger.machine_name",
+    },
+    environment_type: {
+      name: "environment_type",
+      ...column("String", {
+        description: "Environment type",
+        allowedValues: [...ENVIRONMENT_TYPES],
+        customRenderType: "environmentType",
+        example: "PRODUCTION",
+      }),
+      expression: "attributes.trigger.environment_type",
+    },
+    worker_id: {
+      name: "worker_id",
+      ...column("String", {
+        description: "The worker ID that produced this metric",
+        customRenderType: "deploymentId",
+        example: "deployment_cm1a2b3c4d5e",
+      }),
+      expression: "attributes.trigger.worker_id",
+    },
+    worker_version: {
+      name: "worker_version",
+      ...column("String", {
+        description: "The worker version that produced this metric",
+        example: "20240115.1",
+      }),
+      expression: "attributes.trigger.worker_version",
+    },
+  },
+  timeBucketThresholds: [
+    // Metrics are pre-aggregated into 10-second buckets, so 10s is the most granular interval.
+    // All thresholds are shifted coarser compared to the runs table defaults.
+    { maxRangeSeconds: 3 * 60 * 60, interval: { value: 10, unit: "SECOND" } },
+    { maxRangeSeconds: 12 * 60 * 60, interval: { value: 1, unit: "MINUTE" } },
+    { maxRangeSeconds: 2 * 24 * 60 * 60, interval: { value: 5, unit: "MINUTE" } },
+    { maxRangeSeconds: 7 * 24 * 60 * 60, interval: { value: 15, unit: "MINUTE" } },
+    { maxRangeSeconds: 30 * 24 * 60 * 60, interval: { value: 1, unit: "HOUR" } },
+    { maxRangeSeconds: 90 * 24 * 60 * 60, interval: { value: 6, unit: "HOUR" } },
+    { maxRangeSeconds: 180 * 24 * 60 * 60, interval: { value: 1, unit: "DAY" } },
+    { maxRangeSeconds: 365 * 24 * 60 * 60, interval: { value: 1, unit: "WEEK" } },
+  ] satisfies BucketThreshold[],
+};
+
 /**
  * All available schemas for the query editor
  */
-export const querySchemas: TableSchema[] = [runsSchema];
+export const querySchemas: TableSchema[] = [runsSchema, metricsSchema];
 
 /**
  * Default query for the query editor

apps/webapp/app/v3/services/aiQueryService.server.ts

@@ -55,7 +55,7 @@ export class AIQueryService {
 
   constructor(
     private readonly tableSchema: TableSchema[],
-    private readonly model: LanguageModelV1 = openai("gpt-4o-mini")
+    private readonly model: LanguageModelV1 = openai("gpt-4.1-mini")
   ) {}
 
   /**
@@ -65,7 +65,7 @@ export class AIQueryService {
   private buildSetTimeFilterTool() {
     return tool({
       description:
-        "Set the time filter for the query page UI instead of adding triggered_at conditions to the query. ALWAYS use this tool when the user wants to filter by time (e.g., 'last 7 days', 'past hour', 'yesterday'). The UI will apply this filter automatically. Do NOT add triggered_at to the WHERE clause - use this tool instead.",
+        "Set the time filter for the query page UI instead of adding time conditions to the query. ALWAYS use this tool when the user wants to filter by time (e.g., 'last 7 days', 'past hour', 'yesterday'). The UI will apply this filter automatically using the table's time column (triggered_at for runs, bucket_start for metrics). Do NOT add triggered_at or bucket_start to the WHERE clause for time filtering - use this tool instead.",
       parameters: z.object({
         period: z
           .string()
@@ -366,7 +366,7 @@ export class AIQueryService {
    * Build the system prompt for the AI
    */
   private buildSystemPrompt(schemaDescription: string): string {
-    return `You are an expert SQL assistant that generates TSQL queries for a task run analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
+    return `You are an expert SQL assistant that generates TSQL queries for a task analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
 
 ## Your Task
 Convert natural language requests into valid TSQL SELECT queries. Always validate your queries using the validateTSQLQuery tool before returning them.
@@ -374,6 +374,13 @@ Convert natural language requests into valid TSQL SELECT queries. Always validat
 ## Available Schema
 ${schemaDescription}
 
+## Choosing the Right Table
+
+- **runs** — Task run records (status, timing, cost, output, etc.). Use for questions about runs, tasks, failures, durations, costs, queues.
+- **metrics** — Host and runtime metrics collected during task execution (CPU, memory). Use for questions about resource usage, CPU utilization, memory consumption, or performance monitoring. Each row is a 10-second aggregation bucket tied to a specific run.
+
+When the user mentions "CPU", "memory", "utilization", "resource usage", or similar terms, query the \`metrics\` table. When they mention "runs", "tasks", "failures", "status", "duration", or "cost", query the \`runs\` table.
+
 ## TSQL Syntax Guide
 
 TSQL supports standard SQL syntax with some ClickHouse-specific features:
@@ -437,16 +444,51 @@ LIMIT 1000
 Only use explicit \`toStartOfHour\`/\`toStartOfDay\` etc. if the user specifically requests a particular bucket size (e.g., "group by hour", "bucket by day").
 
 ### Common Patterns
+
+#### Runs table
 - Status filter: WHERE status = 'Failed' or WHERE status IN ('Failed', 'Crashed')
-- Time filtering: Use the \`setTimeFilter\` tool (NOT triggered_at in WHERE clause)
+- Time filtering: Use the \`setTimeFilter\` tool (NOT triggered_at/bucket_start in WHERE clause)
+
+#### Metrics table
+- Filter by metric name: WHERE metric_name = 'process.cpu.utilization'
+- Filter by run: WHERE run_id = 'run_abc123'
+- Filter by task: WHERE task_identifier = 'my-task'
+- Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p95, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
+- Use \`metric_value\` — the metric's observed value
+- Use prettyFormat(expr, 'bytes') to tell the UI to format values as bytes (e.g., "1.50 GiB") — keeps values numeric for charts
+- Use prettyFormat(expr, 'percent') for percentage values
+- prettyFormat does NOT change the SQL — it only adds a display hint
+- Available format types: bytes, decimalBytes, percent, quantity, duration, durationSeconds, costInDollars
+- For memory metrics (including nodejs.heap.*), always use prettyFormat with 'bytes'
+- For CPU utilization, consider prettyFormat with 'percent'
+
+\`\`\`sql
+-- CPU utilization over time for a task
+SELECT timeBucket(), task_identifier, prettyFormat(avg(metric_value), 'percent') AS avg_cpu
+FROM metrics
+WHERE metric_name = 'process.cpu.utilization'
+GROUP BY timeBucket, task_identifier
+ORDER BY timeBucket
+LIMIT 1000
+\`\`\`
+
+\`\`\`sql
+-- Peak memory usage per run
+SELECT run_id, task_identifier, prettyFormat(max(metric_value), 'bytes') AS peak_memory
+FROM metrics
+WHERE metric_name = 'process.memory.usage'
+GROUP BY run_id, task_identifier
+ORDER BY peak_memory DESC
+LIMIT 100
+\`\`\`
 
 ## Important Rules
 
 1. NEVER use SELECT * - ClickHouse is a columnar database where SELECT * has very poor performance
 2. Always select only the specific columns needed for the request
 3. When column selection is ambiguous, use the core columns marked [CORE] in the schema
-4. **TIME FILTERING**: When the user wants to filter by time (e.g., "last 7 days", "past hour", "yesterday"), ALWAYS use the \`setTimeFilter\` tool instead of adding \`triggered_at\` conditions to the query. The UI has a time filter that will apply this automatically.
-5. Do NOT add \`triggered_at\` to WHERE clauses - use \`setTimeFilter\` tool instead. If the user doesn't specify a time period, do NOT add any time filter (the UI defaults to 7 days).
+4. **TIME FILTERING**: When the user wants to filter by time (e.g., "last 7 days", "past hour", "yesterday"), ALWAYS use the \`setTimeFilter\` tool instead of adding time conditions to the WHERE clause. The UI has a time filter that will apply this automatically. This applies to both the \`runs\` table (triggered_at) and the \`metrics\` table (bucket_start).
+5. Do NOT add \`triggered_at\` or \`bucket_start\` to WHERE clauses for time filtering - use \`setTimeFilter\` tool instead. If the user doesn't specify a time period, do NOT add any time filter (the UI defaults to 7 days).
 6. **TIME BUCKETING**: When the user wants to see data over time or in time buckets, use \`timeBucket()\` in SELECT and reference it as \`timeBucket\` in GROUP BY / ORDER BY. Only use manual bucketing functions (toStartOfHour, toStartOfDay, etc.) when the user explicitly requests a specific bucket size.
 7. ALWAYS use the validateTSQLQuery tool to check your query before returning it
 8. If validation fails, fix the issues and try again (up to 3 attempts)
@@ -472,7 +514,7 @@ If you cannot generate a valid query, explain why briefly.`;
    * Build the system prompt for edit mode
    */
   private buildEditSystemPrompt(schemaDescription: string): string {
-    return `You are an expert SQL assistant that modifies existing TSQL queries for a task run analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
+    return `You are an expert SQL assistant that modifies existing TSQL queries for a task analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
 
 ## Your Task
 Modify the provided TSQL query according to the user's instructions. Make only the changes requested - preserve the existing query structure where possible.
@@ -480,6 +522,11 @@ Modify the provided TSQL query according to the user's instructions. Make only t
 ## Available Schema
 ${schemaDescription}
 
+## Choosing the Right Table
+
+- **runs** — Task run records (status, timing, cost, output, etc.). Use for questions about runs, tasks, failures, durations, costs, queues.
+- **metrics** — Host and runtime metrics collected during task execution (CPU, memory). Use for questions about resource usage, CPU utilization, memory consumption, or performance monitoring. Each row is a 10-second aggregation bucket tied to a specific run.
+
 ## TSQL Syntax Guide
 
 TSQL supports standard SQL syntax with some ClickHouse-specific features:
@@ -539,11 +586,18 @@ ORDER BY timeBucket
 LIMIT 1000
 \`\`\`
 
+### Common Metrics Patterns
+- Filter by metric: WHERE metric_name = 'process.cpu.utilization'
+- Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p50, nodejs.event_loop.delay.p99, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
+- Use \`metric_value\` — the metric's observed value
+- Use prettyFormat(expr, 'bytes') for memory metrics (including nodejs.heap.*), prettyFormat(expr, 'percent') for CPU utilization
+- prettyFormat does NOT change the SQL — it only adds a display hint for the UI
+
 ## Important Rules
 
 1. NEVER use SELECT * - ClickHouse is a columnar database where SELECT * has very poor performance
 2. If the existing query uses SELECT *, replace it with specific columns (use core columns marked [CORE] as defaults)
-3. **TIME FILTERING**: When the user wants to change time filtering (e.g., "change to last 30 days"), use the \`setTimeFilter\` tool instead of modifying \`triggered_at\` conditions. If the existing query has \`triggered_at\` in WHERE, consider removing it and using \`setTimeFilter\` instead.
+3. **TIME FILTERING**: When the user wants to change time filtering (e.g., "change to last 30 days"), use the \`setTimeFilter\` tool instead of modifying time column conditions. If the existing query has \`triggered_at\` or \`bucket_start\` in WHERE for time filtering, consider removing it and using \`setTimeFilter\` instead.
 4. **TIME BUCKETING**: When adding time-series grouping, use \`timeBucket()\` in SELECT and reference it as \`timeBucket\` in GROUP BY / ORDER BY. Only use manual bucketing functions (toStartOfHour, toStartOfDay, etc.) when the user explicitly requests a specific bucket size.
 5. ALWAYS use the validateTSQLQuery tool to check your modified query before returning it
 6. If validation fails, fix the issues and try again (up to 3 attempts)

apps/webapp/package.json

@@ -70,7 +70,7 @@
     "@opentelemetry/exporter-logs-otlp-http": "0.203.0",
     "@opentelemetry/exporter-metrics-otlp-proto": "0.203.0",
     "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
-    "@opentelemetry/host-metrics": "^0.36.0",
+    "@opentelemetry/host-metrics": "^0.37.0",
     "@opentelemetry/instrumentation": "0.203.0",
     "@opentelemetry/instrumentation-aws-sdk": "^0.57.0",
     "@opentelemetry/instrumentation-express": "^0.52.0",

internal-packages/clickhouse/schema/017_create_metrics_v1.sql

@@ -0,0 +1,44 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS trigger_dev.metrics_v1
+(
+  organization_id     LowCardinality(String),
+  project_id          LowCardinality(String),
+  environment_id      String CODEC(ZSTD(1)),
+  metric_name         LowCardinality(String),
+  metric_type         LowCardinality(String),
+  metric_subject      String CODEC(ZSTD(1)),
+  bucket_start        DateTime CODEC(Delta(4), ZSTD(1)),
+  value               Float64 DEFAULT 0 CODEC(ZSTD(1)),
+  attributes          JSON(
+    `trigger.run_id` String,
+    `trigger.task_slug` String,
+    `trigger.attempt_number` Int64,
+    `trigger.environment_type` LowCardinality(String),
+    `trigger.machine_id` String,
+    `trigger.machine_name` LowCardinality(String),
+    `trigger.worker_id` String,
+    `trigger.worker_version` String,
+    `system.cpu.logical_number` String,
+    `system.cpu.state` LowCardinality(String),
+    `system.memory.state` LowCardinality(String),
+    `system.device` String,
+    `system.filesystem.type` LowCardinality(String),
+    `system.filesystem.mountpoint` String,
+    `system.filesystem.mode` LowCardinality(String),
+    `system.filesystem.state` LowCardinality(String),
+    `disk.io.direction` LowCardinality(String),
+    `process.cpu.state` LowCardinality(String),
+    `network.io.direction` LowCardinality(String),
+    max_dynamic_paths=8
+  ),
+  INDEX idx_run_id attributes.trigger.run_id TYPE bloom_filter(0.001) GRANULARITY 1,
+  INDEX idx_task_slug attributes.trigger.task_slug TYPE bloom_filter(0.001) GRANULARITY 1
+)
+ENGINE = MergeTree()
+PARTITION BY toDate(bucket_start)
+ORDER BY (organization_id, project_id, environment_id, metric_name, metric_subject, bucket_start)
+TTL bucket_start + INTERVAL 60 DAY
+SETTINGS ttl_only_drop_parts = 1;
+
+-- +goose Down
+DROP TABLE IF EXISTS trigger_dev.metrics_v1;

internal-packages/clickhouse/schema/018_add_machine_id_to_task_events_v2.sql

@@ -0,0 +1,7 @@
+-- +goose Up
+ALTER TABLE trigger_dev.task_events_v2
+ADD COLUMN machine_id String DEFAULT '' CODEC(ZSTD(1));
+
+-- +goose Down
+ALTER TABLE trigger_dev.task_events_v2
+DROP COLUMN machine_id;

internal-packages/clickhouse/src/index.ts

@@ -26,12 +26,14 @@ import {
   getLogDetailQueryBuilderV2,
   getLogsSearchListQueryBuilder,
 } from "./taskEvents.js";
+import { insertMetrics } from "./metrics.js";
 import { Logger, type LogLevel } from "@trigger.dev/core/logger";
 import type { Agent as HttpAgent } from "http";
 import type { Agent as HttpsAgent } from "https";
 
 export type * from "./taskRuns.js";
 export type * from "./taskEvents.js";
+export type * from "./metrics.js";
 export type * from "./client/queryBuilder.js";
 
 // Re-export column constants, indices, and type-safe accessors
@@ -56,7 +58,7 @@ export {
   type FieldMappings,
   type WhereClauseCondition,
 } from "./client/tsql.js";
-export type { OutputColumnMetadata } from "@internal/tsql";
+export type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql";
 
 // Errors
 export { QueryError } from "./client/errors.js";
@@ -206,6 +208,12 @@ export class ClickHouse {
     };
   }
 
+  get metrics() {
+    return {
+      insert: insertMetrics(this.writer),
+    };
+  }
+
   get taskEventsV2() {
     return {
       insert: insertTaskEventsV2(this.writer),

internal-packages/clickhouse/src/metrics.ts

@@ -0,0 +1,29 @@
+import { z } from "zod";
+import { ClickhouseWriter } from "./client/types.js";
+
+export const MetricsV1Input = z.object({
+  organization_id: z.string(),
+  project_id: z.string(),
+  environment_id: z.string(),
+  metric_name: z.string(),
+  metric_type: z.string(),
+  metric_subject: z.string(),
+  bucket_start: z.string(),
+  value: z.number(),
+  attributes: z.unknown(),
+});
+
+export type MetricsV1Input = z.input<typeof MetricsV1Input>;
+
+export function insertMetrics(ch: ClickhouseWriter) {
+  return ch.insertUnsafe<MetricsV1Input>({
+    name: "insertMetrics",
+    table: "trigger_dev.metrics_v1",
+    settings: {
+      enable_json_type: 1,
+      type_json_skip_duplicated_paths: 1,
+      input_format_json_throw_on_bad_escape_sequence: 0,
+      input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
+    },
+  });
+}

internal-packages/clickhouse/src/taskEvents.ts

@@ -19,6 +19,7 @@ export const TaskEventV1Input = z.object({
   attributes: z.unknown(),
   metadata: z.string(),
   expires_at: z.string(),
+  machine_id: z.string().optional(),
 });
 
 export type TaskEventV1Input = z.input<typeof TaskEventV1Input>;
@@ -153,6 +154,7 @@ export const TaskEventV2Input = z.object({
   attributes: z.unknown(),
   metadata: z.string(),
   expires_at: z.string(),
+  machine_id: z.string().optional(),
   // inserted_at has a default value in the table, so it's optional for inserts
   inserted_at: z.string().optional(),
 });

internal-packages/otlp-importer/src/index.ts

@@ -10,6 +10,12 @@ import {
   ExportLogsServiceResponse,
 } from "./generated/opentelemetry/proto/collector/logs/v1/logs_service";
 
+import {
+  ExportMetricsPartialSuccess,
+  ExportMetricsServiceRequest,
+  ExportMetricsServiceResponse,
+} from "./generated/opentelemetry/proto/collector/metrics/v1/metrics_service";
+
 import type {
   AnyValue,
   KeyValue,
@@ -33,6 +39,21 @@ import {
   Status,
   Status_StatusCode,
 } from "./generated/opentelemetry/proto/trace/v1/trace";
+import {
+  ResourceMetrics,
+  ScopeMetrics,
+  Metric,
+  Gauge,
+  Sum,
+  Histogram,
+  ExponentialHistogram,
+  Summary,
+  NumberDataPoint,
+  HistogramDataPoint,
+  ExponentialHistogramDataPoint,
+  SummaryDataPoint,
+  AggregationTemporality,
+} from "./generated/opentelemetry/proto/metrics/v1/metrics";
 
 export {
   LogRecord,
@@ -57,3 +78,21 @@ export {
 export { ExportTracePartialSuccess, ExportTraceServiceRequest, ExportTraceServiceResponse };
 
 export { ExportLogsPartialSuccess, ExportLogsServiceRequest, ExportLogsServiceResponse };
+
+export { ExportMetricsPartialSuccess, ExportMetricsServiceRequest, ExportMetricsServiceResponse };
+
+export {
+  ResourceMetrics,
+  ScopeMetrics,
+  Metric,
+  Gauge,
+  Sum,
+  Histogram,
+  ExponentialHistogram,
+  Summary,
+  NumberDataPoint,
+  HistogramDataPoint,
+  ExponentialHistogramDataPoint,
+  SummaryDataPoint,
+  AggregationTemporality,
+};

internal-packages/tsql/src/index.ts

@@ -109,6 +109,7 @@ export {
   type ClickHouseType,
   type ColumnSchema,
   type FieldMappings,
+  type ColumnFormatType,
   type OutputColumnMetadata,
   type RequiredFilter,
   type SchemaRegistry,
@@ -133,7 +134,9 @@ export {
 
 // Re-export time bucket utilities
 export {
+  BUCKET_THRESHOLDS,
   calculateTimeBucketInterval,
+  type BucketThreshold,
   type TimeBucketInterval,
 } from "./query/time_buckets.js";
 

internal-packages/tsql/src/query/printer.test.ts

@@ -2,7 +2,13 @@ import { describe, it, expect, beforeEach } from "vitest";
 import { parseTSQLSelect, parseTSQLExpr, compileTSQL } from "../index.js";
 import { ClickHousePrinter, printToClickHouse, type PrintResult } from "./printer.js";
 import { createPrinterContext, PrinterContext } from "./printer_context.js";
-import { createSchemaRegistry, column, type TableSchema, type SchemaRegistry } from "./schema.js";
+import {
+  createSchemaRegistry,
+  column,
+  type TableSchema,
+  type SchemaRegistry,
+} from "./schema.js";
+import type { BucketThreshold } from "./time_buckets.js";
 import { QueryError, SyntaxError } from "./errors.js";
 
 /**
@@ -2335,16 +2341,19 @@ describe("Basic column metadata", () => {
         name: "status",
         type: "LowCardinality(String)",
         customRenderType: "runStatus",
+        format: "runStatus",
       });
       expect(columns[1]).toEqual({
         name: "usage_duration_ms",
         type: "UInt32",
         customRenderType: "duration",
+        format: "duration",
       });
       expect(columns[2]).toEqual({
         name: "cost_in_cents",
         type: "Float64",
         customRenderType: "cost",
+        format: "cost",
       });
     });
 
@@ -2687,6 +2696,133 @@ describe("Basic column metadata", () => {
       expect(columns[2].name).toBe("avg");
     });
   });
+
+  describe("prettyFormat()", () => {
+    it("should strip prettyFormat from SQL and attach format to column metadata", () => {
+      const ctx = createMetadataTestContext();
+      const { sql, columns } = printQuery(
+        "SELECT prettyFormat(usage_duration_ms, 'bytes') AS memory FROM runs",
+        ctx
+      );
+
+      // SQL should not contain prettyFormat
+      expect(sql).not.toContain("prettyFormat");
+      expect(sql).toContain("usage_duration_ms");
+
+      expect(columns).toHaveLength(1);
+      expect(columns[0].name).toBe("memory");
+      expect(columns[0].format).toBe("bytes");
+    });
+
+    it("should work with aggregation wrapping", () => {
+      const ctx = createMetadataTestContext();
+      const { sql, columns } = printQuery(
+        "SELECT prettyFormat(avg(usage_duration_ms), 'bytes') AS avg_memory FROM runs",
+        ctx
+      );
+
+      expect(sql).not.toContain("prettyFormat");
+      expect(sql).toContain("avg(usage_duration_ms)");
+
+      expect(columns).toHaveLength(1);
+      expect(columns[0].name).toBe("avg_memory");
+      expect(columns[0].format).toBe("bytes");
+      expect(columns[0].type).toBe("Float64");
+    });
+
+    it("should work without explicit alias", () => {
+      const ctx = createMetadataTestContext();
+      const { sql, columns } = printQuery(
+        "SELECT prettyFormat(usage_duration_ms, 'percent') FROM runs",
+        ctx
+      );
+
+      expect(sql).not.toContain("prettyFormat");
+      expect(columns).toHaveLength(1);
+      expect(columns[0].name).toBe("usage_duration_ms");
+      expect(columns[0].format).toBe("percent");
+    });
+
+    it("should throw for invalid format type", () => {
+      const ctx = createMetadataTestContext();
+      expect(() => {
+        printQuery(
+          "SELECT prettyFormat(usage_duration_ms, 'invalid') FROM runs",
+          ctx
+        );
+      }).toThrow(QueryError);
+      expect(() => {
+        printQuery(
+          "SELECT prettyFormat(usage_duration_ms, 'invalid') FROM runs",
+          ctx
+        );
+      }).toThrow(/Unknown format type/);
+    });
+
+    it("should throw for wrong argument count", () => {
+      const ctx = createMetadataTestContext();
+      expect(() => {
+        printQuery("SELECT prettyFormat(usage_duration_ms) FROM runs", ctx);
+      }).toThrow(QueryError);
+      expect(() => {
+        printQuery("SELECT prettyFormat(usage_duration_ms) FROM runs", ctx);
+      }).toThrow(/requires exactly 2 arguments/);
+    });
+
+    it("should throw when second argument is not a string literal", () => {
+      const ctx = createMetadataTestContext();
+      expect(() => {
+        printQuery(
+          "SELECT prettyFormat(usage_duration_ms, 123) FROM runs",
+          ctx
+        );
+      }).toThrow(QueryError);
+      expect(() => {
+        printQuery(
+          "SELECT prettyFormat(usage_duration_ms, 123) FROM runs",
+          ctx
+        );
+      }).toThrow(/must be a string literal/);
+    });
+
+    it("should override schema-level customRenderType", () => {
+      const ctx = createMetadataTestContext();
+      const { columns } = printQuery(
+        "SELECT prettyFormat(usage_duration_ms, 'bytes') AS mem FROM runs",
+        ctx
+      );
+
+      expect(columns).toHaveLength(1);
+      // prettyFormat's format should take precedence
+      expect(columns[0].format).toBe("bytes");
+      // customRenderType from schema should NOT be set since prettyFormat overrides
+      // The source column had customRenderType: "duration" but prettyFormat replaces it
+    });
+
+    it("should auto-populate format from customRenderType when not explicitly set", () => {
+      const ctx = createMetadataTestContext();
+      const { columns } = printQuery(
+        "SELECT usage_duration_ms, cost_in_cents FROM runs",
+        ctx
+      );
+
+      expect(columns).toHaveLength(2);
+      // customRenderType should auto-populate format
+      expect(columns[0].customRenderType).toBe("duration");
+      expect(columns[0].format).toBe("duration");
+      expect(columns[1].customRenderType).toBe("cost");
+      expect(columns[1].format).toBe("cost");
+    });
+
+    it("should not set format when column has no customRenderType", () => {
+      const ctx = createMetadataTestContext();
+      const { columns } = printQuery("SELECT run_id FROM runs", ctx);
+
+      expect(columns).toHaveLength(1);
+      expect(columns[0].format).toBeUndefined();
+      expect(columns[0].customRenderType).toBeUndefined();
+    });
+  });
 });
 
 describe("Unknown column blocking", () => {
@@ -3570,4 +3706,73 @@ describe("timeBucket()", () => {
       expect(Object.values(params)).toContain("org_test123");
     });
   });
+
+  describe("per-table timeBucketThresholds", () => {
+    const customThresholds: BucketThreshold[] = [
+      // 10-second minimum granularity (e.g., for pre-aggregated metrics)
+      { maxRangeSeconds: 10 * 60, interval: { value: 10, unit: "SECOND" } },
+      { maxRangeSeconds: 30 * 60, interval: { value: 30, unit: "SECOND" } },
+      { maxRangeSeconds: 2 * 60 * 60, interval: { value: 1, unit: "MINUTE" } },
+    ];
+
+    const schemaWithCustomThresholds: TableSchema = {
+      ...timeBucketSchema,
+      name: "metrics",
+      timeBucketThresholds: customThresholds,
+    };
+
+    it("should use custom thresholds when defined on the table schema", () => {
+      // 3-minute range: global default would give 5 SECOND, custom gives 10 SECOND
+      const threeMinuteRange = {
+        from: new Date("2024-01-01T00:00:00Z"),
+        to: new Date("2024-01-01T00:03:00Z"),
+      };
+
+      const schema = createSchemaRegistry([schemaWithCustomThresholds]);
+      const ctx = createPrinterContext({
+        schema,
+        enforcedWhereClause: {
+          organization_id: { op: "eq", value: "org_test123" },
+          project_id: { op: "eq", value: "proj_test456" },
+          environment_id: { op: "eq", value: "env_test789" },
+        },
+        timeRange: threeMinuteRange,
+      });
+
+      const ast = parseTSQLSelect(
+        "SELECT timeBucket(), count() FROM metrics GROUP BY timeBucket"
+      );
+      const { sql } = printToClickHouse(ast, ctx);
+
+      // Custom thresholds: under 10 min → 10 SECOND (not the global 5 SECOND)
+      expect(sql).toContain("toStartOfInterval(created_at, INTERVAL 10 SECOND)");
+    });
+
+    it("should fall back to global defaults when no custom thresholds are defined", () => {
+      // 3-minute range with standard schema (no custom thresholds)
+      const threeMinuteRange = {
+        from: new Date("2024-01-01T00:00:00Z"),
+        to: new Date("2024-01-01T00:03:00Z"),
+      };
+
+      const schema = createSchemaRegistry([timeBucketSchema]);
+      const ctx = createPrinterContext({
+        schema,
+        enforcedWhereClause: {
+          organization_id: { op: "eq", value: "org_test123" },
+          project_id: { op: "eq", value: "proj_test456" },
+          environment_id: { op: "eq", value: "env_test789" },
+        },
+        timeRange: threeMinuteRange,
+      });
+
+      const ast = parseTSQLSelect(
+        "SELECT timeBucket(), count() FROM runs GROUP BY timeBucket"
+      );
+      const { sql } = printToClickHouse(ast, ctx);
+
+      // Global default: under 5 min → 5 SECOND
+      expect(sql).toContain("toStartOfInterval(created_at, INTERVAL 5 SECOND)");
+    });
+  });
 });

internal-packages/tsql/src/query/printer.ts

@@ -59,6 +59,7 @@ import {
   ClickHouseType,
   hasFieldMapping,
   getInternalValueFromMappingCaseInsensitive,
+  type ColumnFormatType,
 } from "./schema";
 
 /**
@@ -739,6 +740,14 @@ export class ClickHousePrinter {
         metadata.description = sourceColumn.description;
       }
 
+      // Set format hint from prettyFormat() or auto-populate from customRenderType
+      const sourceWithFormat = sourceColumn as (Partial<ColumnSchema> & { format?: ColumnFormatType }) | null;
+      if (sourceWithFormat?.format) {
+        metadata.format = sourceWithFormat.format;
+      } else if (sourceColumn?.customRenderType) {
+        metadata.format = sourceColumn.customRenderType as ColumnFormatType;
+      }
+
       this.outputColumns.push(metadata);
     }
 
@@ -932,6 +941,53 @@ export class ClickHousePrinter {
       };
     }
 
+    // Handle prettyFormat(expr, 'formatType') — metadata-only wrapper
+    if ((col as Call).expression_type === "call") {
+      const call = col as Call;
+      if (call.name.toLowerCase() === "prettyformat") {
+        if (call.args.length !== 2) {
+          throw new QueryError(
+            "prettyFormat() requires exactly 2 arguments: prettyFormat(expression, 'formatType')"
+          );
+        }
+        const formatArg = call.args[1];
+        if (
+          (formatArg as Constant).expression_type !== "constant" ||
+          typeof (formatArg as Constant).value !== "string"
+        ) {
+          throw new QueryError(
+            "prettyFormat() second argument must be a string literal format type"
+          );
+        }
+        const formatType = (formatArg as Constant).value as string;
+        const validFormats = [
+          "bytes",
+          "decimalBytes",
+          "quantity",
+          "percent",
+          "duration",
+          "durationSeconds",
+          "costInDollars",
+          "cost",
+        ];
+        if (!validFormats.includes(formatType)) {
+          throw new QueryError(
+            `Unknown format type '${formatType}'. Valid types: ${validFormats.join(", ")}`
+          );
+        }
+        const innerAnalysis = this.analyzeSelectColumn(call.args[0]);
+        return {
+          outputName: innerAnalysis.outputName,
+          sourceColumn: {
+            ...(innerAnalysis.sourceColumn ?? {}),
+            type: innerAnalysis.sourceColumn?.type ?? innerAnalysis.inferredType ?? undefined,
+            format: formatType as ColumnFormatType,
+          } as Partial<ColumnSchema> & { format?: ColumnFormatType },
+          inferredType: innerAnalysis.inferredType,
+        };
+      }
+    }
+
     // Handle Call (function/aggregation) - infer type from function
     if ((col as Call).expression_type === "call") {
       const call = col as Call;
@@ -1559,13 +1615,20 @@ export class ClickHousePrinter {
       joinStrings.push(`AS ${this.printIdentifier(node.alias)}`);
     }
 
-    // Always add FINAL for direct table references to ensure deduplicated results
-    // from ReplacingMergeTree tables in ClickHouse
+    // Add FINAL for direct table references to ReplacingMergeTree tables
+    // to ensure deduplicated results. Only applied when the table schema
+    // opts in via `useFinal: true` (not needed for plain MergeTree tables).
     if (node.table) {
       const tableExpr = node.table;
-      const isDirectTable = (tableExpr as Field).expression_type === "field";
-      if (isDirectTable) {
-        joinStrings.push("FINAL");
+      if ((tableExpr as Field).expression_type === "field") {
+        const field = tableExpr as Field;
+        const tableName = field.chain[0];
+        if (typeof tableName === "string") {
+          const tableSchema = this.lookupTable(tableName);
+          if (tableSchema.useFinal) {
+            joinStrings.push("FINAL");
+          }
+        }
       }
     }
 
@@ -2802,6 +2865,14 @@ export class ClickHousePrinter {
   private visitCall(node: Call): string {
     const name = node.name;
 
+    // Handle prettyFormat() — strip wrapper, only emit the inner expression
+    if (name.toLowerCase() === "prettyformat") {
+      if (node.args.length !== 2) {
+        throw new QueryError("prettyFormat() requires exactly 2 arguments");
+      }
+      return this.visit(node.args[0]);
+    }
+
     // Handle timeBucket() - special TSQL function for automatic time bucketing
     if (name.toLowerCase() === "timebucket") {
       return this.visitTimeBucket(node);
@@ -2978,8 +3049,12 @@ export class ClickHousePrinter {
       );
     }
 
-    // Calculate the appropriate interval
-    const interval = calculateTimeBucketInterval(timeRange.from, timeRange.to);
+    // Calculate the appropriate interval (use table-specific thresholds if defined)
+    const interval = calculateTimeBucketInterval(
+      timeRange.from,
+      timeRange.to,
+      tableSchema.timeBucketThresholds
+    );
 
     // Emit toStartOfInterval(column, INTERVAL N UNIT)
     return `toStartOfInterval(${escapeClickHouseIdentifier(clickhouseColumnName)}, INTERVAL ${interval.value} ${interval.unit})`;

internal-packages/tsql/src/query/schema.ts

@@ -2,6 +2,7 @@
 // Defines allowed tables, columns, and tenant isolation configuration
 
 import { QueryError } from "./errors";
+import type { BucketThreshold } from "./time_buckets";
 
 /**
  * ClickHouse data types supported by TSQL
@@ -269,6 +270,33 @@ export interface ColumnSchema {
  */
 export type FieldMappings = Record<string, Record<string, string>>;
 
+/**
+ * Display format types for column values.
+ *
+ * These tell the UI how to render values without changing the underlying data type.
+ * Includes both existing custom render types and new format hint types.
+ */
+export type ColumnFormatType =
+  // Existing custom render types
+  | "runId"
+  | "runStatus"
+  | "duration"
+  | "durationSeconds"
+  | "costInDollars"
+  | "cost"
+  | "machine"
+  | "environment"
+  | "environmentType"
+  | "project"
+  | "queue"
+  | "tags"
+  | "number"
+  // Format hint types (used by prettyFormat())
+  | "bytes"
+  | "decimalBytes"
+  | "quantity"
+  | "percent";
+
 /**
  * Metadata for a column in query results.
  *
@@ -290,6 +318,16 @@ export interface OutputColumnMetadata {
    * Only present for columns or virtual columns defined in the table schema.
    */
   description?: string;
+  /**
+   * Display format hint — tells the UI how to render numeric values.
+   *
+   * Set by `prettyFormat(expr, 'formatType')` in TSQL queries.
+   * The underlying value remains numeric (for charts), but the UI uses this
+   * hint for axis labels, table cells, and tooltips.
+   *
+   * Also auto-populated from `customRenderType` when not explicitly set.
+   */
+  format?: ColumnFormatType;
 }
 
 /**
@@ -354,6 +392,19 @@ export interface TableSchema {
    * ```
    */
   timeConstraint?: string;
+  /**
+   * Custom time bucket thresholds for this table.
+   * When set, timeBucket() uses these instead of the global defaults.
+   * Useful when the table's time granularity differs from the standard (e.g., metrics
+   * pre-aggregated into 10-second buckets shouldn't go below 10-second intervals).
+   */
+  timeBucketThresholds?: BucketThreshold[];
+  /**
+   * Whether to add the FINAL keyword when querying this table.
+   * This should be set to `true` for ReplacingMergeTree tables where deduplication
+   * is needed to get correct results. Not needed for plain MergeTree tables.
+   */
+  useFinal?: boolean;
 }
 
 /**

internal-packages/tsql/src/query/time_buckets.ts

@@ -17,13 +17,23 @@ export interface TimeBucketInterval {
 }
 
 /**
- * Time bucket thresholds: each entry defines a maximum time range duration (in seconds)
+ * A threshold mapping a maximum time range duration to a bucket interval.
+ */
+export interface BucketThreshold {
+  /** Maximum range duration in seconds for this threshold to apply */
+  maxRangeSeconds: number;
+  /** The bucket interval to use when the range is under maxRangeSeconds */
+  interval: TimeBucketInterval;
+}
+
+/**
+ * Default time bucket thresholds: each entry defines a maximum time range duration (in seconds)
  * and the corresponding bucket interval to use.
  *
  * The intervals are chosen to produce roughly 50-100 data points for the given range.
  * Entries are ordered from smallest to largest range.
  */
-const BUCKET_THRESHOLDS: Array<{ maxRangeSeconds: number; interval: TimeBucketInterval }> = [
+export const BUCKET_THRESHOLDS: BucketThreshold[] = [
   // Under 5 minutes → 5 second buckets (max 60 buckets)
   { maxRangeSeconds: 5 * 60, interval: { value: 5, unit: "SECOND" } },
   // Under 30 minutes → 30 second buckets (max 60 buckets)
@@ -73,10 +83,14 @@ const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" };
  * ); // { value: 6, unit: "HOUR" }
  * ```
  */
-export function calculateTimeBucketInterval(from: Date, to: Date): TimeBucketInterval {
+export function calculateTimeBucketInterval(
+  from: Date,
+  to: Date,
+  thresholds?: BucketThreshold[]
+): TimeBucketInterval {
   const rangeSeconds = Math.abs(to.getTime() - from.getTime()) / 1000;
 
-  for (const threshold of BUCKET_THRESHOLDS) {
+  for (const threshold of thresholds ?? BUCKET_THRESHOLDS) {
     if (rangeSeconds < threshold.maxRangeSeconds) {
       return threshold.interval;
     }

packages/cli-v3/src/dev/taskRunProcessPool.ts

@@ -2,6 +2,7 @@ import {
   MachinePresetResources,
   ServerBackgroundWorker,
   WorkerManifest,
+  generateFriendlyId,
 } from "@trigger.dev/core/v3";
 import { TaskRunProcess } from "../executions/taskRunProcess.js";
 import { logger } from "../utilities/logger.js";
@@ -23,6 +24,8 @@ export class TaskRunProcessPool {
   private readonly maxExecutionsPerProcess: number;
   private readonly executionCountsPerProcess: Map<number, number> = new Map();
   private readonly deprecatedVersions: Set<string> = new Set();
+  private readonly idleTimers: Map<TaskRunProcess, NodeJS.Timeout> = new Map();
+  private static readonly IDLE_TIMEOUT_MS = 30_000;
 
   constructor(options: TaskRunProcessPoolOptions) {
     this.options = options;
@@ -38,6 +41,7 @@ export class TaskRunProcessPool {
     const versionProcesses = this.availableProcessesByVersion.get(version) || [];
 
     const processesToKill = versionProcesses.filter((process) => !process.isExecuting());
+    processesToKill.forEach((process) => this.clearIdleTimer(process));
     Promise.all(processesToKill.map((process) => this.killProcess(process))).then(() => {
       this.availableProcessesByVersion.delete(version);
     });
@@ -71,6 +75,7 @@ export class TaskRunProcessPool {
           version,
           availableProcesses.filter((p) => p !== reusableProcess)
         );
+        this.clearIdleTimer(reusableProcess);
 
         if (!this.busyProcessesByVersion.has(version)) {
           this.busyProcessesByVersion.set(version, new Set());
@@ -106,6 +111,7 @@ export class TaskRunProcessPool {
       env: {
         ...this.options.env,
         ...env,
+        TRIGGER_MACHINE_ID: generateFriendlyId("machine"),
       },
       serverWorker,
       machineResources,
@@ -154,6 +160,7 @@ export class TaskRunProcessPool {
           this.availableProcessesByVersion.set(version, []);
         }
         this.availableProcessesByVersion.get(version)!.push(process);
+        this.startIdleTimer(process, version);
       } catch (error) {
         logger.debug("[TaskRunProcessPool] Failed to cleanup process for reuse, killing it", {
           error,
@@ -213,7 +220,42 @@ export class TaskRunProcessPool {
     return process.isHealthy;
   }
 
+  private startIdleTimer(process: TaskRunProcess, version: string): void {
+    this.clearIdleTimer(process);
+
+    const timer = setTimeout(() => {
+      // Synchronously remove from available pool before async kill to prevent race with getProcess()
+      const available = this.availableProcessesByVersion.get(version);
+      if (available) {
+        const index = available.indexOf(process);
+        if (index !== -1) {
+          available.splice(index, 1);
+        }
+      }
+      this.idleTimers.delete(process);
+
+      logger.debug("[TaskRunProcessPool] Idle timeout reached, killing process", {
+        pid: process.pid,
+        version,
+      });
+
+      this.killProcess(process);
+    }, TaskRunProcessPool.IDLE_TIMEOUT_MS);
+
+    this.idleTimers.set(process, timer);
+  }
+
+  private clearIdleTimer(process: TaskRunProcess): void {
+    const timer = this.idleTimers.get(process);
+    if (timer) {
+      clearTimeout(timer);
+      this.idleTimers.delete(process);
+    }
+  }
+
   private async killProcess(process: TaskRunProcess): Promise<void> {
+    this.clearIdleTimer(process);
+
     if (!process.isHealthy) {
       logger.debug("[TaskRunProcessPool] Process is not healthy, skipping cleanup", {
         processId: process.pid,
@@ -245,6 +287,12 @@ export class TaskRunProcessPool {
       versions: Array.from(this.availableProcessesByVersion.keys()),
     });
 
+    // Clear all idle timers
+    for (const timer of this.idleTimers.values()) {
+      clearTimeout(timer);
+    }
+    this.idleTimers.clear();
+
     // Kill all available processes across all versions
     const allAvailableProcesses = Array.from(this.availableProcessesByVersion.values()).flat();
     await Promise.all(allAvailableProcesses.map((process) => this.killProcess(process)));

packages/cli-v3/src/entryPoints/dev-run-controller.ts

@@ -596,7 +596,7 @@ export class DevRunController {
     const { taskRunProcess, isReused } = await this.opts.taskRunProcessPool.getProcess(
       this.opts.worker.manifest,
       {
-        id: "unmanaged",
+        id: this.opts.worker.serverWorker.id,
         contentHash: this.opts.worker.build.contentHash,
         version: this.opts.worker.serverWorker?.version,
         engine: "V2",

packages/cli-v3/src/entryPoints/dev-run-worker.ts

@@ -204,12 +204,18 @@ async function doBootstrap() {
 
     const tracingSDK = new TracingSDK({
       url: env.TRIGGER_OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
+      metricsUrl: env.TRIGGER_OTEL_METRICS_ENDPOINT,
       instrumentations: config.telemetry?.instrumentations ?? config.instrumentations ?? [],
       exporters: config.telemetry?.exporters ?? [],
       logExporters: config.telemetry?.logExporters ?? [],
+      metricExporters: config.telemetry?.metricExporters ?? [],
+      metricReaders: config.telemetry?.metricReaders ?? [],
       diagLogLevel: (env.TRIGGER_OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
       forceFlushTimeoutMillis: 30_000,
       resource: config.telemetry?.resource,
+      hostMetrics: true,
+      hostMetricGroups: ["process.cpu", "process.memory"],
+      nodejsRuntimeMetrics: true,
     });
 
     const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION);
@@ -619,8 +625,11 @@ const zodIpc = new ZodIpcConnection({
       }
       await flushAll(timeoutInMs);
     },
-    FLUSH: async ({ timeoutInMs }) => {
+    FLUSH: async ({ timeoutInMs, disableContext }) => {
       await flushAll(timeoutInMs);
+      if (disableContext) {
+        taskContext.disable();
+      }
     },
     RESOLVE_WAITPOINT: async ({ waitpoint }) => {
       _sharedWorkerRuntime?.resolveWaitpoints([waitpoint]);

packages/cli-v3/src/entryPoints/managed-run-worker.ts

@@ -183,12 +183,19 @@ async function doBootstrap() {
 
     const tracingSDK = new TracingSDK({
       url: env.TRIGGER_OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
-      instrumentations: config.instrumentations ?? [],
+      metricsUrl: env.TRIGGER_OTEL_METRICS_ENDPOINT,
+      instrumentations: config.telemetry?.instrumentations ?? config.instrumentations ?? [],
       diagLogLevel: (env.TRIGGER_OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
       forceFlushTimeoutMillis: 30_000,
       exporters: config.telemetry?.exporters ?? [],
       logExporters: config.telemetry?.logExporters ?? [],
+      metricExporters: config.telemetry?.metricExporters ?? [],
+      metricReaders: config.telemetry?.metricReaders ?? [],
       resource: config.telemetry?.resource,
+      hostMetrics: true,
+      nodejsRuntimeMetrics: true,
+      filesystemMetrics: true,
+      diskIoMetrics: true,
     });
 
     const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION);
@@ -607,8 +614,11 @@ const zodIpc = new ZodIpcConnection({
       }
       await flushAll(timeoutInMs);
     },
-    FLUSH: async ({ timeoutInMs }) => {
+    FLUSH: async ({ timeoutInMs, disableContext }) => {
       await flushAll(timeoutInMs);
+      if (disableContext) {
+        taskContext.disable();
+      }
     },
     RESOLVE_WAITPOINT: async ({ waitpoint }) => {
       _sharedWorkerRuntime?.resolveWaitpoints([waitpoint]);

packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts

@@ -1,4 +1,4 @@
-import { WorkerManifest } from "@trigger.dev/core/v3";
+import { WorkerManifest, generateFriendlyId } from "@trigger.dev/core/v3";
 import { TaskRunProcess } from "../../executions/taskRunProcess.js";
 import { RunnerEnv } from "./env.js";
 import { RunLogger, SendDebugLogOptions } from "./logger.js";
@@ -22,6 +22,7 @@ export class TaskRunProcessProvider {
   private readonly logger: RunLogger;
   private readonly processKeepAliveEnabled: boolean;
   private readonly processKeepAliveMaxExecutionCount: number;
+  private readonly machineId = generateFriendlyId("machine");
 
   // Process keep-alive state
   private persistentProcess: TaskRunProcess | null = null;
@@ -250,7 +251,7 @@ export class TaskRunProcessProvider {
       workerManifest: this.workerManifest,
       env: processEnv,
       serverWorker: {
-        id: "managed",
+        id: this.env.TRIGGER_DEPLOYMENT_ID,
         contentHash: this.env.TRIGGER_CONTENT_HASH,
         version: this.env.TRIGGER_DEPLOYMENT_VERSION,
         engine: "V2",
@@ -269,6 +270,7 @@ export class TaskRunProcessProvider {
     return {
       ...taskRunEnv,
       ...this.env.gatherProcessEnv(),
+      TRIGGER_MACHINE_ID: this.machineId,
       HEARTBEAT_INTERVAL_MS: String(this.env.TRIGGER_HEARTBEAT_INTERVAL_SECONDS * 1000),
     };
   }

packages/cli-v3/src/executions/taskRunProcess.ts

@@ -126,7 +126,7 @@ export class TaskRunProcess {
       return;
     }
 
-    await tryCatch(this.#flush());
+    await tryCatch(this.#flush({ disableContext: !kill }));
 
     if (kill) {
       await this.#gracefullyTerminate(this.options.gracefulTerminationTimeoutInMs);
@@ -240,10 +240,10 @@ export class TaskRunProcess {
     return this;
   }
 
-  async #flush(timeoutInMs: number = 5_000) {
+  async #flush({ timeoutInMs = 5_000, disableContext = false } = {}) {
     logger.debug("flushing task run process", { pid: this.pid });
 
-    await this._ipc?.sendWithAck("FLUSH", { timeoutInMs }, timeoutInMs + 1_000);
+    await this._ipc?.sendWithAck("FLUSH", { timeoutInMs, disableContext }, timeoutInMs + 1_000);
   }
 
   async #cancel(timeoutInMs: number = 30_000) {

packages/core/package.json

@@ -176,10 +176,13 @@
     "@opentelemetry/api-logs": "0.203.0",
     "@opentelemetry/core": "2.0.1",
     "@opentelemetry/exporter-logs-otlp-http": "0.203.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "0.203.0",
+    "@opentelemetry/host-metrics": "^0.37.0",
     "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
     "@opentelemetry/instrumentation": "0.203.0",
     "@opentelemetry/resources": "2.0.1",
     "@opentelemetry/sdk-logs": "0.203.0",
+    "@opentelemetry/sdk-metrics": "2.0.1",
     "@opentelemetry/sdk-trace-base": "2.0.1",
     "@opentelemetry/sdk-trace-node": "2.0.1",
     "@opentelemetry/semantic-conventions": "1.36.0",

packages/core/src/v3/config.ts

@@ -1,5 +1,6 @@
 import type { Instrumentation } from "@opentelemetry/instrumentation";
 import type { SpanExporter } from "@opentelemetry/sdk-trace-base";
+import type { MetricReader, PushMetricExporter } from "@opentelemetry/sdk-metrics";
 import type { BuildExtension } from "./build/extensions.js";
 import type {
   AnyOnFailureHookFunction,
@@ -109,6 +110,20 @@ export type TriggerConfig = {
      */
     logExporters?: Array<LogRecordExporter>;
 
+    /**
+     * Metric exporters to use for OpenTelemetry. This is useful if you want to export metrics to external services.
+     * Each exporter is automatically wrapped in a PeriodicExportingMetricReader.
+     *
+     * For more control over the reader configuration, use `metricReaders` instead.
+     */
+    metricExporters?: Array<PushMetricExporter>;
+
+    /**
+     * Metric readers for OpenTelemetry. Add custom metric readers to export
+     * metrics to external services alongside the default Trigger.dev exporter.
+     */
+    metricReaders?: Array<MetricReader>;
+
     /**
      * Resource to use for OpenTelemetry. This is useful if you want to add custom resources to your tasks.
      *

packages/core/src/v3/index.ts

@@ -49,6 +49,7 @@ export {
   NULL_SENTINEL,
 } from "./utils/flattenAttributes.js";
 export { omit } from "./utils/omit.js";
+export { generateFriendlyId, fromFriendlyId } from "./isomorphic/friendlyId.js";
 export {
   calculateNextRetryDelay,
   calculateResetAt,

packages/core/src/v3/otel/diskIoMetrics.ts

@@ -0,0 +1,92 @@
+import { type MeterProvider } from "@opentelemetry/sdk-metrics";
+import * as fs from "node:fs";
+import * as fsPromises from "node:fs/promises";
+
+const SECTOR_SIZE = 512;
+
+const FILTERED_DEVICE_PREFIXES = ["loop", "ram", "dm-"];
+
+type DiskStats = {
+  device: string;
+  readsCompleted: number;
+  sectorsRead: number;
+  writesCompleted: number;
+  sectorsWritten: number;
+};
+
+function parseProcDiskstats(content: string): DiskStats[] {
+  const entries: DiskStats[] = [];
+
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    const fields = trimmed.split(/\s+/);
+    if (fields.length < 14) continue;
+
+    const device = fields[2]!;
+
+    if (FILTERED_DEVICE_PREFIXES.some((prefix) => device.startsWith(prefix))) {
+      continue;
+    }
+
+    entries.push({
+      device,
+      readsCompleted: parseInt(fields[3]!, 10),
+      sectorsRead: parseInt(fields[5]!, 10),
+      writesCompleted: parseInt(fields[7]!, 10),
+      sectorsWritten: parseInt(fields[9]!, 10),
+    });
+  }
+
+  return entries;
+}
+
+export function startDiskIoMetrics(meterProvider: MeterProvider) {
+  try {
+    fs.accessSync("/proc/diskstats", fs.constants.R_OK);
+  } catch {
+    return;
+  }
+
+  const meter = meterProvider.getMeter("system-disk", "1.0.0");
+
+  const ioCounter = meter.createObservableCounter("system.disk.io", {
+    description: "Disk I/O bytes read and written per device",
+    unit: "By",
+  });
+
+  const opsCounter = meter.createObservableCounter("system.disk.operations", {
+    description: "Disk read/write operation counts per device",
+    unit: "{operation}",
+  });
+
+  meter.addBatchObservableCallback(
+    async (obs) => {
+      try {
+        const content = await fsPromises.readFile("/proc/diskstats", "utf-8");
+        const stats = parseProcDiskstats(content);
+
+        for (const entry of stats) {
+          const readAttrs = {
+            "system.device": entry.device,
+            "disk.io.direction": "read",
+          };
+          const writeAttrs = {
+            "system.device": entry.device,
+            "disk.io.direction": "write",
+          };
+
+          obs.observe(ioCounter, entry.sectorsRead * SECTOR_SIZE, readAttrs);
+          obs.observe(ioCounter, entry.sectorsWritten * SECTOR_SIZE, writeAttrs);
+
+          obs.observe(opsCounter, entry.readsCompleted, readAttrs);
+          obs.observe(opsCounter, entry.writesCompleted, writeAttrs);
+        }
+      } catch {
+        // Skip entire cycle on failure
+      }
+    },
+    [ioCounter, opsCounter]
+  );
+}

packages/core/src/v3/otel/filesystemMetrics.ts

@@ -0,0 +1,134 @@
+import { type MeterProvider } from "@opentelemetry/sdk-metrics";
+import * as fs from "node:fs";
+import * as fsPromises from "node:fs/promises";
+
+const VIRTUAL_FS_TYPES = new Set([
+  "proc",
+  "sysfs",
+  "devpts",
+  "tmpfs",
+  "devtmpfs",
+  "cgroup",
+  "cgroup2",
+  "squashfs",
+  "autofs",
+  "debugfs",
+  "securityfs",
+  "pstore",
+  "bpf",
+  "tracefs",
+  "hugetlbfs",
+  "mqueue",
+  "fusectl",
+  "configfs",
+  "binfmt_misc",
+]);
+
+type MountEntry = {
+  device: string;
+  mountpoint: string;
+  fsType: string;
+  options: string;
+};
+
+function parseProcMounts(content: string): MountEntry[] {
+  const entries: MountEntry[] = [];
+
+  for (const line of content.split("\n")) {
+    if (!line.trim()) continue;
+
+    const parts = line.split(" ");
+    if (parts.length < 4) continue;
+
+    const fsType = parts[2]!;
+    if (VIRTUAL_FS_TYPES.has(fsType)) continue;
+
+    entries.push({
+      device: parts[0]!,
+      mountpoint: unescapeMountPath(parts[1]!),
+      fsType,
+      options: parts[3]!,
+    });
+  }
+
+  return entries;
+}
+
+function unescapeMountPath(path: string): string {
+  return path.replace(/\\040/g, " ").replace(/\\011/g, "\t");
+}
+
+export function startFilesystemMetrics(meterProvider: MeterProvider) {
+  try {
+    fs.accessSync("/proc/mounts", fs.constants.R_OK);
+  } catch {
+    return;
+  }
+
+  if (typeof fsPromises.statfs !== "function") {
+    return;
+  }
+
+  const meter = meterProvider.getMeter("system-filesystem", "1.0.0");
+
+  const usageCounter = meter.createObservableUpDownCounter("system.filesystem.usage", {
+    description: "Filesystem bytes used, free, and reserved per mountpoint",
+    unit: "By",
+  });
+
+  const utilizationGauge = meter.createObservableGauge("system.filesystem.utilization", {
+    description: "Fraction of filesystem space used (0-1)",
+    unit: "1",
+  });
+
+  meter.addBatchObservableCallback(
+    async (obs) => {
+      try {
+        const mountsContent = await fsPromises.readFile("/proc/mounts", "utf-8");
+        const mounts = parseProcMounts(mountsContent);
+
+        for (const mount of mounts) {
+          try {
+            const stats = await fsPromises.statfs(mount.mountpoint);
+            const bsize = stats.bsize;
+            const total = stats.blocks * bsize;
+            const free = stats.bavail * bsize;
+            const reserved = (stats.bfree - stats.bavail) * bsize;
+            const used = total - stats.bfree * bsize;
+
+            const mode = mount.options.startsWith("ro") ? "ro" : "rw";
+
+            const baseAttrs = {
+              "system.device": mount.device,
+              "system.filesystem.type": mount.fsType,
+              "system.filesystem.mountpoint": mount.mountpoint,
+              "system.filesystem.mode": mode,
+            };
+
+            obs.observe(usageCounter, used, {
+              ...baseAttrs,
+              "system.filesystem.state": "used",
+            });
+            obs.observe(usageCounter, free, {
+              ...baseAttrs,
+              "system.filesystem.state": "free",
+            });
+            obs.observe(usageCounter, reserved, {
+              ...baseAttrs,
+              "system.filesystem.state": "reserved",
+            });
+
+            if (total > 0) {
+              obs.observe(utilizationGauge, used / total, baseAttrs);
+            }
+          } catch {
+            // Skip this mount on statfs failure
+          }
+        }
+      } catch {
+        // Skip entire cycle on failure
+      }
+    },
+    [usageCounter, utilizationGauge]
+  );
+}

packages/core/src/v3/otel/machineId.ts

@@ -0,0 +1,4 @@
+import { generateFriendlyId } from "../isomorphic/friendlyId.js";
+import { getEnvVar } from "../utils/getEnv.js";
+
+export const machineId = getEnvVar("TRIGGER_MACHINE_ID") ?? generateFriendlyId("machine");

packages/core/src/v3/otel/nodejsRuntimeMetrics.ts

@@ -0,0 +1,81 @@
+import { type MeterProvider } from "@opentelemetry/sdk-metrics";
+import type { ObservableGauge } from "@opentelemetry/api";
+import { performance, monitorEventLoopDelay } from "node:perf_hooks";
+
+function tryMonitorEventLoopDelay() {
+  try {
+    const eld = monitorEventLoopDelay({ resolution: 20 });
+    eld.enable();
+    return eld;
+  } catch {
+    // monitorEventLoopDelay is not implemented in Bun
+    return undefined;
+  }
+}
+
+export function startNodejsRuntimeMetrics(meterProvider: MeterProvider) {
+  const meter = meterProvider.getMeter("nodejs-runtime", "1.0.0");
+
+  // Event loop utilization (diff between collection intervals)
+  let lastElu = performance.eventLoopUtilization();
+
+  const eluGauge = meter.createObservableGauge("nodejs.event_loop.utilization", {
+    description: "Event loop utilization over the last collection interval",
+    unit: "1",
+  });
+
+  // Event loop delay histogram (from perf_hooks) — not available in Bun
+  const eld = tryMonitorEventLoopDelay();
+
+  const observables: ObservableGauge[] = [eluGauge];
+
+  let eldP95: ObservableGauge | undefined;
+  let eldMax: ObservableGauge | undefined;
+
+  if (eld) {
+    eldP95 = meter.createObservableGauge("nodejs.event_loop.delay.p95", {
+      description: "p95 event loop delay",
+      unit: "s",
+    });
+    eldMax = meter.createObservableGauge("nodejs.event_loop.delay.max", {
+      description: "Max event loop delay",
+      unit: "s",
+    });
+    observables.push(eldP95, eldMax);
+  }
+
+  // Heap metrics
+  const heapUsed = meter.createObservableGauge("nodejs.heap.used", {
+    description: "V8 heap used",
+    unit: "By",
+  });
+  const heapTotal = meter.createObservableGauge("nodejs.heap.total", {
+    description: "V8 heap total allocated",
+    unit: "By",
+  });
+  observables.push(heapUsed, heapTotal);
+
+  // Single batch callback for all metrics
+  meter.addBatchObservableCallback(
+    (obs) => {
+      // ELU
+      const currentElu = performance.eventLoopUtilization();
+      const diff = performance.eventLoopUtilization(currentElu, lastElu);
+      lastElu = currentElu;
+      obs.observe(eluGauge, diff.utilization);
+
+      // Event loop delay (nanoseconds -> seconds)
+      if (eld && eldP95 && eldMax) {
+        obs.observe(eldP95, eld.percentile(95) / 1e9);
+        obs.observe(eldMax, eld.max / 1e9);
+        eld.reset();
+      }
+
+      // Heap
+      const mem = process.memoryUsage();
+      obs.observe(heapUsed, mem.heapUsed);
+      obs.observe(heapTotal, mem.heapTotal);
+    },
+    observables
+  );
+}

packages/core/src/v3/otel/tracingSDK.ts

@@ -4,11 +4,14 @@ import {
   TraceFlags,
   TracerProvider,
   diag,
+  metrics,
 } from "@opentelemetry/api";
 import { logs } from "@opentelemetry/api-logs";
 import { TraceState } from "@opentelemetry/core";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { HostMetrics } from "@opentelemetry/host-metrics";
 import { registerInstrumentations, type Instrumentation } from "@opentelemetry/instrumentation";
 import {
   detectResources,
@@ -24,6 +27,13 @@ import {
   ReadableLogRecord,
   SimpleLogRecordProcessor,
 } from "@opentelemetry/sdk-logs";
+import {
+  AggregationType,
+  MeterProvider,
+  PeriodicExportingMetricReader,
+  type MetricReader,
+  type PushMetricExporter,
+} from "@opentelemetry/sdk-metrics";
 import { RandomIdGenerator, SpanProcessor } from "@opentelemetry/sdk-trace-base";
 import {
   BatchSpanProcessor,
@@ -32,7 +42,6 @@ import {
   SimpleSpanProcessor,
   SpanExporter,
 } from "@opentelemetry/sdk-trace-node";
-import { SemanticResourceAttributes, SEMATTRS_HTTP_URL } from "@opentelemetry/semantic-conventions";
 import { VERSION } from "../../version.js";
 import {
   OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT,
@@ -47,11 +56,17 @@ import {
 import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
 import { taskContext } from "../task-context-api.js";
 import {
+  BufferingMetricExporter,
   TaskContextLogProcessor,
+  TaskContextMetricExporter,
   TaskContextSpanProcessor,
 } from "../taskContext/otelProcessors.js";
 import { traceContext } from "../trace-context-api.js";
 import { getEnvVar } from "../utils/getEnv.js";
+import { machineId } from "./machineId.js";
+import { startDiskIoMetrics } from "./diskIoMetrics.js";
+import { startFilesystemMetrics } from "./filesystemMetrics.js";
+import { startNodejsRuntimeMetrics } from "./nodejsRuntimeMetrics.js";
 
 export type TracingDiagnosticLogLevel =
   | "none"
@@ -64,12 +79,26 @@ export type TracingDiagnosticLogLevel =
 
 export type TracingSDKConfig = {
   url: string;
+  metricsUrl?: string;
   forceFlushTimeoutMillis?: number;
   instrumentations?: Instrumentation[];
   exporters?: SpanExporter[];
   logExporters?: LogRecordExporter[];
+  metricExporters?: PushMetricExporter[];
+  metricReaders?: MetricReader[];
   diagLogLevel?: TracingDiagnosticLogLevel;
   resource?: Resource;
+  hostMetrics?: boolean;
+  /** Limit host metrics collection to specific groups (e.g. ["process.cpu", "process.memory"]) */
+  hostMetricGroups?: string[];
+  /** Enable Node.js runtime metrics (event loop utilization, heap usage, etc.) */
+  nodejsRuntimeMetrics?: boolean;
+  /** Enable filesystem metrics (Linux only, reads /proc/mounts + fs.statfs) */
+  filesystemMetrics?: boolean;
+  /** Enable disk I/O metrics (Linux only, reads /proc/diskstats) */
+  diskIoMetrics?: boolean;
+  /** Metric instrument name patterns to drop (supports wildcards, e.g. "system.cpu.*") */
+  droppedMetrics?: string[];
 };
 
 const idGenerator = new RandomIdGenerator();
@@ -78,6 +107,7 @@ export class TracingSDK {
   private readonly _logProvider: LoggerProvider;
   private readonly _spanExporter: SpanExporter;
   private readonly _traceProvider: NodeTracerProvider;
+  private readonly _meterProvider: MeterProvider;
 
   public readonly getLogger: LoggerProvider["getLogger"];
   public readonly getTracer: TracerProvider["getTracer"];
@@ -99,13 +129,13 @@ export class TracingSDK {
     })
       .merge(
         resourceFromAttributes({
-          [SemanticResourceAttributes.CLOUD_PROVIDER]: "trigger.dev",
-          [SemanticResourceAttributes.SERVICE_NAME]:
-            getEnvVar("TRIGGER_OTEL_SERVICE_NAME") ?? "trigger.dev",
+          "cloud.provider": "trigger.dev",
+          "service.name": getEnvVar("TRIGGER_OTEL_SERVICE_NAME") ?? "trigger.dev",
           [SemanticInternalAttributes.TRIGGER]: true,
           [SemanticInternalAttributes.CLI_VERSION]: VERSION,
           [SemanticInternalAttributes.SDK_VERSION]: VERSION,
           [SemanticInternalAttributes.SDK_LANGUAGE]: "typescript",
+          [SemanticInternalAttributes.MACHINE_ID]: machineId,
         })
       )
       .merge(resourceFromAttributes(envResourceAttributes))
@@ -259,16 +289,99 @@ export class TracingSDK {
 
     logs.setGlobalLoggerProvider(loggerProvider);
 
+    // Metrics setup
+    const metricsUrl =
+      config.metricsUrl ??
+      getEnvVar("TRIGGER_OTEL_METRICS_ENDPOINT") ??
+      `${config.url}/v1/metrics`;
+
+    const rawMetricExporter = new OTLPMetricExporter({
+      url: metricsUrl,
+      timeoutMillis: config.forceFlushTimeoutMillis,
+    });
+
+    const collectionIntervalMs = parseInt(
+      getEnvVar("TRIGGER_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS") ?? "10000"
+    );
+    const exportIntervalMs = parseInt(
+      getEnvVar("TRIGGER_OTEL_METRICS_EXPORT_INTERVAL_MILLIS") ?? "30000"
+    );
+
+    // Chain: PeriodicReader(10s) → TaskContextMetricExporter → BufferingMetricExporter(30s) → OTLP
+    const bufferingExporter = new BufferingMetricExporter(rawMetricExporter, exportIntervalMs);
+    const metricExporter = new TaskContextMetricExporter(bufferingExporter);
+
+    const exportTimeoutMillis = parseInt(
+      getEnvVar("TRIGGER_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS") ?? "30000"
+    );
+
+    const metricReaders: MetricReader[] = [
+      new PeriodicExportingMetricReader({
+        exporter: metricExporter,
+        exportIntervalMillis: collectionIntervalMs,
+        exportTimeoutMillis: Math.min(exportTimeoutMillis, collectionIntervalMs),
+      }),
+      ...(config.metricExporters ?? []).map(
+        (exporter) =>
+          new PeriodicExportingMetricReader({
+            exporter,
+            exportIntervalMillis: collectionIntervalMs,
+            exportTimeoutMillis: Math.min(exportTimeoutMillis, collectionIntervalMs),
+          })
+      ),
+      ...(config.metricReaders ?? []),
+    ];
+
+    const meterProvider = new MeterProvider({
+      resource: commonResources,
+      readers: metricReaders,
+      views: (config.droppedMetrics ?? []).map((pattern) => ({
+        instrumentName: pattern,
+        aggregation: { type: AggregationType.DROP },
+      })),
+    });
+
+    this._meterProvider = meterProvider;
+    metrics.setGlobalMeterProvider(meterProvider);
+
+    if (config.hostMetrics) {
+      const hostMetrics = new HostMetrics({
+        meterProvider,
+        metricGroups: config.hostMetricGroups,
+      });
+      hostMetrics.start();
+    }
+
+    if (config.nodejsRuntimeMetrics) {
+      startNodejsRuntimeMetrics(meterProvider);
+    }
+
+    if (config.filesystemMetrics) {
+      startFilesystemMetrics(meterProvider);
+    }
+
+    if (config.diskIoMetrics) {
+      startDiskIoMetrics(meterProvider);
+    }
+
     this.getLogger = loggerProvider.getLogger.bind(loggerProvider);
     this.getTracer = traceProvider.getTracer.bind(traceProvider);
   }
 
   public async flush() {
-    await Promise.all([this._traceProvider.forceFlush(), this._logProvider.forceFlush()]);
+    await Promise.all([
+      this._traceProvider.forceFlush(),
+      this._logProvider.forceFlush(),
+      this._meterProvider.forceFlush(),
+    ]);
   }
 
   public async shutdown() {
-    await Promise.all([this._traceProvider.shutdown(), this._logProvider.shutdown()]);
+    await Promise.all([
+      this._traceProvider.shutdown(),
+      this._logProvider.shutdown(),
+      this._meterProvider.shutdown(),
+    ]);
   }
 }
 
@@ -465,7 +578,7 @@ function isSpanInternalOnly(span: ReadableSpan): boolean {
     return true;
   }
 
-  const httpUrl = span.attributes[SEMATTRS_HTTP_URL] ?? span.attributes["url.full"];
+  const httpUrl = span.attributes["http.url"] ?? span.attributes["url.full"];
 
   const url = safeParseUrl(httpUrl);
 

packages/core/src/v3/schemas/messages.ts

@@ -213,6 +213,7 @@ export const WorkerToExecutorMessageCatalog = {
   FLUSH: {
     message: z.object({
       timeoutInMs: z.number(),
+      disableContext: z.boolean().optional(),
     }),
     callback: z.void(),
   },

packages/core/src/v3/semanticInternalAttributes.ts

@@ -19,6 +19,7 @@ export const SemanticInternalAttributes = {
   TASK_EXPORT_NAME: "ctx.task.exportName",
   QUEUE_NAME: "ctx.queue.name",
   QUEUE_ID: "ctx.queue.id",
+  MACHINE_ID: "ctx.machine.id",
   MACHINE_PRESET_NAME: "ctx.machine.name",
   MACHINE_PRESET_CPU: "ctx.machine.cpu",
   MACHINE_PRESET_MEMORY: "ctx.machine.memory",
@@ -65,4 +66,5 @@ export const SemanticInternalAttributes = {
   WARM_START: "warm_start",
   ATTEMPT_EXECUTION_COUNT: "$trigger.executionCount",
   TASK_EVENT_STORE: "$trigger.taskEventStore",
+  RUN_TAGS: "ctx.run.tags",
 };

packages/core/src/v3/taskContext/index.ts

@@ -1,13 +1,14 @@
 import { Attributes } from "@opentelemetry/api";
 import { ServerBackgroundWorker, TaskRunContext } from "../schemas/index.js";
 import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
-import { getGlobal, registerGlobal, unregisterGlobal } from "../utils/globals.js";
+import { getGlobal, registerGlobal } from "../utils/globals.js";
 import { TaskContext } from "./types.js";
 
 const API_NAME = "task-context";
 
 export class TaskContextAPI {
   private static _instance?: TaskContextAPI;
+  private _runDisabled = false;
 
   private constructor() {}
 
@@ -23,6 +24,10 @@ export class TaskContextAPI {
     return this.#getTaskContext() !== undefined;
   }
 
+  get isRunDisabled(): boolean {
+    return this._runDisabled;
+  }
+
   get ctx(): TaskRunContext | undefined {
     return this.#getTaskContext()?.ctx;
   }
@@ -98,11 +103,12 @@ export class TaskContextAPI {
   }
 
   public disable() {
-    unregisterGlobal(API_NAME);
+    this._runDisabled = true;
   }
 
   public setGlobalTaskContext(taskContext: TaskContext): boolean {
-    return registerGlobal(API_NAME, taskContext);
+    this._runDisabled = false;
+    return registerGlobal(API_NAME, taskContext, true);
   }
 
   #getTaskContext(): TaskContext | undefined {

packages/core/src/v3/taskContext/otelProcessors.ts

@@ -1,5 +1,15 @@
-import { Context, trace, Tracer } from "@opentelemetry/api";
+import { Attributes, Context, trace, Tracer } from "@opentelemetry/api";
+import { ExportResult, ExportResultCode } from "@opentelemetry/core";
 import { LogRecordProcessor, SdkLogRecord } from "@opentelemetry/sdk-logs";
+import type {
+  AggregationOption,
+  AggregationTemporality,
+  InstrumentType,
+  MetricData,
+  PushMetricExporter,
+  ResourceMetrics,
+  ScopeMetrics,
+} from "@opentelemetry/sdk-metrics";
 import { Span, SpanProcessor } from "@opentelemetry/sdk-trace-base";
 import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
 import { taskContext } from "../task-context-api.js";
@@ -104,3 +114,194 @@ export class TaskContextLogProcessor implements LogRecordProcessor {
     return this._innerProcessor.shutdown();
   }
 }
+
+export class TaskContextMetricExporter implements PushMetricExporter {
+  selectAggregationTemporality?: (instrumentType: InstrumentType) => AggregationTemporality;
+  selectAggregation?: (instrumentType: InstrumentType) => AggregationOption;
+
+  constructor(private _innerExporter: PushMetricExporter) {
+    if (_innerExporter.selectAggregationTemporality) {
+      this.selectAggregationTemporality =
+        _innerExporter.selectAggregationTemporality.bind(_innerExporter);
+    }
+    if (_innerExporter.selectAggregation) {
+      this.selectAggregation = _innerExporter.selectAggregation.bind(_innerExporter);
+    }
+  }
+
+  export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void {
+    if (!taskContext.ctx) {
+      // No task context yet — pass through without adding context attributes
+      this._innerExporter.export(metrics, resultCallback);
+      return;
+    }
+
+    const ctx = taskContext.ctx;
+
+    let contextAttrs: Attributes;
+
+    if (taskContext.isRunDisabled) {
+      // Between runs: keep environment/project/org/machine attrs, strip run-specific ones
+      contextAttrs = {
+        [SemanticInternalAttributes.ENVIRONMENT_ID]: ctx.environment.id,
+        [SemanticInternalAttributes.ENVIRONMENT_TYPE]: ctx.environment.type,
+        [SemanticInternalAttributes.ORGANIZATION_ID]: ctx.organization.id,
+        [SemanticInternalAttributes.PROJECT_ID]: ctx.project.id,
+        [SemanticInternalAttributes.MACHINE_PRESET_NAME]: ctx.machine?.name,
+      };
+    } else {
+      // During a run: full context attrs
+      contextAttrs = {
+        [SemanticInternalAttributes.RUN_ID]: ctx.run.id,
+        [SemanticInternalAttributes.TASK_SLUG]: ctx.task.id,
+        [SemanticInternalAttributes.ATTEMPT_NUMBER]: ctx.attempt.number,
+        [SemanticInternalAttributes.ENVIRONMENT_ID]: ctx.environment.id,
+        [SemanticInternalAttributes.ORGANIZATION_ID]: ctx.organization.id,
+        [SemanticInternalAttributes.PROJECT_ID]: ctx.project.id,
+        [SemanticInternalAttributes.MACHINE_PRESET_NAME]: ctx.machine?.name,
+        [SemanticInternalAttributes.ENVIRONMENT_TYPE]: ctx.environment.type,
+      };
+    }
+
+    if (taskContext.worker) {
+      contextAttrs[SemanticInternalAttributes.WORKER_ID] = taskContext.worker.id;
+      contextAttrs[SemanticInternalAttributes.WORKER_VERSION] = taskContext.worker.version;
+    }
+
+    if (!taskContext.isRunDisabled && ctx.run.tags?.length) {
+      contextAttrs[SemanticInternalAttributes.RUN_TAGS] = ctx.run.tags;
+    }
+
+    const modified: ResourceMetrics = {
+      resource: metrics.resource,
+      scopeMetrics: metrics.scopeMetrics.map((scope) => ({
+        ...scope,
+        metrics: scope.metrics.map(
+          (metric) =>
+            ({
+              ...metric,
+              dataPoints: metric.dataPoints.map((dp) => ({
+                ...dp,
+                attributes: { ...dp.attributes, ...contextAttrs },
+              })),
+            }) as MetricData
+        ),
+      })),
+    };
+
+    this._innerExporter.export(modified, resultCallback);
+  }
+
+  forceFlush(): Promise<void> {
+    return this._innerExporter.forceFlush();
+  }
+
+  shutdown(): Promise<void> {
+    return this._innerExporter.shutdown();
+  }
+}
+
+export class BufferingMetricExporter implements PushMetricExporter {
+  selectAggregationTemporality?: (instrumentType: InstrumentType) => AggregationTemporality;
+  selectAggregation?: (instrumentType: InstrumentType) => AggregationOption;
+
+  private _buffer: ResourceMetrics[] = [];
+  private _lastFlushTime = Date.now();
+
+  constructor(
+    private _innerExporter: PushMetricExporter,
+    private _flushIntervalMs: number
+  ) {
+    if (_innerExporter.selectAggregationTemporality) {
+      this.selectAggregationTemporality =
+        _innerExporter.selectAggregationTemporality.bind(_innerExporter);
+    }
+    if (_innerExporter.selectAggregation) {
+      this.selectAggregation = _innerExporter.selectAggregation.bind(_innerExporter);
+    }
+  }
+
+  export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void {
+    this._buffer.push(metrics);
+
+    const now = Date.now();
+    if (now - this._lastFlushTime >= this._flushIntervalMs) {
+      this._lastFlushTime = now;
+      const merged = this._mergeBuffer();
+      this._innerExporter.export(merged, resultCallback);
+    } else {
+      resultCallback({ code: ExportResultCode.SUCCESS });
+    }
+  }
+
+  forceFlush(): Promise<void> {
+    if (this._buffer.length > 0) {
+      this._lastFlushTime = Date.now();
+      const merged = this._mergeBuffer();
+      return new Promise<void>((resolve, reject) => {
+        this._innerExporter.export(merged, (result) => {
+          if (result.code === ExportResultCode.SUCCESS) {
+            resolve();
+          } else {
+            reject(result.error ?? new Error("Export failed"));
+          }
+        });
+      }).then(() => this._innerExporter.forceFlush());
+    }
+    return this._innerExporter.forceFlush();
+  }
+
+  shutdown(): Promise<void> {
+    return this.forceFlush().then(() => this._innerExporter.shutdown());
+  }
+
+  private _mergeBuffer(): ResourceMetrics {
+    const batch = this._buffer;
+    this._buffer = [];
+
+    if (batch.length === 1) {
+      return batch[0]!;
+    }
+
+    const base = batch[0]!;
+
+    // Merge all scopeMetrics by scope name, then metrics by descriptor name
+    const scopeMap = new Map<string, { scope: ScopeMetrics["scope"]; metricsMap: Map<string, MetricData> }>();
+
+    for (const rm of batch) {
+      for (const sm of rm.scopeMetrics) {
+        const scopeKey = sm.scope.name;
+        let scopeEntry = scopeMap.get(scopeKey);
+        if (!scopeEntry) {
+          scopeEntry = { scope: sm.scope, metricsMap: new Map() };
+          scopeMap.set(scopeKey, scopeEntry);
+        }
+
+        for (const metric of sm.metrics) {
+          const metricKey = metric.descriptor.name;
+          const existing = scopeEntry.metricsMap.get(metricKey);
+          if (existing) {
+            // Append data points from this collection to the existing metric
+            scopeEntry.metricsMap.set(metricKey, {
+              ...existing,
+              dataPoints: [...existing.dataPoints, ...metric.dataPoints],
+            } as MetricData);
+          } else {
+            scopeEntry.metricsMap.set(metricKey, {
+              ...metric,
+              dataPoints: [...metric.dataPoints],
+            } as MetricData);
+          }
+        }
+      }
+    }
+
+    return {
+      resource: base.resource,
+      scopeMetrics: Array.from(scopeMap.values()).map(({ scope, metricsMap }) => ({
+        scope,
+        metrics: Array.from(metricsMap.values()),
+      })),
+    };
+  }
+}

packages/core/src/v3/workers/index.ts

@@ -14,6 +14,7 @@ export { StandardResourceCatalog } from "../resource-catalog/standardResourceCat
 export {
   TaskContextSpanProcessor,
   TaskContextLogProcessor,
+  TaskContextMetricExporter,
 } from "../taskContext/otelProcessors.js";
 export * from "../usage-api.js";
 export { DevUsageManager } from "../usage/devUsageManager.js";

packages/trigger-sdk/src/v3/otel.ts

@@ -1,7 +1,9 @@
+import { metrics } from "@opentelemetry/api";
 import { traceContext } from "@trigger.dev/core/v3";
 
 export const otel = {
   withExternalTrace: <T>(fn: () => T): T => {
     return traceContext.withExternalTrace(fn);
   },
+  metrics,
 };

pnpm-lock.yaml

@@ -345,8 +345,8 @@ importers:
         specifier: 0.203.0
         version: 0.203.0(@opentelemetry/api@1.9.0)
       '@opentelemetry/host-metrics':
-        specifier: ^0.36.0
-        version: 0.36.0(@opentelemetry/api@1.9.0)
+        specifier: ^0.37.0
+        version: 0.37.0(@opentelemetry/api@1.9.0)
       '@opentelemetry/instrumentation':
         specifier: 0.203.0
         version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0)
@@ -1698,9 +1698,15 @@ importers:
       '@opentelemetry/exporter-logs-otlp-http':
         specifier: 0.203.0
         version: 0.203.0(@opentelemetry/api@1.9.0)
+      '@opentelemetry/exporter-metrics-otlp-http':
+        specifier: 0.203.0
+        version: 0.203.0(@opentelemetry/api@1.9.0)
       '@opentelemetry/exporter-trace-otlp-http':
         specifier: 0.203.0
         version: 0.203.0(@opentelemetry/api@1.9.0)
+      '@opentelemetry/host-metrics':
+        specifier: ^0.37.0
+        version: 0.37.0(@opentelemetry/api@1.9.0)
       '@opentelemetry/instrumentation':
         specifier: 0.203.0
         version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0)
@@ -1710,6 +1716,9 @@ importers:
       '@opentelemetry/sdk-logs':
         specifier: 0.203.0
         version: 0.203.0(@opentelemetry/api@1.9.0)
+      '@opentelemetry/sdk-metrics':
+        specifier: 2.0.1
+        version: 2.0.1(@opentelemetry/api@1.9.0)
       '@opentelemetry/sdk-trace-base':
         specifier: 2.0.1
         version: 2.0.1(@opentelemetry/api@1.9.0)
@@ -6477,8 +6486,8 @@ packages:
     peerDependencies:
       '@opentelemetry/api': ^1.0.0
 
-  '@opentelemetry/host-metrics@0.36.0':
-    resolution: {integrity: sha512-14lNY57qa21V3ZOl6xrqLMHR0HGlnPIApR6hr3oCw/Dqs5IzxhTwt2X8Stn82vWJJis7j/ezn11oODsizHj2dQ==}
+  '@opentelemetry/host-metrics@0.37.0':
+    resolution: {integrity: sha512-gf6nRFci0PTni9R1QQKjZ2uZE4Y6olLKhlwdM0qqLbbn3SBVKyP2jyBMiosBTHtRNLjY7s8hzQ44eLdK5wkGNQ==}
     engines: {node: ^18.19.0 || >=20.6.0}
     peerDependencies:
       '@opentelemetry/api': ^1.3.0
@@ -23121,7 +23130,7 @@ snapshots:
   '@epic-web/test-server@0.1.0(bufferutil@4.0.9)':
     dependencies:
       '@hono/node-server': 1.12.2(hono@4.5.11)
-      '@hono/node-ws': 1.0.4(@hono/node-server@1.12.2(hono@4.5.11))(bufferutil@4.0.9)
+      '@hono/node-ws': 1.0.4(@hono/node-server@1.12.2(hono@4.11.8))(bufferutil@4.0.9)
       '@open-draft/deferred-promise': 2.2.0
       '@types/ws': 8.5.12
       hono: 4.5.11
@@ -23876,7 +23885,7 @@ snapshots:
     dependencies:
       hono: 4.11.8
 
-  '@hono/node-ws@1.0.4(@hono/node-server@1.12.2(hono@4.5.11))(bufferutil@4.0.9)':
+  '@hono/node-ws@1.0.4(@hono/node-server@1.12.2(hono@4.11.8))(bufferutil@4.0.9)':
     dependencies:
       '@hono/node-server': 1.12.2(hono@4.5.11)
       ws: 8.18.3(bufferutil@4.0.9)
@@ -24909,7 +24918,7 @@ snapshots:
       '@opentelemetry/sdk-trace-base': 2.0.1(@opentelemetry/api@1.9.0)
       '@opentelemetry/semantic-conventions': 1.36.0
 
-  '@opentelemetry/host-metrics@0.36.0(@opentelemetry/api@1.9.0)':
+  '@opentelemetry/host-metrics@0.37.0(@opentelemetry/api@1.9.0)':
     dependencies:
       '@opentelemetry/api': 1.9.0
       systeminformation: 5.27.14

references/bun-catalog/src/trigger/bun.ts

@@ -8,6 +8,8 @@ export const bunTask = task({
     const query = db.query("select 'Hello world' as message;");
     console.log(query.get()); // => { message: "Hello world" }
 
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
     return {
       message: "Query executed",
       bunVersion: Bun.version,

references/bun-catalog/trigger.config.ts

@@ -2,7 +2,7 @@ import { defineConfig } from "@trigger.dev/sdk";
 
 export default defineConfig({
   runtime: "bun",
-  project: "proj_uxbxncnbsyamyxeqtucu",
+  project: process.env.TRIGGER_PROJECT_REF!,
   maxDuration: 3600,
   machine: "small-2x",
   retries: {

references/hello-world/src/trigger/metrics.ts

@@ -0,0 +1,326 @@
+import { batch, logger, otel, task } from "@trigger.dev/sdk";
+import { createHash } from "node:crypto";
+import { setTimeout } from "node:timers/promises";
+
+// Custom metrics — instruments are created once at module level
+const meter = otel.metrics.getMeter("hello-world");
+const itemsProcessedCounter = meter.createCounter("items.processed", {
+  description: "Total number of items processed",
+  unit: "items",
+});
+const itemDurationHistogram = meter.createHistogram("item.duration", {
+  description: "Time spent processing each item",
+  unit: "ms",
+});
+const queueDepthGauge = meter.createUpDownCounter("queue.depth", {
+  description: "Current simulated queue depth",
+  unit: "items",
+});
+
+/**
+ * Tight computational loop that produces sustained high CPU utilization.
+ * Uses repeated SHA-256 hashing to keep the CPU busy.
+ */
+export const cpuIntensive = task({
+  id: "cpu-intensive",
+  run: async (
+    {
+      durationSeconds = 60,
+    }: {
+      durationSeconds?: number;
+    },
+    { ctx }
+  ) => {
+    logger.info("Starting CPU-intensive workload", { durationSeconds });
+
+    const deadline = Date.now() + durationSeconds * 1000;
+    let iterations = 0;
+    let data = Buffer.from("seed-data-for-hashing");
+
+    while (Date.now() < deadline) {
+      // Tight hashing loop — ~100ms chunks then yield to event loop
+      const chunkEnd = Date.now() + 100;
+      while (Date.now() < chunkEnd) {
+        data = createHash("sha256").update(data).digest();
+        iterations++;
+      }
+      // Yield to let metrics collection and heartbeats run
+      await setTimeout(1);
+    }
+
+    logger.info("CPU-intensive workload complete", { iterations });
+    return { iterations };
+  },
+});
+
+/**
+ * Progressively allocates memory in steps, holds it, then releases.
+ * Produces a staircase-shaped memory usage graph.
+ */
+export const memoryRamp = task({
+  id: "memory-ramp",
+  run: async (
+    {
+      steps = 6,
+      stepSizeMb = 50,
+      stepIntervalSeconds = 5,
+      holdSeconds = 15,
+    }: {
+      steps?: number;
+      stepSizeMb?: number;
+      stepIntervalSeconds?: number;
+      holdSeconds?: number;
+    },
+    { ctx }
+  ) => {
+    logger.info("Starting memory ramp", { steps, stepSizeMb, stepIntervalSeconds, holdSeconds });
+
+    const allocations: Buffer[] = [];
+
+    // Ramp up — allocate in steps
+    for (let i = 0; i < steps; i++) {
+      const buf = Buffer.alloc(stepSizeMb * 1024 * 1024, 0xff);
+      allocations.push(buf);
+      logger.info(`Allocated step ${i + 1}/${steps}`, {
+        totalAllocatedMb: (i + 1) * stepSizeMb,
+      });
+      await setTimeout(stepIntervalSeconds * 1000);
+    }
+
+    // Hold at peak
+    logger.info("Holding at peak memory", { totalMb: steps * stepSizeMb });
+    await setTimeout(holdSeconds * 1000);
+
+    // Release
+    allocations.length = 0;
+    global.gc?.();
+    logger.info("Released all allocations");
+
+    // Let metrics capture the drop
+    await setTimeout(10_000);
+
+    logger.info("Memory ramp complete");
+    return { peakMb: steps * stepSizeMb };
+  },
+});
+
+/**
+ * Alternates between CPU-intensive bursts and idle sleep periods.
+ * Produces a sawtooth/square-wave CPU utilization pattern.
+ */
+export const burstyWorkload = task({
+  id: "bursty-workload",
+  run: async (
+    {
+      cycles = 5,
+      burstSeconds = 5,
+      idleSeconds = 5,
+    }: {
+      cycles?: number;
+      burstSeconds?: number;
+      idleSeconds?: number;
+    },
+    { ctx }
+  ) => {
+    logger.info("Starting bursty workload", { cycles, burstSeconds, idleSeconds });
+
+    for (let cycle = 0; cycle < cycles; cycle++) {
+      // Burst phase — hash as fast as possible
+      logger.info(`Cycle ${cycle + 1}/${cycles}: burst phase`);
+      const burstDeadline = Date.now() + burstSeconds * 1000;
+      let data = Buffer.from(`burst-cycle-${cycle}`);
+      while (Date.now() < burstDeadline) {
+        const chunkEnd = Date.now() + 100;
+        while (Date.now() < chunkEnd) {
+          data = createHash("sha256").update(data).digest();
+        }
+        await setTimeout(1);
+      }
+
+      // Idle phase
+      logger.info(`Cycle ${cycle + 1}/${cycles}: idle phase`);
+      await setTimeout(idleSeconds * 1000);
+    }
+
+    logger.info("Bursty workload complete", { totalCycles: cycles });
+    return { cycles };
+  },
+});
+
+/**
+ * Simulates a data processing pipeline with distinct phases:
+ *   1. Read phase — light CPU, growing memory (buffering data)
+ *   2. Process phase — high CPU, stable memory (crunching data)
+ *   3. Write phase — low CPU, memory drops (streaming out results)
+ *
+ * Shows clear phase transitions in both CPU and memory graphs.
+ */
+export const sustainedWorkload = task({
+  id: "sustained-workload",
+  run: async (
+    {
+      readSeconds = 20,
+      processSeconds = 20,
+      writeSeconds = 20,
+      dataSizeMb = 100,
+    }: {
+      readSeconds?: number;
+      processSeconds?: number;
+      writeSeconds?: number;
+      dataSizeMb?: number;
+    },
+    { ctx }
+  ) => {
+    logger.info("Starting sustained workload — read phase", { readSeconds, dataSizeMb });
+
+    // Phase 1: Read — gradually accumulate buffers (memory ramp, low CPU)
+    const chunks: Buffer[] = [];
+    const chunkCount = 10;
+    const chunkSize = Math.floor((dataSizeMb * 1024 * 1024) / chunkCount);
+    const readInterval = (readSeconds * 1000) / chunkCount;
+
+    for (let i = 0; i < chunkCount; i++) {
+      chunks.push(Buffer.alloc(chunkSize, i));
+      logger.info(`Read ${i + 1}/${chunkCount} chunks`);
+      await setTimeout(readInterval);
+    }
+
+    // Phase 2: Process — hash all chunks repeatedly (high CPU, stable memory)
+    logger.info("Entering process phase", { processSeconds });
+    const processDeadline = Date.now() + processSeconds * 1000;
+    let hashCount = 0;
+
+    while (Date.now() < processDeadline) {
+      const chunkEnd = Date.now() + 100;
+      while (Date.now() < chunkEnd) {
+        for (const chunk of chunks) {
+          createHash("sha256").update(chunk).digest();
+          hashCount++;
+        }
+      }
+      await setTimeout(1);
+    }
+
+    // Phase 3: Write — release memory gradually (low CPU, memory drops)
+    logger.info("Entering write phase", { writeSeconds });
+    const writeInterval = (writeSeconds * 1000) / chunkCount;
+
+    for (let i = chunkCount - 1; i >= 0; i--) {
+      chunks.pop();
+      logger.info(`Wrote and released chunk ${chunkCount - i}/${chunkCount}`);
+      await setTimeout(writeInterval);
+    }
+
+    global.gc?.();
+    await setTimeout(5000);
+
+    logger.info("Sustained workload complete", { hashCount });
+    return { hashCount };
+  },
+});
+
+/**
+ * Parent task that fans out multiple child tasks in parallel.
+ * Useful for seeing per-run breakdowns in metrics queries grouped by run_id.
+ */
+export const concurrentLoad = task({
+  id: "concurrent-load",
+  run: async (
+    {
+      concurrency = 3,
+      taskType = "bursty-workload" as "cpu-intensive" | "bursty-workload",
+      durationSeconds = 30,
+    }: {
+      concurrency?: number;
+      taskType?: "cpu-intensive" | "bursty-workload";
+      durationSeconds?: number;
+    },
+    { ctx }
+  ) => {
+    logger.info("Starting concurrent load", { concurrency, taskType, durationSeconds });
+
+    const items = Array.from({ length: concurrency }, (_, i) => {
+      if (taskType === "cpu-intensive") {
+        return { id: cpuIntensive.id, payload: { durationSeconds } };
+      }
+      return {
+        id: burstyWorkload.id,
+        payload: { cycles: 3, burstSeconds: 5, idleSeconds: 5 },
+      };
+    });
+
+    const results = await batch.triggerAndWait<typeof cpuIntensive | typeof burstyWorkload>(items);
+
+    logger.info("All children completed", {
+      count: results.runs.length,
+    });
+
+    return { childRunIds: results.runs.map((r) => r.id) };
+  },
+});
+
+/**
+ * Demonstrates custom OTEL metrics: counter, histogram, and up-down counter.
+ * Simulates processing a queue of items with varying processing times.
+ */
+export const customMetrics = task({
+  id: "custom-metrics",
+  run: async (
+    {
+      itemCount = 20,
+      minProcessingMs = 50,
+      maxProcessingMs = 500,
+      batchSize = 5,
+    }: {
+      itemCount?: number;
+      minProcessingMs?: number;
+      maxProcessingMs?: number;
+      batchSize?: number;
+    },
+    { ctx }
+  ) => {
+    logger.info("Starting custom metrics demo", { itemCount, batchSize });
+
+    // Simulate items arriving in the queue
+    queueDepthGauge.add(itemCount);
+
+    let totalProcessed = 0;
+
+    for (let i = 0; i < itemCount; i += batchSize) {
+      const currentBatch = Math.min(batchSize, itemCount - i);
+
+      for (let j = 0; j < currentBatch; j++) {
+        const processingTime =
+          minProcessingMs + Math.random() * (maxProcessingMs - minProcessingMs);
+
+        // Simulate work
+        const start = performance.now();
+        let data = Buffer.from(`item-${i + j}`);
+        const deadline = Date.now() + processingTime;
+        while (Date.now() < deadline) {
+          data = createHash("sha256").update(data).digest();
+        }
+        const elapsed = performance.now() - start;
+
+        // Record metrics
+        itemsProcessedCounter.add(1, { "item.type": j % 2 === 0 ? "even" : "odd" });
+        itemDurationHistogram.record(elapsed, { "item.type": j % 2 === 0 ? "even" : "odd" });
+        queueDepthGauge.add(-1);
+
+        totalProcessed++;
+      }
+
+      logger.info(`Processed batch`, {
+        batchNumber: Math.floor(i / batchSize) + 1,
+        totalProcessed,
+        remaining: itemCount - totalProcessed,
+      });
+
+      // Brief pause between batches
+      await setTimeout(1000);
+    }
+
+    logger.info("Custom metrics demo complete", { totalProcessed });
+    return { totalProcessed };
+  },
+});