feat(supervisor): add prometheus metric for outbound http requests (#4350)

Adds Prometheus metrics so the supervisor's outbound HTTP calls are
observable - including client-side failures that previously only
surfaced as a log line.

- `supervisor_outbound_request_total{name, method, status, outcome}` -
counts every outbound request. `outcome` separates a transport failure
(`network_error`), an HTTP error response (`http_error`), a response
that failed schema validation (`invalid_response`), and success (`ok`).
- `supervisor_outbound_request_duration_seconds{name, outcome}` -
latency histogram. Leaner labels than the counter (no `status`) to avoid
bucket×label cardinality; buckets match the existing dequeue-latency
histogram since these calls share the same retrying HTTP client and
long-poll envelope.

Coverage:
- The warm-start request (a one-off `fetch`) - instrumented inline; the
response status code is now also included in the failure log (it was
previously dropped).
- All worker API client calls (`SupervisorHttpClient`: dequeue, run
attempt start/complete, heartbeats, snapshots, continue, suspend,
debug-log, connect) - routed through a single instrumented `request()`
helper that reports via an optional `onHttpRequestComplete` callback on
the client, which the supervisor wires into the counter + histogram.

Low cardinality by design: `name` is a **static per-endpoint label**
(e.g. `dequeue`, `start_run_attempt`), never the interpolated URL - so
no run/snapshot IDs land in labels, mirroring the templated `route`
labels on the inbound HTTP server.

Registered on the existing metrics registry, exposed on `/metrics` with
no new wiring. Internal-only change (no package release needed), so the
changelog note is a single `.server-changes` entry.

.server-changes/supervisor-outbound-request-metrics.md

@@ -0,0 +1,6 @@
+---
+area: supervisor
+type: improvement
+---
+
+Improved supervisor observability: it now reports metrics for its outbound requests, making failed calls to upstream services easier to monitor.

apps/supervisor/src/index.ts

@@ -22,7 +22,7 @@ import {
   isKubernetesEnvironment,
 } from "@trigger.dev/core/v3/serverOnly";
 import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js";
-import { collectDefaultMetrics, Gauge, Histogram } from "prom-client";
+import { collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client";
 import { register } from "./metrics.js";
 import { PodCleaner } from "./services/podCleaner.js";
 import { FailedPodHandler } from "./services/failedPodHandler.js";
@@ -60,6 +60,21 @@ const workloadCreateDuration = new Histogram({
   registers: [register],
 });
 
+const outboundRequestsTotal = new Counter({
+  name: "supervisor_outbound_request_total",
+  help: "Count of outbound HTTP requests from the supervisor, by target name, method, response status, and outcome (ok, http_error, invalid_response, network_error).",
+  labelNames: ["name", "method", "status", "outcome"],
+  registers: [register],
+});
+
+const outboundRequestDuration = new Histogram({
+  name: "supervisor_outbound_request_duration_seconds",
+  help: "Duration of outbound HTTP requests from the supervisor, by target name and outcome. Includes the HTTP client's internal retries and backoff.",
+  labelNames: ["name", "outcome"],
+  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 11, 12.5, 15, 20, 30, 60],
+  registers: [register],
+});
+
 class ManagedSupervisor {
   private readonly workerSession: SupervisorSession;
   private readonly metricsServer?: HttpServer;
@@ -322,6 +337,10 @@ class ManagedSupervisor {
       runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED,
       heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS,
       sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS,
+      onHttpRequestComplete: ({ name, method, status, outcome, durationMs }) => {
+        outboundRequestsTotal.inc({ name, method, status, outcome });
+        outboundRequestDuration.observe({ name, outcome }, durationMs / 1000);
+      },
       preDequeue: async () => {
         // Synchronous, hot-path-safe cached read; false when no monitors are active.
         const skipForBackpressure = this.backpressureMonitors.some((m) => m.shouldSkipDequeue());
@@ -692,6 +711,18 @@ class ManagedSupervisor {
       headers.traceparent = traceparent;
     }
 
+    const requestStart = performance.now();
+    const record = (
+      status: string,
+      outcome: "ok" | "http_error" | "invalid_response" | "network_error"
+    ) => {
+      outboundRequestsTotal.inc({ name: "warm_start", method: "POST", status, outcome });
+      outboundRequestDuration.observe(
+        { name: "warm_start", outcome },
+        (performance.now() - requestStart) / 1000
+      );
+    };
+
     try {
       const res = await fetch(warmStartUrlWithPath.href, {
         method: "POST",
@@ -700,8 +731,10 @@ class ManagedSupervisor {
       });
 
       if (!res.ok) {
+        record(String(res.status), "http_error");
         this.logger.error("Warm start failed", {
           runId: dequeuedMessage.run.id,
+          statusCode: res.status,
         });
         return false;
       }
@@ -710,6 +743,7 @@ class ManagedSupervisor {
       const parsedData = z.object({ didWarmStart: z.boolean() }).safeParse(data);
 
       if (!parsedData.success) {
+        record(String(res.status), "invalid_response");
         this.logger.error("Warm start response invalid", {
           runId: dequeuedMessage.run.id,
           data,
@@ -717,8 +751,11 @@ class ManagedSupervisor {
         return false;
       }
 
+      record(String(res.status), "ok");
+
       return parsedData.data.didWarmStart;
     } catch (error) {
+      record("none", "network_error");
       this.logger.error("Warm start error", {
         runId: dequeuedMessage.run.id,
         error,

packages/core/src/v3/runEngineWorker/supervisor/http.ts

@@ -21,9 +21,9 @@ import {
   WorkerApiSuspendRunResponseBody,
   WorkerApiRunSnapshotsSinceResponseBody,
 } from "./schemas.js";
-import type { SupervisorClientCommonOptions } from "./types.js";
+import type { SupervisorClientCommonOptions, SupervisorHttpRequestMetric } from "./types.js";
 import { getDefaultWorkerHeaders } from "./util.js";
-import { wrapZodFetch } from "../../zodfetch.js";
+import { wrapZodFetch, type ApiResult, type ZodFetchOptions } from "../../zodfetch.js";
 import { createHeaders } from "../util.js";
 import { WORKER_HEADERS } from "../consts.js";
 import { SimpleStructuredLogger } from "../../utils/structuredLogger.js";
@@ -36,6 +36,7 @@ export class SupervisorHttpClient {
   private readonly instanceName: string;
   private readonly defaultHeaders: Record<string, string>;
   private readonly sendRunDebugLogs: boolean;
+  private readonly onHttpRequestComplete?: (metric: SupervisorHttpRequestMetric) => void;
 
   private readonly logger = new SimpleStructuredLogger("supervisor-http-client");
 
@@ -45,6 +46,7 @@ export class SupervisorHttpClient {
     this.instanceName = opts.instanceName;
     this.defaultHeaders = getDefaultWorkerHeaders(opts);
     this.sendRunDebugLogs = opts.sendRunDebugLogs ?? false;
+    this.onHttpRequestComplete = opts.onHttpRequestComplete;
 
     if (!this.apiUrl) {
       throw new Error("apiURL is required and needs to be a non-empty string");
@@ -59,8 +61,55 @@ export class SupervisorHttpClient {
     }
   }
 
+  private async request<T extends z.ZodTypeAny>(
+    name: string,
+    schema: T,
+    url: string,
+    requestInit?: RequestInit,
+    options?: ZodFetchOptions<z.output<T>>
+  ): Promise<ApiResult<z.infer<T>>> {
+    const start = performance.now();
+    const result = await wrapZodFetch(schema, url, requestInit, options);
+
+    if (this.onHttpRequestComplete) {
+      const durationMs = performance.now() - start;
+      const method = requestInit?.method ?? "GET";
+
+      if (result.success) {
+        this.onHttpRequestComplete({ name, method, status: "2xx", outcome: "ok", durationMs });
+      } else if (result.statusCode === 200) {
+        this.onHttpRequestComplete({
+          name,
+          method,
+          status: "200",
+          outcome: "invalid_response",
+          durationMs,
+        });
+      } else if (typeof result.statusCode === "number") {
+        this.onHttpRequestComplete({
+          name,
+          method,
+          status: String(result.statusCode),
+          outcome: "http_error",
+          durationMs,
+        });
+      } else {
+        this.onHttpRequestComplete({
+          name,
+          method,
+          status: "none",
+          outcome: "network_error",
+          durationMs,
+        });
+      }
+    }
+
+    return result;
+  }
+
   async connect(body: WorkerApiConnectRequestBody) {
-    return wrapZodFetch(
+    return this.request(
+      "connect",
       WorkerApiConnectResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/connect`,
       {
@@ -75,7 +124,8 @@ export class SupervisorHttpClient {
   }
 
   async dequeue(body: WorkerApiDequeueRequestBody) {
-    return wrapZodFetch(
+    return this.request(
+      "dequeue",
       WorkerApiDequeueResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/dequeue`,
       {
@@ -91,7 +141,8 @@ export class SupervisorHttpClient {
 
   /** @deprecated Not currently used */
   async dequeueFromVersion(deploymentId: string, maxRunCount = 1, runnerId?: string) {
-    return wrapZodFetch(
+    return this.request(
+      "dequeue_from_version",
       WorkerApiDequeueResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/deployments/${deploymentId}/dequeue?maxRunCount=${maxRunCount}`,
       {
@@ -104,7 +155,8 @@ export class SupervisorHttpClient {
   }
 
   async heartbeatWorker(body: WorkerApiHeartbeatRequestBody) {
-    return wrapZodFetch(
+    return this.request(
+      "heartbeat_worker",
       WorkerApiHeartbeatResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/heartbeat`,
       {
@@ -124,7 +176,8 @@ export class SupervisorHttpClient {
     body: WorkerApiRunHeartbeatRequestBody,
     runnerId?: string
   ) {
-    return wrapZodFetch(
+    return this.request(
+      "heartbeat_run",
       WorkerApiRunHeartbeatResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/heartbeat`,
       {
@@ -146,7 +199,8 @@ export class SupervisorHttpClient {
     runnerId?: string,
     environmentId?: string
   ) {
-    return wrapZodFetch(
+    return this.request(
+      "start_run_attempt",
       WorkerApiRunAttemptStartResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/attempts/start`,
       {
@@ -168,7 +222,8 @@ export class SupervisorHttpClient {
     runnerId?: string,
     environmentId?: string
   ) {
-    return wrapZodFetch(
+    return this.request(
+      "complete_run_attempt",
       WorkerApiRunAttemptCompleteResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/attempts/complete`,
       {
@@ -184,7 +239,8 @@ export class SupervisorHttpClient {
   }
 
   async getLatestSnapshot(runId: string, runnerId?: string, environmentId?: string) {
-    return wrapZodFetch(
+    return this.request(
+      "get_latest_snapshot",
       WorkerApiRunLatestSnapshotResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/latest`,
       {
@@ -204,7 +260,8 @@ export class SupervisorHttpClient {
     runnerId?: string,
     environmentId?: string
   ) {
-    return wrapZodFetch(
+    return this.request(
+      "get_snapshots_since",
       WorkerApiRunSnapshotsSinceResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/since/${snapshotId}`,
       {
@@ -224,7 +281,8 @@ export class SupervisorHttpClient {
     }
 
     try {
-      const res = await wrapZodFetch(
+      const res = await this.request(
+        "send_debug_log",
         z.unknown(),
         `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/logs/debug`,
         {
@@ -252,7 +310,8 @@ export class SupervisorHttpClient {
     runnerId?: string,
     environmentId?: string
   ) {
-    return wrapZodFetch(
+    return this.request(
+      "continue_run_execution",
       WorkerApiContinueRunExecutionRequestBody,
       `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/continue`,
       {
@@ -292,7 +351,8 @@ export class SupervisorHttpClient {
     runnerId?: string;
     body: WorkerApiSuspendRunRequestBody;
   }) {
-    return wrapZodFetch(
+    return this.request(
+      "submit_suspend_completion",
       WorkerApiSuspendRunResponseBody,
       `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/suspend`,
       {

packages/core/src/v3/runEngineWorker/supervisor/types.ts

@@ -1,5 +1,13 @@
 import type { MachineResources } from "../../schemas/runEngine.js";
 
+export type SupervisorHttpRequestMetric = {
+  name: string;
+  method: string;
+  status: string;
+  outcome: "ok" | "http_error" | "invalid_response" | "network_error";
+  durationMs: number;
+};
+
 export type SupervisorClientCommonOptions = {
   apiUrl: string;
   workerToken: string;
@@ -7,6 +15,7 @@ export type SupervisorClientCommonOptions = {
   deploymentId?: string;
   managedWorkerSecret?: string;
   sendRunDebugLogs?: boolean;
+  onHttpRequestComplete?: (metric: SupervisorHttpRequestMetric) => void;
 };
 
 export type PreDequeueFn = () => Promise<{