From 643ea3907f49a9172001f4f80fb4e2354e40f803 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Sat, 8 Aug 2026 10:09:04 +0000 Subject: [PATCH] fix(dashboard-agent): read the report's untrustworthy reason under its current name `curateReport` still read `facts.staleReason`, renamed to `untrustworthyReason` in dc3b50260 and split into telemetry_stale / telemetry_absent / flow_unmeasured. The read had been undefined since, so the agent got "untrustworthy" with no why, and the prompt still told it every such case was stale telemetry. `facts` is `z.record(z.unknown())`, so nothing typechecked the key. The new test goes presenter -> reports route JSON -> curateReport without naming a facts key on the way in, and asserts curation carries every key the presenter emits. --- apps/webapp/test/reportCurationTrust.test.ts | 81 +++++++++++++++++++ .../dashboard-agent/package.json | 1 + .../__snapshots__/prompt-prefix.test.ts.snap | 20 ++--- .../dashboard-agent/src/tool-curation.ts | 2 +- .../dashboard-agent/src/tool-schemas.ts | 2 +- 5 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 apps/webapp/test/reportCurationTrust.test.ts diff --git a/apps/webapp/test/reportCurationTrust.test.ts b/apps/webapp/test/reportCurationTrust.test.ts new file mode 100644 index 000000000..aeb3de7e0 --- /dev/null +++ b/apps/webapp/test/reportCurationTrust.test.ts @@ -0,0 +1,81 @@ +import { curateReport } from "@internal/dashboard-agent/tool-curation"; +import { describe, expect, it } from "vitest"; +import { interpret, type HealthInput } from "~/presenters/v3/reports/health/health"; +import { reportResponse } from "~/presenters/v3/reports/reportsApi.server"; + +/** + * The real chain the agent's `get_report` runs: the health presenter builds the view model, the + * reports route serializes it as `format=json`, and the agent curates that body. Nothing here + * names a `facts` key on the way in, so a producer/consumer rename fails instead of passing. + */ +async function curatedFacts(input: HealthInput): Promise> { + const body = await reportResponse(interpret(input), "json").json(); + return curateReport(body).facts as Record; +} + +const HEALTHY: HealthInput = { + scope: "prod", + period: "last 1h", + baselineLabel: "vs your 7d normal", + generatedAt: "2026-07-20T12:00:00.000Z", + windowMinutes: 60, + flowSource: "queue_metrics_v1", + pending: { now: 84, normal: 120, series: [110, 96, 88, 90, 84], estimated: false }, + startLatency: { p95Ms: 6000, normalP95Ms: 7000, series: [6500, 6200, 6000, 5900, 6000] }, + throughput: { + finishedPerMin: 1000, + completedPerMin: 1000, + triggeredPerMin: 1000, + normalTriggeredPerMin: 1000, + }, + failures: { rate: 0.009, normalRate: 0.011, series: [0.01, 0.009, 0.009] }, + duration: { p95Ms: 1100, normalP95Ms: 1180 }, + liveness: { telemetryAgeMs: 2000 }, + flowEvidence: { + runningSeries: [40, 45, 50, 48, 44], + envLimit: 100, + throttledShare: 0, + worstQueue: null, + dlqDelta: 0, + }, +}; + +describe("the agent's curated report keeps the reason its numbers can't be trusted", () => { + it("carries telemetry_stale", async () => { + const facts = await curatedFacts({ ...HEALTHY, liveness: { telemetryAgeMs: 30 * 60_000 } }); + expect(facts.trustworthy).toBe(false); + expect(facts.untrustworthyReason).toBe("telemetry_stale"); + }); + + it("carries telemetry_absent", async () => { + const facts = await curatedFacts({ ...HEALTHY, liveness: { telemetryAgeMs: null } }); + expect(facts.trustworthy).toBe(false); + expect(facts.untrustworthyReason).toBe("telemetry_absent"); + }); + + it("carries flow_unmeasured", async () => { + const facts = await curatedFacts({ + ...HEALTHY, + pending: { now: 0, series: [], estimated: true, availability: "unknown" }, + }); + expect(facts.trustworthy).toBe(false); + expect(facts.untrustworthyReason).toBe("flow_unmeasured"); + }); + + it("states no reason on a trustworthy report", async () => { + const facts = await curatedFacts(HEALTHY); + expect(facts.trustworthy).toBe(true); + expect(facts.untrustworthyReason).toBeUndefined(); + }); + + it("carries every fact key the presenter emits, so the next rename fails here", async () => { + const emitted = interpret(HEALTHY).facts; + const curated = await curatedFacts(HEALTHY); + // `telemetry` is deliberately dropped: `trustworthy` + the reason already say what the agent acts on. + const dropped = new Set(["telemetry"]); + for (const key of Object.keys(emitted)) { + if (dropped.has(key)) continue; + expect(Object.keys(curated)).toContain(key); + } + }); +}); diff --git a/internal-packages/dashboard-agent/package.json b/internal-packages/dashboard-agent/package.json index b0b95d895..2d89391e2 100644 --- a/internal-packages/dashboard-agent/package.json +++ b/internal-packages/dashboard-agent/package.json @@ -7,6 +7,7 @@ "type": "module", "exports": { ".": "./src/index.ts", + "./tool-curation": "./src/tool-curation.ts", "./tool-schemas": "./src/tool-schemas.ts", "./prompt-prefix": "./src/prompt-prefix.ts" }, diff --git a/internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap b/internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap index eae43421f..293c37efd 100644 --- a/internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap +++ b/internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap @@ -4,8 +4,8 @@ exports[`the prefix stays inside its budget > matches the committed measurement { "assistant": { "prompt": { - "chars": 19266, - "estimatedTokens": 4817, + "chars": 19285, + "estimatedTokens": 4821, }, "tools": { "chars": 38555, @@ -13,15 +13,15 @@ exports[`the prefix stays inside its budget > matches the committed measurement "estimatedTokens": 9639, }, "total": { - "chars": 57822, - "estimatedTokens": 14456, - "fingerprint": "f6751f8f", + "chars": 57841, + "estimatedTokens": 14460, + "fingerprint": "52bee164", }, }, "code": { "prompt": { - "chars": 22021, - "estimatedTokens": 5505, + "chars": 22040, + "estimatedTokens": 5510, }, "tools": { "chars": 41564, @@ -29,9 +29,9 @@ exports[`the prefix stays inside its budget > matches the committed measurement "estimatedTokens": 10391, }, "total": { - "chars": 63586, - "estimatedTokens": 15897, - "fingerprint": "6432d8b0", + "chars": 63605, + "estimatedTokens": 15901, + "fingerprint": "037bcfc3", }, }, } diff --git a/internal-packages/dashboard-agent/src/tool-curation.ts b/internal-packages/dashboard-agent/src/tool-curation.ts index 6510ce73f..785a5f511 100644 --- a/internal-packages/dashboard-agent/src/tool-curation.ts +++ b/internal-packages/dashboard-agent/src/tool-curation.ts @@ -189,7 +189,7 @@ export function curateReport(data: unknown) { })), facts: { trustworthy: facts.trustworthy, - staleReason: facts.staleReason, + untrustworthyReason: facts.untrustworthyReason, flowSource: facts.flowSource, pendingEstimated: facts.pendingEstimated, throughput: facts.throughput, diff --git a/internal-packages/dashboard-agent/src/tool-schemas.ts b/internal-packages/dashboard-agent/src/tool-schemas.ts index f0af2fde7..b0f47db1b 100644 --- a/internal-packages/dashboard-agent/src/tool-schemas.ts +++ b/internal-packages/dashboard-agent/src/tool-schemas.ts @@ -458,7 +458,7 @@ Knowing where the user is, and taking them places: Is anything wrong?: - For "is anything wrong", "how is prod doing", "is everything healthy", start with get_report. It grades flow, execution, and liveness together, which is a better first answer than any single query. -- If the report's facts.trustworthy is false, the underlying telemetry is stale: say the data can't be trusted right now and what would confirm it. Do NOT diagnose a cause or recommend an action off untrusted numbers. +- If the report's facts.trustworthy is false, say why from facts.untrustworthyReason (telemetry_stale, telemetry_absent or flow_unmeasured) and what would confirm it. Do NOT diagnose a cause or recommend an action off untrusted numbers. - When the report points at flow (runs not starting), follow up with get_queue on the queue it names to see depth, wait time, and throttling. When it points at execution, follow up with list_errors / get_run_trace. - When something started failing at a particular time, check list_deploys for a deploy in that window, and correlate_version on a failing run to see the exact commit and pull request it ran.