fix(dashboard-agent): read the report's untrustworthy reason under its current name
`curateReport` still read `facts.staleReason`, renamed to `untrustworthyReason`
in dc3b50260 and split into telemetry_stale / telemetry_absent / flow_unmeasured.
The read had been undefined since, so the agent got "untrustworthy" with no why,
and the prompt still told it every such case was stale telemetry.
`facts` is `z.record(z.unknown())`, so nothing typechecked the key. The new test
goes presenter -> reports route JSON -> curateReport without naming a facts key
on the way in, and asserts curation carries every key the presenter emits.
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
import { curateReport } from "@internal/dashboard-agent/tool-curation";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { interpret, type HealthInput } from "~/presenters/v3/reports/health/health";
|
||||
import { reportResponse } from "~/presenters/v3/reports/reportsApi.server";
|
||||
|
||||
/**
|
||||
* The real chain the agent's `get_report` runs: the health presenter builds the view model, the
|
||||
* reports route serializes it as `format=json`, and the agent curates that body. Nothing here
|
||||
* names a `facts` key on the way in, so a producer/consumer rename fails instead of passing.
|
||||
*/
|
||||
async function curatedFacts(input: HealthInput): Promise<Record<string, unknown>> {
|
||||
const body = await reportResponse(interpret(input), "json").json();
|
||||
return curateReport(body).facts as Record<string, unknown>;
|
||||
}
|
||||
|
||||
const HEALTHY: HealthInput = {
|
||||
scope: "prod",
|
||||
period: "last 1h",
|
||||
baselineLabel: "vs your 7d normal",
|
||||
generatedAt: "2026-07-20T12:00:00.000Z",
|
||||
windowMinutes: 60,
|
||||
flowSource: "queue_metrics_v1",
|
||||
pending: { now: 84, normal: 120, series: [110, 96, 88, 90, 84], estimated: false },
|
||||
startLatency: { p95Ms: 6000, normalP95Ms: 7000, series: [6500, 6200, 6000, 5900, 6000] },
|
||||
throughput: {
|
||||
finishedPerMin: 1000,
|
||||
completedPerMin: 1000,
|
||||
triggeredPerMin: 1000,
|
||||
normalTriggeredPerMin: 1000,
|
||||
},
|
||||
failures: { rate: 0.009, normalRate: 0.011, series: [0.01, 0.009, 0.009] },
|
||||
duration: { p95Ms: 1100, normalP95Ms: 1180 },
|
||||
liveness: { telemetryAgeMs: 2000 },
|
||||
flowEvidence: {
|
||||
runningSeries: [40, 45, 50, 48, 44],
|
||||
envLimit: 100,
|
||||
throttledShare: 0,
|
||||
worstQueue: null,
|
||||
dlqDelta: 0,
|
||||
},
|
||||
};
|
||||
|
||||
describe("the agent's curated report keeps the reason its numbers can't be trusted", () => {
|
||||
it("carries telemetry_stale", async () => {
|
||||
const facts = await curatedFacts({ ...HEALTHY, liveness: { telemetryAgeMs: 30 * 60_000 } });
|
||||
expect(facts.trustworthy).toBe(false);
|
||||
expect(facts.untrustworthyReason).toBe("telemetry_stale");
|
||||
});
|
||||
|
||||
it("carries telemetry_absent", async () => {
|
||||
const facts = await curatedFacts({ ...HEALTHY, liveness: { telemetryAgeMs: null } });
|
||||
expect(facts.trustworthy).toBe(false);
|
||||
expect(facts.untrustworthyReason).toBe("telemetry_absent");
|
||||
});
|
||||
|
||||
it("carries flow_unmeasured", async () => {
|
||||
const facts = await curatedFacts({
|
||||
...HEALTHY,
|
||||
pending: { now: 0, series: [], estimated: true, availability: "unknown" },
|
||||
});
|
||||
expect(facts.trustworthy).toBe(false);
|
||||
expect(facts.untrustworthyReason).toBe("flow_unmeasured");
|
||||
});
|
||||
|
||||
it("states no reason on a trustworthy report", async () => {
|
||||
const facts = await curatedFacts(HEALTHY);
|
||||
expect(facts.trustworthy).toBe(true);
|
||||
expect(facts.untrustworthyReason).toBeUndefined();
|
||||
});
|
||||
|
||||
it("carries every fact key the presenter emits, so the next rename fails here", async () => {
|
||||
const emitted = interpret(HEALTHY).facts;
|
||||
const curated = await curatedFacts(HEALTHY);
|
||||
// `telemetry` is deliberately dropped: `trustworthy` + the reason already say what the agent acts on.
|
||||
const dropped = new Set(["telemetry"]);
|
||||
for (const key of Object.keys(emitted)) {
|
||||
if (dropped.has(key)) continue;
|
||||
expect(Object.keys(curated)).toContain(key);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./tool-curation": "./src/tool-curation.ts",
|
||||
"./tool-schemas": "./src/tool-schemas.ts",
|
||||
"./prompt-prefix": "./src/prompt-prefix.ts"
|
||||
},
|
||||
|
||||
@@ -4,8 +4,8 @@ exports[`the prefix stays inside its budget > matches the committed measurement
|
||||
{
|
||||
"assistant": {
|
||||
"prompt": {
|
||||
"chars": 19266,
|
||||
"estimatedTokens": 4817,
|
||||
"chars": 19285,
|
||||
"estimatedTokens": 4821,
|
||||
},
|
||||
"tools": {
|
||||
"chars": 38555,
|
||||
@@ -13,15 +13,15 @@ exports[`the prefix stays inside its budget > matches the committed measurement
|
||||
"estimatedTokens": 9639,
|
||||
},
|
||||
"total": {
|
||||
"chars": 57822,
|
||||
"estimatedTokens": 14456,
|
||||
"fingerprint": "f6751f8f",
|
||||
"chars": 57841,
|
||||
"estimatedTokens": 14460,
|
||||
"fingerprint": "52bee164",
|
||||
},
|
||||
},
|
||||
"code": {
|
||||
"prompt": {
|
||||
"chars": 22021,
|
||||
"estimatedTokens": 5505,
|
||||
"chars": 22040,
|
||||
"estimatedTokens": 5510,
|
||||
},
|
||||
"tools": {
|
||||
"chars": 41564,
|
||||
@@ -29,9 +29,9 @@ exports[`the prefix stays inside its budget > matches the committed measurement
|
||||
"estimatedTokens": 10391,
|
||||
},
|
||||
"total": {
|
||||
"chars": 63586,
|
||||
"estimatedTokens": 15897,
|
||||
"fingerprint": "6432d8b0",
|
||||
"chars": 63605,
|
||||
"estimatedTokens": 15901,
|
||||
"fingerprint": "037bcfc3",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -189,7 +189,7 @@ export function curateReport(data: unknown) {
|
||||
})),
|
||||
facts: {
|
||||
trustworthy: facts.trustworthy,
|
||||
staleReason: facts.staleReason,
|
||||
untrustworthyReason: facts.untrustworthyReason,
|
||||
flowSource: facts.flowSource,
|
||||
pendingEstimated: facts.pendingEstimated,
|
||||
throughput: facts.throughput,
|
||||
|
||||
@@ -458,7 +458,7 @@ Knowing where the user is, and taking them places:
|
||||
|
||||
Is anything wrong?:
|
||||
- For "is anything wrong", "how is prod doing", "is everything healthy", start with get_report. It grades flow, execution, and liveness together, which is a better first answer than any single query.
|
||||
- If the report's facts.trustworthy is false, the underlying telemetry is stale: say the data can't be trusted right now and what would confirm it. Do NOT diagnose a cause or recommend an action off untrusted numbers.
|
||||
- If the report's facts.trustworthy is false, say why from facts.untrustworthyReason (telemetry_stale, telemetry_absent or flow_unmeasured) and what would confirm it. Do NOT diagnose a cause or recommend an action off untrusted numbers.
|
||||
- When the report points at flow (runs not starting), follow up with get_queue on the queue it names to see depth, wait time, and throttling. When it points at execution, follow up with list_errors / get_run_trace.
|
||||
- When something started failing at a particular time, check list_deploys for a deploy in that window, and correlate_version on a failing run to see the exact commit and pull request it ran.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user