feat: expose verifier evidence via fleet receipts API
Add three new read-only endpoints to the Runtime API under the fleet
run resource:
GET /v1/fleet/runs/{run_id}/receipts
Lists all durable receipts for every completed task in the run.
Each entry includes: run_id, task_id, worker_id, attempt,
terminal_seq, completed_at, result, failure_kind, failure_class
(human-readable), retry_eligible, score, artifacts summary, and
evidence_available flag.
GET /v1/fleet/runs/{run_id}/receipts/{task_id}
Returns the same receipt detail for a single task. Returns 404
when the run or task has no durable receipt yet.
GET /v1/fleet/runs/{run_id}/receipts/{task_id}/evidence
Reads the Receipt-kind artifact file (bounded to 64 KiB) and
returns its structured JSON content alongside path, checksum,
size_bytes, and a truncated flag. Returns 404 when no receipt
artifact exists or the file is not yet written.
Key design points:
- retry_eligible is true only for transport failures; verifier and
task failures require human review or a code change.
- failure_class provides a plain-English description of each
FleetTaskFailureKind so managed clients can explain a failure
without hard-coding enum labels.
- Evidence is kept behind an explicit inspection endpoint and capped
at MAX_RECEIPT_EVIDENCE_READ_BYTES (64 KiB); raw artifact paths are
exposed for off-band retrieval.
- All three endpoints are read-only; no action (rerun/cancel) is
wired here — those delegate to the existing execution owner.
Tests added:
- fleet_receipt_json_pass_result_has_no_failure_fields
- fleet_receipt_json_verifier_failure_is_not_retry_eligible
- fleet_receipt_json_transport_failure_is_retry_eligible
- fleet_receipt_json_receipt_artifact_sets_evidence_available
- fleet_receipt_api_list_and_get_round_trip (integration: list,
get, evidence, and 404 for missing task)
Closes #5073
This commit is contained in:
copilot-swe-agent[bot]
committed by
Hmbown
Hmbown
parent
01c96a419b
commit
10723dc8e6
@@ -714,6 +714,18 @@ pub fn build_router(state: RuntimeApiState) -> Router {
|
||||
get(replay_fleet_events),
|
||||
)
|
||||
.route("/v1/fleet/runs/{run_id}/stop", post(stop_fleet_run))
|
||||
.route(
|
||||
"/v1/fleet/runs/{run_id}/receipts",
|
||||
get(list_fleet_run_receipts),
|
||||
)
|
||||
.route(
|
||||
"/v1/fleet/runs/{run_id}/receipts/{task_id}",
|
||||
get(get_fleet_run_receipt),
|
||||
)
|
||||
.route(
|
||||
"/v1/fleet/runs/{run_id}/receipts/{task_id}/evidence",
|
||||
get(inspect_fleet_run_receipt_evidence),
|
||||
)
|
||||
.route("/v1/fleet/workers/{worker_id}", get(get_fleet_worker))
|
||||
.route(
|
||||
"/v1/fleet/workers/{worker_id}/interrupt",
|
||||
@@ -1701,6 +1713,117 @@ async fn stop_fleet_run(
|
||||
})))
|
||||
}
|
||||
|
||||
/// Maximum bytes read from a receipt evidence file for the inspection endpoint.
|
||||
const MAX_RECEIPT_EVIDENCE_READ_BYTES: u64 = 65_536;
|
||||
|
||||
async fn list_fleet_run_receipts(
|
||||
State(state): State<RuntimeApiState>,
|
||||
Path(run_id): Path<String>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let manager = open_fleet_manager(&state)?;
|
||||
let ledger_state = manager
|
||||
.rebuild_state()
|
||||
.map_err(|err| ApiError::internal(format!("Failed to rebuild fleet state: {err}")))?;
|
||||
if !ledger_state.runs.contains_key(&run_id) {
|
||||
return Err(ApiError::not_found(format!(
|
||||
"fleet run '{run_id}' not found"
|
||||
)));
|
||||
}
|
||||
let run_id_parsed = FleetRunId::from(run_id.clone());
|
||||
let receipts: Vec<Value> = ledger_state
|
||||
.receipts
|
||||
.values()
|
||||
.filter(|r| r.run_id == run_id_parsed)
|
||||
.map(fleet_receipt_json)
|
||||
.collect();
|
||||
Ok(Json(json!({
|
||||
"run_id": run_id,
|
||||
"receipts": receipts,
|
||||
})))
|
||||
}
|
||||
|
||||
async fn get_fleet_run_receipt(
|
||||
State(state): State<RuntimeApiState>,
|
||||
Path((run_id, task_id)): Path<(String, String)>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let manager = open_fleet_manager(&state)?;
|
||||
let ledger_state = manager
|
||||
.rebuild_state()
|
||||
.map_err(|err| ApiError::internal(format!("Failed to rebuild fleet state: {err}")))?;
|
||||
let key = format!("{run_id}:{task_id}");
|
||||
let receipt = ledger_state
|
||||
.receipts
|
||||
.get(&key)
|
||||
.ok_or_else(|| {
|
||||
ApiError::not_found(format!(
|
||||
"no receipt found for run '{run_id}' task '{task_id}'"
|
||||
))
|
||||
})?;
|
||||
Ok(Json(fleet_receipt_json(receipt)))
|
||||
}
|
||||
|
||||
async fn inspect_fleet_run_receipt_evidence(
|
||||
State(state): State<RuntimeApiState>,
|
||||
Path((run_id, task_id)): Path<(String, String)>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let manager = open_fleet_manager(&state)?;
|
||||
let ledger_state = manager
|
||||
.rebuild_state()
|
||||
.map_err(|err| ApiError::internal(format!("Failed to rebuild fleet state: {err}")))?;
|
||||
let key = format!("{run_id}:{task_id}");
|
||||
let receipt = ledger_state
|
||||
.receipts
|
||||
.get(&key)
|
||||
.ok_or_else(|| {
|
||||
ApiError::not_found(format!(
|
||||
"no receipt found for run '{run_id}' task '{task_id}'"
|
||||
))
|
||||
})?;
|
||||
// Locate the most recent Receipt-kind artifact.
|
||||
let receipt_artifact = receipt
|
||||
.artifacts
|
||||
.iter()
|
||||
.rfind(|a| a.kind == FleetArtifactKind::Receipt)
|
||||
.ok_or_else(|| {
|
||||
ApiError::not_found(format!(
|
||||
"no verifier evidence file for run '{run_id}' task '{task_id}'"
|
||||
))
|
||||
})?;
|
||||
let abs_path = state.workspace.join(&receipt_artifact.path);
|
||||
let metadata = std::fs::metadata(&abs_path).map_err(|err| {
|
||||
ApiError::not_found(format!(
|
||||
"evidence file not readable for run '{run_id}' task '{task_id}': {err}"
|
||||
))
|
||||
})?;
|
||||
let size_bytes = metadata.len();
|
||||
let truncated = size_bytes > MAX_RECEIPT_EVIDENCE_READ_BYTES;
|
||||
let raw = {
|
||||
use std::io::Read;
|
||||
let file = std::fs::File::open(&abs_path).map_err(|err| {
|
||||
ApiError::internal(format!("Failed to open evidence file: {err}"))
|
||||
})?;
|
||||
let mut buf = Vec::new();
|
||||
file.take(MAX_RECEIPT_EVIDENCE_READ_BYTES)
|
||||
.read_to_end(&mut buf)
|
||||
.map_err(|err| ApiError::internal(format!("Failed to read evidence file: {err}")))?;
|
||||
buf
|
||||
};
|
||||
// Parse as JSON if possible; fall back to a raw string representation.
|
||||
let content: Value = serde_json::from_slice(&raw).unwrap_or_else(|_| {
|
||||
Value::String(String::from_utf8_lossy(&raw).into_owned())
|
||||
});
|
||||
Ok(Json(json!({
|
||||
"run_id": run_id,
|
||||
"task_id": task_id,
|
||||
"path": receipt_artifact.path,
|
||||
"checksum": receipt_artifact.checksum,
|
||||
"size_bytes": size_bytes,
|
||||
"truncated": truncated,
|
||||
"content": content,
|
||||
})))
|
||||
}
|
||||
|
||||
|
||||
fn open_fleet_manager(state: &RuntimeApiState) -> Result<FleetManager, ApiError> {
|
||||
let (exec_config, fleet_config, session_model, route_config) = {
|
||||
let config = state.config.read();
|
||||
@@ -1852,6 +1975,64 @@ fn fleet_artifact_json(artifact: &codewhale_protocol::fleet::FleetArtifactRef) -
|
||||
})
|
||||
}
|
||||
|
||||
fn fleet_receipt_json(receipt: &codewhale_protocol::fleet::FleetReceipt) -> Value {
|
||||
use codewhale_protocol::fleet::{FleetTaskFailureKind, FleetTaskResult};
|
||||
|
||||
let result_label = match receipt.result {
|
||||
FleetTaskResult::Pass => "pass",
|
||||
FleetTaskResult::Partial => "partial",
|
||||
FleetTaskResult::Fail => "fail",
|
||||
FleetTaskResult::Skip => "skip",
|
||||
FleetTaskResult::Timeout => "timeout",
|
||||
};
|
||||
let (failure_kind_label, failure_class, retry_eligible) =
|
||||
match receipt.failure_kind.as_ref() {
|
||||
Some(FleetTaskFailureKind::Transport) => (
|
||||
Some("transport"),
|
||||
Some("Infrastructure or network failure during task transport"),
|
||||
true,
|
||||
),
|
||||
Some(FleetTaskFailureKind::Task) => (
|
||||
Some("task"),
|
||||
Some("Task logic exited unsuccessfully"),
|
||||
false,
|
||||
),
|
||||
Some(FleetTaskFailureKind::Verifier) => (
|
||||
Some("verifier"),
|
||||
Some("Verifier rejected the task output; manual review or code change required"),
|
||||
false,
|
||||
),
|
||||
None => (None, None, false),
|
||||
};
|
||||
let evidence_available = receipt
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|a| a.kind == FleetArtifactKind::Receipt);
|
||||
let score_json = receipt.score.as_ref().map(|s| {
|
||||
json!({
|
||||
"value": s.value,
|
||||
"max": s.max,
|
||||
"notes": s.notes,
|
||||
})
|
||||
});
|
||||
json!({
|
||||
"run_id": receipt.run_id.0.clone(),
|
||||
"task_id": receipt.task_id.clone(),
|
||||
"worker_id": receipt.worker_id.clone(),
|
||||
"attempt": receipt.attempt,
|
||||
"terminal_seq": receipt.terminal_seq,
|
||||
"completed_at": receipt.completed_at.clone(),
|
||||
"result": result_label,
|
||||
"failure_kind": failure_kind_label,
|
||||
"failure_class": failure_class,
|
||||
"retry_eligible": retry_eligible,
|
||||
"score": score_json,
|
||||
"artifacts": receipt.artifacts.iter().map(fleet_artifact_json).collect::<Vec<_>>(),
|
||||
"evidence_available": evidence_available,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
fn fleet_event_json(event: &codewhale_protocol::fleet::FleetWorkerEvent) -> Value {
|
||||
json!({
|
||||
"seq": event.seq,
|
||||
|
||||
@@ -7414,3 +7414,298 @@ async fn cors_layer_advertises_exact_supported_headers_and_never_an_extra() -> R
|
||||
handle.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fleet_receipt_json_pass_result_has_no_failure_fields() {
|
||||
use codewhale_protocol::fleet::{FleetReceipt, FleetRunId, FleetTaskResult};
|
||||
let receipt = FleetReceipt {
|
||||
run_id: FleetRunId::from("run-1"),
|
||||
task_id: "task-a".to_string(),
|
||||
worker_id: "worker-1".to_string(),
|
||||
attempt: Some(1),
|
||||
terminal_seq: Some(42),
|
||||
completed_at: "2025-01-01T00:00:00Z".to_string(),
|
||||
result: FleetTaskResult::Pass,
|
||||
failure_kind: None,
|
||||
artifacts: Vec::new(),
|
||||
score: None,
|
||||
resolved_route: None,
|
||||
effective_permissions: None,
|
||||
};
|
||||
let value = fleet_receipt_json(&receipt);
|
||||
assert_eq!(value["run_id"], "run-1");
|
||||
assert_eq!(value["task_id"], "task-a");
|
||||
assert_eq!(value["worker_id"], "worker-1");
|
||||
assert_eq!(value["attempt"], 1);
|
||||
assert_eq!(value["terminal_seq"], 42);
|
||||
assert_eq!(value["result"], "pass");
|
||||
assert!(value["failure_kind"].is_null());
|
||||
assert!(value["failure_class"].is_null());
|
||||
assert_eq!(value["retry_eligible"], false);
|
||||
assert_eq!(value["evidence_available"], false);
|
||||
assert!(value["score"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fleet_receipt_json_verifier_failure_is_not_retry_eligible() {
|
||||
use codewhale_protocol::fleet::{
|
||||
FleetReceipt, FleetRunId, FleetTaskFailureKind, FleetTaskResult,
|
||||
};
|
||||
let receipt = FleetReceipt {
|
||||
run_id: FleetRunId::from("run-v"),
|
||||
task_id: "task-v".to_string(),
|
||||
worker_id: "worker-v".to_string(),
|
||||
attempt: Some(1),
|
||||
terminal_seq: None,
|
||||
completed_at: "2025-01-01T00:00:00Z".to_string(),
|
||||
result: FleetTaskResult::Fail,
|
||||
failure_kind: Some(FleetTaskFailureKind::Verifier),
|
||||
artifacts: Vec::new(),
|
||||
score: None,
|
||||
resolved_route: None,
|
||||
effective_permissions: None,
|
||||
};
|
||||
let value = fleet_receipt_json(&receipt);
|
||||
assert_eq!(value["result"], "fail");
|
||||
assert_eq!(value["failure_kind"], "verifier");
|
||||
assert_eq!(value["retry_eligible"], false);
|
||||
assert!(
|
||||
value["failure_class"]
|
||||
.as_str()
|
||||
.is_some_and(|s| s.contains("Verifier")),
|
||||
"failure_class should describe a verifier rejection"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fleet_receipt_json_transport_failure_is_retry_eligible() {
|
||||
use codewhale_protocol::fleet::{
|
||||
FleetReceipt, FleetRunId, FleetTaskFailureKind, FleetTaskResult,
|
||||
};
|
||||
let receipt = FleetReceipt {
|
||||
run_id: FleetRunId::from("run-t"),
|
||||
task_id: "task-t".to_string(),
|
||||
worker_id: "worker-t".to_string(),
|
||||
attempt: Some(1),
|
||||
terminal_seq: None,
|
||||
completed_at: "2025-01-01T00:00:00Z".to_string(),
|
||||
result: FleetTaskResult::Fail,
|
||||
failure_kind: Some(FleetTaskFailureKind::Transport),
|
||||
artifacts: Vec::new(),
|
||||
score: None,
|
||||
resolved_route: None,
|
||||
effective_permissions: None,
|
||||
};
|
||||
let value = fleet_receipt_json(&receipt);
|
||||
assert_eq!(value["failure_kind"], "transport");
|
||||
assert_eq!(value["retry_eligible"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fleet_receipt_json_receipt_artifact_sets_evidence_available() {
|
||||
use codewhale_protocol::fleet::{
|
||||
FleetArtifactKind, FleetArtifactRef, FleetReceipt, FleetRunId, FleetScore, FleetTaskResult,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
let receipt = FleetReceipt {
|
||||
run_id: FleetRunId::from("run-e"),
|
||||
task_id: "task-e".to_string(),
|
||||
worker_id: "worker-e".to_string(),
|
||||
attempt: Some(1),
|
||||
terminal_seq: None,
|
||||
completed_at: "2025-01-01T00:00:00Z".to_string(),
|
||||
result: FleetTaskResult::Pass,
|
||||
failure_kind: None,
|
||||
artifacts: vec![FleetArtifactRef {
|
||||
kind: FleetArtifactKind::Receipt,
|
||||
path: PathBuf::from(".codewhale/fleet/run-e/task-e/worker-e/receipt.json"),
|
||||
checksum: Some("sha256:abc123".to_string()),
|
||||
mime_type: Some("application/json".to_string()),
|
||||
size_bytes: Some(512),
|
||||
}],
|
||||
score: Some(FleetScore {
|
||||
value: 1.0,
|
||||
max: Some(1.0),
|
||||
notes: Some("all checks pass".to_string()),
|
||||
}),
|
||||
resolved_route: None,
|
||||
effective_permissions: None,
|
||||
};
|
||||
let value = fleet_receipt_json(&receipt);
|
||||
assert_eq!(value["evidence_available"], true);
|
||||
assert_eq!(value["score"]["value"], 1.0);
|
||||
assert_eq!(value["score"]["max"], 1.0);
|
||||
assert_eq!(value["score"]["notes"], "all checks pass");
|
||||
assert_eq!(value["artifacts"][0]["kind"], "receipt");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fleet_receipt_api_list_and_get_round_trip() -> Result<()> {
|
||||
use crate::fleet::ledger::FleetLedger;
|
||||
use crate::fleet::task_spec::{
|
||||
FleetTaskSpecDocument, FleetTaskVerification, prepare_verification_receipt,
|
||||
};
|
||||
use crate::fleet::task_spec::FleetTaskVerificationInput;
|
||||
use codewhale_protocol::fleet::{FleetScore, FleetTaskResult};
|
||||
|
||||
let root =
|
||||
std::env::temp_dir().join(format!("codewhale-receipt-api-{}", Uuid::new_v4()));
|
||||
let workspace = root.join("workspace");
|
||||
fs::create_dir_all(&workspace)?;
|
||||
|
||||
let task = codewhale_protocol::fleet::FleetTaskSpec {
|
||||
id: "task-receipt".to_string(),
|
||||
name: "Receipt Task".to_string(),
|
||||
description: None,
|
||||
objective: Some("Test receipt API".to_string()),
|
||||
instructions: "run tests".to_string(),
|
||||
worker: Some(codewhale_protocol::fleet::FleetTaskWorkerProfile {
|
||||
agent_profile: None,
|
||||
role: Some("reviewer".to_string()),
|
||||
loadout: None,
|
||||
model_class: None,
|
||||
model: None,
|
||||
tool_profile: Some("read-only".to_string()),
|
||||
tools: Vec::new(),
|
||||
capabilities: Vec::new(),
|
||||
}),
|
||||
workspace: None,
|
||||
input_files: Vec::new(),
|
||||
context: Vec::new(),
|
||||
budget: None,
|
||||
tags: Vec::new(),
|
||||
expected_artifacts: Vec::new(),
|
||||
scorer: None,
|
||||
retry_policy: None,
|
||||
alert_policy: None,
|
||||
timeout_seconds: None,
|
||||
metadata: std::collections::BTreeMap::new(),
|
||||
};
|
||||
let manager = crate::fleet::manager::FleetManager::open(&workspace)?
|
||||
.with_session_model(crate::config::DEFAULT_TEXT_MODEL);
|
||||
let report = manager.create_run(
|
||||
FleetTaskSpecDocument {
|
||||
name: Some("receipt-api-smoke".to_string()),
|
||||
labels: std::collections::BTreeMap::new(),
|
||||
security_policy: None,
|
||||
workers: Vec::new(),
|
||||
tasks: vec![task],
|
||||
},
|
||||
1,
|
||||
)?;
|
||||
let run_id = report.run_id.clone();
|
||||
|
||||
// Directly record a synthetic receipt so we don't need a live worker.
|
||||
let ledger = FleetLedger::open(&workspace)?;
|
||||
let verification_input = FleetTaskVerificationInput {
|
||||
run_id: run_id.clone(),
|
||||
task_id: "task-receipt".to_string(),
|
||||
worker_id: "worker-1".to_string(),
|
||||
attempt: 1,
|
||||
exit_code: Some(0),
|
||||
artifacts: Vec::new(),
|
||||
resolved_route: None,
|
||||
effective_permissions: None,
|
||||
};
|
||||
let receipt = prepare_verification_receipt(
|
||||
&workspace,
|
||||
&verification_input,
|
||||
FleetTaskVerification {
|
||||
result: FleetTaskResult::Pass,
|
||||
failure_kind: None,
|
||||
score: FleetScore {
|
||||
value: 1.0,
|
||||
max: Some(1.0),
|
||||
notes: Some("exit_code=0".to_string()),
|
||||
},
|
||||
evidence: vec!["exit_code=0".to_string()],
|
||||
},
|
||||
)?;
|
||||
ledger.record_receipt(receipt)?;
|
||||
|
||||
let sessions_dir = root.join("sessions");
|
||||
let Some((addr, _runtime_threads, handle)) =
|
||||
spawn_test_server_with_root_token_mobile_workspace(
|
||||
root.clone(),
|
||||
sessions_dir,
|
||||
None,
|
||||
false,
|
||||
workspace,
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let client = crate::tls::reqwest_client();
|
||||
|
||||
// List receipts for the run.
|
||||
let list: serde_json::Value = client
|
||||
.get(format!(
|
||||
"http://{addr}/v1/fleet/runs/{}/receipts",
|
||||
run_id.0
|
||||
))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json()
|
||||
.await?;
|
||||
assert_eq!(list["run_id"], run_id.0.as_str());
|
||||
assert_eq!(list["receipts"].as_array().map(|a| a.len()), Some(1));
|
||||
let receipt_entry = &list["receipts"][0];
|
||||
assert_eq!(receipt_entry["task_id"], "task-receipt");
|
||||
assert_eq!(receipt_entry["result"], "pass");
|
||||
assert_eq!(receipt_entry["retry_eligible"], false);
|
||||
assert_eq!(receipt_entry["evidence_available"], true);
|
||||
|
||||
// Get specific receipt by task_id.
|
||||
let detail: serde_json::Value = client
|
||||
.get(format!(
|
||||
"http://{addr}/v1/fleet/runs/{}/receipts/task-receipt",
|
||||
run_id.0
|
||||
))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json()
|
||||
.await?;
|
||||
assert_eq!(detail["run_id"], run_id.0.as_str());
|
||||
assert_eq!(detail["task_id"], "task-receipt");
|
||||
assert_eq!(detail["worker_id"], "worker-1");
|
||||
assert_eq!(detail["attempt"], 1);
|
||||
assert_eq!(detail["result"], "pass");
|
||||
assert!(detail["failure_kind"].is_null());
|
||||
assert_eq!(detail["evidence_available"], true);
|
||||
|
||||
// Inspect evidence content.
|
||||
let evidence: serde_json::Value = client
|
||||
.get(format!(
|
||||
"http://{addr}/v1/fleet/runs/{}/receipts/task-receipt/evidence",
|
||||
run_id.0
|
||||
))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json()
|
||||
.await?;
|
||||
assert_eq!(evidence["run_id"], run_id.0.as_str());
|
||||
assert_eq!(evidence["task_id"], "task-receipt");
|
||||
assert_eq!(evidence["truncated"], false);
|
||||
assert!(
|
||||
evidence["content"].is_object(),
|
||||
"evidence content should parse as JSON object"
|
||||
);
|
||||
assert_eq!(evidence["content"]["task_id"], "task-receipt");
|
||||
|
||||
// Missing task returns 404.
|
||||
let missing = client
|
||||
.get(format!(
|
||||
"http://{addr}/v1/fleet/runs/{}/receipts/no-such-task",
|
||||
run_id.0
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(missing.status(), 404);
|
||||
|
||||
handle.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user