Add Agent Tuning Sprint proof harness.

Pinned code-review fixture, quality gate, and poc.py doctor/run/compare/verify
so the paid sprint can be demoed without inventing savings. Clippy clean on
local_runner and OCLA adapter tests so pre-commit --all-targets passes.

Co-authored-by: Cursor <cursoragent@cursor.com>

.gitignore

@@ -93,3 +93,8 @@ packages/lean-ctx-bin/bin/
 *.egg-info/
 _archive/bench/lifecycle/runs/
 _archive/bench/lifecycle/.cache/
+
+# Sprint POC local run dirs
+.sprint-poc/
+
+.pytest_cache/

examples/sprint-poc/README.md

@@ -0,0 +1,37 @@
+# Sprint POC — 15-minute evidence harness
+
+Pinned code-review fixture + `LeanCTX.wrap()` for the paid Agent Tuning Sprint.
+
+**Integrity:** no synthetic cost. A cheaper treatment is a win only if **both**
+arms pass `expected-findings-v1`.
+
+## Setup
+
+```bash
+cd examples/sprint-poc
+python3 -m venv .venv
+source .venv/bin/activate
+pip install openai lean-ctx-python
+export OPENAI_API_KEY=...          # required for `run`, not for tests
+# lean-ctx proxy should already be running
+```
+
+```bash
+python poc.py doctor
+python poc.py run --arm stock --out ../../.sprint-poc
+python poc.py run --arm leanctx --out ../../.sprint-poc
+python poc.py compare --out ../../.sprint-poc
+python poc.py verify --out ../../.sprint-poc
+```
+
+Quality tests (no API key):
+
+```bash
+python -m pytest test_quality.py
+```
+
+## What the buyer sees
+
+Same agent, same `fixture/checkout.py`, two arms: stock vs wrap().
+Output is ReviewResult JSON, a quality gate, and a treatment receipt when
+the local proxy seals one.

examples/sprint-poc/agent.py

@@ -0,0 +1,101 @@
+"""Reference code-review agent for the Agent Tuning Sprint harness.
+
+Stock path has no LeanCTX import. Treatment uses LeanCTX.wrap() and the
+ContextAware `leanctx=` keyword so proxy compression is opt-in, not global.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import re
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parent
+FIXTURE = ROOT / "fixture" / "checkout.py"
+MANIFEST = json.loads((ROOT / "workload-manifest.json").read_text(encoding="utf-8"))
+
+SYSTEM = """You review source for correctness and security.
+Return ONLY JSON with this shape:
+{"findings":[{"id":"sql_injection|missing_authz|discount_off_by_one|other","severity":"high|medium|low","location":"checkout.py:<function>","summary":"<one sentence>"}]}
+Use the canonical ids when the defect matches. Do not invent files that are not in the source.
+"""
+
+
+class ReferenceCodeReviewAgent:
+    name = "ReferenceCodeReviewAgent"
+    version = MANIFEST["agent_version"]
+
+    def describe(self) -> dict[str, str]:
+        return {
+            "agent": f"{self.name} v{self.version}",
+            "framework": "ContextAwareAgent (OpenAI Chat Completions)",
+            "model": os.environ.get("OPENAI_MODEL", MANIFEST["model_default"]),
+            "output_schema": "ReviewResult v1",
+            "fixture": str(FIXTURE.relative_to(ROOT)),
+            "leanctx_attached": "via wrap() only",
+        }
+
+    def run(self, task: str, *, leanctx: Any = None) -> dict[str, Any]:
+        source = FIXTURE.read_text(encoding="utf-8")
+        user = f"{task}\n\nSOURCE fixture/checkout.py:\n{source}"
+        messages = [
+            {"role": "system", "content": SYSTEM},
+            {"role": "user", "content": user},
+        ]
+        if leanctx is not None:
+            compressed = leanctx.compress(messages, model=self._model())
+            messages = list(compressed.messages)
+        content = self._complete(messages)
+        return _parse_review(content)
+
+    def _model(self) -> str:
+        return os.environ.get("OPENAI_MODEL", MANIFEST["model_default"])
+
+    def _complete(self, messages: list[dict[str, str]]) -> str:
+        api_key = os.environ.get("OPENAI_API_KEY", "").strip()
+        if not api_key:
+            raise RuntimeError(
+                "OPENAI_API_KEY is required for sprint-poc run. "
+                "doctor() and quality tests do not call the model."
+            )
+        try:
+            from openai import OpenAI
+        except ImportError as exc:
+            raise RuntimeError(
+                "Install openai to run the live agent: pip install openai lean-ctx-python"
+            ) from exc
+        client = OpenAI(api_key=api_key)
+        response = client.chat.completions.create(
+            model=self._model(),
+            temperature=0,
+            response_format={"type": "json_object"},
+            messages=messages,
+        )
+        choice = response.choices[0].message.content
+        if not choice:
+            raise RuntimeError("model returned empty content")
+        return choice
+
+
+def _parse_review(content: str) -> dict[str, Any]:
+    match = re.search(r"\{.*\}", content, re.DOTALL)
+    raw = match.group(0) if match else content
+    parsed = json.loads(raw)
+    if not isinstance(parsed, dict) or "findings" not in parsed:
+        raise ValueError("model output is not ReviewResult v1 JSON")
+    return parsed
+
+
+def main() -> None:
+    agent = ReferenceCodeReviewAgent()
+    if sys.argv[1:] == ["describe"]:
+        print(json.dumps(agent.describe(), indent=2))
+        return
+    print(json.dumps(agent.describe(), indent=2))
+
+
+if __name__ == "__main__":
+    main()

examples/sprint-poc/expected-findings.json

@@ -0,0 +1,24 @@
+{
+  "evaluator": "expected-findings-v1",
+  "task": "Review fixture/checkout.py for correctness, security, and authorization defects. Return JSON ReviewResult v1.",
+  "required": [
+    {
+      "id": "sql_injection",
+      "file": "checkout.py",
+      "function": "find_order",
+      "must_match": "sql|inject|concatenat|parameter"
+    },
+    {
+      "id": "missing_authz",
+      "file": "checkout.py",
+      "function": "export_orders",
+      "must_match": "auth|permission|unauthor"
+    },
+    {
+      "id": "discount_off_by_one",
+      "file": "checkout.py",
+      "function": "apply_discount",
+      "must_match": "off-by-one|range\\(|index|len\\(prices\\) \\+ 1"
+    }
+  ]
+}

examples/sprint-poc/fixture/checkout.py

@@ -0,0 +1,28 @@
+"""Pinned checkout fixture: real defects a reviewer must locate."""
+
+import sqlite3
+from typing import Any
+
+
+def find_order(conn: sqlite3.Connection, order_id: str) -> Any:
+    # Defect: query is concatenated, not parameterized.
+    cursor = conn.execute(
+        "SELECT * FROM orders WHERE id = '" + order_id + "'"
+    )
+    return cursor.fetchone()
+
+
+def apply_discount(prices: list[int], percent: int) -> list[int]:
+    # Defect: loop walks one past the last price (off-by-one).
+    discounted: list[int] = []
+    for i in range(len(prices) + 1):
+        discounted.append(prices[i] * (100 - percent) // 100)
+    return discounted
+
+
+def export_orders(user: dict[str, Any] | None) -> list[tuple[Any, ...]]:
+    # Defect: no authorization check before reading all orders.
+    conn = sqlite3.connect(":memory:")
+    conn.execute("CREATE TABLE orders (id TEXT, amount INTEGER)")
+    conn.execute("INSERT INTO orders VALUES ('ord_1', 1500)")
+    return conn.execute("SELECT * FROM orders").fetchall()

examples/sprint-poc/poc.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""Sprint POC harness: doctor, stock/treatment run, compare, verify.
+
+Does not invent savings. Failed quality on either arm blocks a win.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parent
+REPO = ROOT.parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from agent import ReferenceCodeReviewAgent  # noqa: E402
+from quality import evaluate, load_expected  # noqa: E402
+
+MANIFEST = json.loads((ROOT / "workload-manifest.json").read_text(encoding="utf-8"))
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="poc.py")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("doctor")
+    run = sub.add_parser("run")
+    run.add_argument("--arm", choices=("stock", "leanctx"), required=True)
+    run.add_argument("--out", type=Path, required=True)
+    cmp_p = sub.add_parser("compare")
+    cmp_p.add_argument("--out", type=Path, required=True)
+    ver = sub.add_parser("verify")
+    ver.add_argument("--out", type=Path, required=True)
+    args = parser.parse_args(argv)
+    if args.cmd == "doctor":
+        return cmd_doctor()
+    if args.cmd == "run":
+        return cmd_run(args.arm, args.out)
+    if args.cmd == "compare":
+        return cmd_compare(args.out)
+    return cmd_verify(args.out)
+
+
+def cmd_doctor() -> int:
+    checks: list[tuple[str, bool, str]] = []
+    checks.append(("python", True, sys.version.split()[0]))
+    checks.append(("fixture", (ROOT / "fixture" / "checkout.py").is_file(), "checkout.py"))
+    checks.append(("expected-findings", (ROOT / "expected-findings.json").is_file(), "json"))
+    checks.append(("kit", (REPO / "kits" / "code-review" / "kit.toml").is_file(), "kits/code-review"))
+    key = bool(os.environ.get("OPENAI_API_KEY", "").strip())
+    checks.append(("OPENAI_API_KEY", key, "set" if key else "missing — run needs it"))
+    proxy = _proxy_reachable()
+    checks.append(("lean-ctx proxy", proxy, "loopback" if proxy else "not reachable"))
+    try:
+        import lean_ctx  # noqa: F401
+
+        checks.append(("lean-ctx-python", True, "import ok"))
+    except ImportError:
+        checks.append(("lean-ctx-python", False, "pip install lean-ctx-python"))
+
+    print("LeanCTX Sprint POC preflight")
+    failed = 0
+    for name, ok, detail in checks:
+        mark = "ok" if ok else "FAIL"
+        print(f"  {mark:4}  {name}: {detail}")
+        if not ok and name != "OPENAI_API_KEY" and name != "lean-ctx proxy":
+            failed += 1
+    ready = failed == 0
+    print("READY" if ready else "NOT READY")
+    if not key:
+        print("  note: live run requires OPENAI_API_KEY; quality tests do not.")
+    return 0 if ready else 1
+
+
+def cmd_run(arm: str, out_root: Path) -> int:
+    run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + f"-{arm}"
+    run_dir = out_root / "runs" / run_id
+    run_dir.mkdir(parents=True, exist_ok=False)
+    shutil.copy(ROOT / "workload-manifest.json", run_dir / "workload-manifest.json")
+
+    agent = ReferenceCodeReviewAgent()
+    task = load_expected()["task"]
+    receipt_payload: dict[str, Any] | None = None
+    try:
+        if arm == "stock":
+            review = agent.run(task, leanctx=None)
+        else:
+            review, receipt_payload = _run_wrapped(agent, task)
+    except Exception as exc:
+        (run_dir / "error.txt").write_text(str(exc), encoding="utf-8")
+        print(f"error: {exc}")
+        return 1
+
+    quality = evaluate(review)
+    (run_dir / f"{arm}-output.json").write_text(
+        json.dumps(review, indent=2) + "\n", encoding="utf-8"
+    )
+    (run_dir / "quality-result.json").write_text(
+        json.dumps(quality, indent=2) + "\n", encoding="utf-8"
+    )
+    if receipt_payload is not None:
+        (run_dir / "execution-receipt.json").write_text(
+            json.dumps(receipt_payload, indent=2) + "\n", encoding="utf-8"
+        )
+
+    print(f"RUN {arm}/{run_id}")
+    print(f"  Quality: {'PASS' if quality['passed'] else 'FAIL'} "
+          f"({quality['matched_count']}/{quality['required_count']})")
+    if quality["missing"]:
+        print(f"  Missing: {', '.join(quality['missing'])}")
+    print(f"  Output:  {run_dir}")
+    return 0 if quality["passed"] else 2
+
+
+def _run_wrapped(agent: ReferenceCodeReviewAgent, task: str) -> tuple[dict[str, Any], dict[str, Any]]:
+    from lean_ctx import LeanCTX
+
+    ctx = LeanCTX(
+        {
+            "project": "sprint-poc",
+            "default_profile": "balanced",
+            "fail_open": False,
+        }
+    )
+    wrapped = ctx.wrap(agent, kit="code-review", profile="balanced")
+    run = wrapped.run(task)
+    review = run.output
+    if not isinstance(review, dict):
+        raise TypeError("wrapped agent must return ReviewResult dict")
+    receipt = run.receipt
+    payload = {
+        "receipt_id": getattr(receipt, "receipt_id", None),
+        "verified": bool(receipt.verify()) if hasattr(receipt, "verify") else False,
+        "savings": _savings_dict(receipt),
+        "degradations": list(getattr(receipt, "degradations", ()) or ()),
+        "coverage": getattr(receipt, "coverage", None),
+        "integrity_status": getattr(receipt, "integrity_status", None),
+    }
+    return review, payload
+
+
+def _savings_dict(receipt: Any) -> dict[str, Any] | None:
+    savings = getattr(receipt, "savings", None)
+    if savings is None:
+        return None
+    return {
+        "original_tokens": getattr(savings, "original_tokens", None),
+        "delivered_tokens": getattr(savings, "delivered_tokens", None),
+        "saved_tokens": getattr(savings, "saved_tokens", None),
+        "methodology": getattr(savings, "methodology", None),
+        "provider_input_tokens": getattr(savings, "provider_input_tokens", None),
+        "provider_output_tokens": getattr(savings, "provider_output_tokens", None),
+        "baseline_cost_micros": getattr(savings, "baseline_cost_micros", None),
+        "treatment_cost_micros": getattr(savings, "treatment_cost_micros", None),
+        "quality_status": getattr(savings, "quality_status", None),
+    }
+
+
+def cmd_compare(out_root: Path) -> int:
+    runs = sorted((out_root / "runs").glob("*"))
+    stock = _latest(runs, "stock")
+    treatment = _latest(runs, "leanctx")
+    if stock is None or treatment is None:
+        print("error: need one stock run and one leanctx run")
+        return 1
+    stock_q = json.loads((stock / "quality-result.json").read_text(encoding="utf-8"))
+    treat_q = json.loads((treatment / "quality-result.json").read_text(encoding="utf-8"))
+    both_pass = bool(stock_q.get("passed") and treat_q.get("passed"))
+    receipt_path = treatment / "execution-receipt.json"
+    savings = None
+    if receipt_path.is_file():
+        savings = json.loads(receipt_path.read_text(encoding="utf-8")).get("savings")
+    comparison = {
+        "baseline": str(stock),
+        "treatment": str(treatment),
+        "quality_both_passed": both_pass,
+        "savings_claim_allowed": both_pass,
+        "savings": savings if both_pass else None,
+        "note": None
+        if both_pass
+        else "Quality gate failed on at least one arm. No savings claim.",
+    }
+    dest = out_root / "comparison.json"
+    dest.write_text(json.dumps(comparison, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(comparison, indent=2))
+    return 0 if both_pass else 2
+
+
+def cmd_verify(out_root: Path) -> int:
+    comparison_path = out_root / "comparison.json"
+    if not comparison_path.is_file():
+        print("error: run compare first")
+        return 1
+    comparison = json.loads(comparison_path.read_text(encoding="utf-8"))
+    receipt = Path(comparison["treatment"]) / "execution-receipt.json"
+    if not receipt.is_file():
+        print("error: treatment receipt missing")
+        return 1
+    payload = json.loads(receipt.read_text(encoding="utf-8"))
+    verified = bool(payload.get("verified"))
+    print(f"treatment receipt verified: {verified}")
+    copy = receipt.with_name("execution-receipt.tampered.json")
+    tampered = dict(payload)
+    tampered["receipt_id"] = "tampered"
+    copy.write_text(json.dumps(tampered, indent=2) + "\n", encoding="utf-8")
+    print(f"wrote tampered copy: {copy}")
+    print("Tamper check is structural: original verified flag vs altered id.")
+    print("Use lean-ctx / SDK verify on a sealed receipt for Ed25519 rejection.")
+    return 0 if verified else 2
+
+
+def _latest(runs: list[Path], arm: str) -> Path | None:
+    matches = [path for path in runs if path.name.endswith(f"-{arm}")]
+    return matches[-1] if matches else None
+
+
+def _proxy_reachable() -> bool:
+    try:
+        result = subprocess.run(
+            ["lean-ctx", "status"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return False
+    return result.returncode == 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

examples/sprint-poc/quality.py

@@ -0,0 +1,66 @@
+"""Deterministic quality gate for the Sprint POC fixture.
+
+Maps reviewer findings onto predeclared defects. Never invents cost or
+savings. A missing required defect is a gate failure.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parent
+EXPECTED_PATH = ROOT / "expected-findings.json"
+
+
+def load_expected(path: Path = EXPECTED_PATH) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def evaluate(review: dict[str, Any], expected: dict[str, Any] | None = None) -> dict[str, Any]:
+    spec = expected or load_expected()
+    findings = review.get("findings") or []
+    if not isinstance(findings, list):
+        return {
+            "evaluator": spec.get("evaluator"),
+            "passed": False,
+            "matched": [],
+            "missing": [item["id"] for item in spec["required"]],
+            "error": "review.findings must be a list",
+        }
+
+    matched: list[str] = []
+    missing: list[str] = []
+    for item in spec["required"]:
+        if _matches(findings, item):
+            matched.append(item["id"])
+        else:
+            missing.append(item["id"])
+
+    return {
+        "evaluator": spec["evaluator"],
+        "passed": not missing,
+        "matched": matched,
+        "missing": missing,
+        "required_count": len(spec["required"]),
+        "matched_count": len(matched),
+    }
+
+
+def _matches(findings: list[Any], item: dict[str, Any]) -> bool:
+    pattern = re.compile(item["must_match"], re.IGNORECASE)
+    for finding in findings:
+        if not isinstance(finding, dict):
+            continue
+        blob = " ".join(
+            str(finding.get(key, ""))
+            for key in ("id", "location", "summary", "function", "file")
+        )
+        id_hit = str(finding.get("id", "")).lower() == item["id"]
+        loc_hit = item["file"] in blob and item["function"] in blob
+        text_hit = pattern.search(blob) is not None
+        if id_hit or (loc_hit and text_hit) or (item["file"] in blob and text_hit):
+            return True
+    return False

examples/sprint-poc/test_quality.py

@@ -0,0 +1,66 @@
+from quality import evaluate
+
+
+def test_all_required_findings_pass():
+    review = {
+        "findings": [
+            {
+                "id": "sql_injection",
+                "location": "checkout.py:find_order",
+                "summary": "SQL query is concatenated instead of parameterized",
+            },
+            {
+                "id": "missing_authz",
+                "location": "checkout.py:export_orders",
+                "summary": "Exports orders without an authorization check",
+            },
+            {
+                "id": "discount_off_by_one",
+                "location": "checkout.py:apply_discount",
+                "summary": "range(len(prices) + 1) indexes past the last price",
+            },
+        ]
+    }
+    result = evaluate(review)
+    assert result["passed"] is True
+    assert result["missing"] == []
+
+
+def test_missing_defect_fails_closed():
+    review = {
+        "findings": [
+            {
+                "id": "sql_injection",
+                "location": "checkout.py:find_order",
+                "summary": "concatenated SQL",
+            }
+        ]
+    }
+    result = evaluate(review)
+    assert result["passed"] is False
+    assert "missing_authz" in result["missing"]
+    assert "discount_off_by_one" in result["missing"]
+
+
+def test_keyword_match_without_canonical_id():
+    review = {
+        "findings": [
+            {
+                "id": "other",
+                "location": "checkout.py:find_order",
+                "summary": "SQL injection via string concatenation",
+            },
+            {
+                "id": "other",
+                "location": "checkout.py:export_orders",
+                "summary": "missing authorization before SELECT *",
+            },
+            {
+                "id": "other",
+                "location": "checkout.py:apply_discount",
+                "summary": "off-by-one in range(len(prices) + 1)",
+            },
+        ]
+    }
+    result = evaluate(review)
+    assert result["passed"] is True

examples/sprint-poc/workload-manifest.json

@@ -0,0 +1,11 @@
+{
+  "workload_id": "sprint-poc-code-review-v1",
+  "agent": "ReferenceCodeReviewAgent",
+  "agent_version": "1.0.0",
+  "fixture": "fixture/checkout.py",
+  "quality_evaluator": "expected-findings-v1",
+  "model_env": "OPENAI_MODEL",
+  "model_default": "gpt-4.1-mini",
+  "kit": "code-review",
+  "comparison_rule": "reject if agent, fixture, or evaluator hash differ between arms"
+}

rust/src/core/local_runner/runner.rs

@@ -210,7 +210,7 @@ mod tests {
                 agent: "mock".into(),
                 model: "test-model".into(),
                 success: self.should_succeed,
-                exit_code: if self.should_succeed { 0 } else { 1 },
+                exit_code: i32::from(!self.should_succeed),
                 stdout: "task output".into(),
                 stderr: String::new(),
                 duration_ms: 1000,
@@ -308,7 +308,7 @@ mod tests {
                 id: "test".into(),
                 name: "Test".into(),
                 description: "Test task".into(),
-                kind: kind.clone(),
+                kind: *kind,
                 timeout_ms: None,
             };
             let prompt = task_to_prompt(&task);

rust/src/core/ocla/adapters/external_process.rs

@@ -164,8 +164,14 @@ mod tests {
         let _parsed: serde_json::Value =
             serde_json::from_str(&result.observation.output_ref.as_deref().unwrap_or(""))
                 .unwrap_or_default();
-        assert!(result.observation.metrics.get("word_count").copied() == Some(3));
-        assert!(result.observation.metrics.get("line_count").copied() == Some(1));
+        assert_eq!(
+            result.observation.metrics.get("word_count").copied(),
+            Some(3)
+        );
+        assert_eq!(
+            result.observation.metrics.get("line_count").copied(),
+            Some(1)
+        );
     }
 
     #[test]
@@ -176,7 +182,7 @@ mod tests {
             capability_id: "capability://example/word-count-optimizer".into(),
             capability_version: "1.0.0".into(),
             input: CapabilityInput::ShellCommand {
-                command: "".into(),
+                command: String::new(),
                 workdir: None,
             },
             policy_constraints: PolicyConstraints::default(),