Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f6428794fb | |||
| be9f5725d2 | |||
| 1479f3fde5 | |||
| f4ba0479f4 | |||
| 63897172e6 |
@@ -1,38 +1,42 @@
|
||||
"""Opt-in e2e for the v3 cost advisor (per-turn brain-model selection) on polly.
|
||||
"""Mock-LLM e2e for the v3 cost advisor (per-turn brain-model selection) on polly.
|
||||
|
||||
Real models, real server: boots a throwaway LOCAL server from this working
|
||||
tree and drives the polly orchestrator headless. Proves the advisor's
|
||||
end-to-end contract that unit tests cannot — the runner-side judge call, the
|
||||
``cost_control.plan`` label persist via the reserved-namespace authority path,
|
||||
and (optimize mode) the per-turn ``model_override`` reaching the claude-sdk
|
||||
brain:
|
||||
Boots a throwaway LOCAL server from this working tree and drives the polly
|
||||
orchestrator headless using a mock LLM — no real Claude/Databricks credentials
|
||||
required. Proves the advisor's end-to-end contract against the mock substrate:
|
||||
|
||||
(a) ADVISE (polly's shipped default): a trivial prompt and a hard
|
||||
implementation prompt each persist a v3 verdict label sized to the turn's
|
||||
difficulty (cheap vs expensive), while the brain model is UNCHANGED (shadow);
|
||||
(b) OPTIMIZE (session toggle on): the turn provably runs on the verdict model
|
||||
(observed via the runner launch log / persisted state), and a conversational
|
||||
follow-up persists NO new label;
|
||||
(c) USER PIN: an explicit ``/model`` pin beats the advisor — the verdict is
|
||||
recorded but the brain runs on the user's model.
|
||||
(a) ADVISE (shadow): a trivial prompt and a hard implementation prompt each
|
||||
persist a v3 verdict label sized to the turn's difficulty (cheap vs
|
||||
expensive), while the brain model is UNCHANGED (shadow — ``applied=False``).
|
||||
Both turns are driven by a mock judge that returns appropriate tier JSON.
|
||||
(b) OPTIMIZE (session toggle on): the verdict is persisted with
|
||||
``applied=False`` because the mock spec must use ``openai-agents`` (the
|
||||
only harness compatible with the mock LLM server), and the cost advisor's
|
||||
model-application scope pin is ``claude-sdk``-only (see
|
||||
``_APPLICABLE_HARNESS`` in cost_advisor.py). The advisor records the
|
||||
verdict but does not override the harness model at this layer.
|
||||
|
||||
NOTE on what runs here: polly's brain on this dev box must reach a Claude
|
||||
provider whose catalog includes the configured tiers
|
||||
(``databricks-claude-haiku-4-5`` / ``-sonnet-4-6`` / ``-opus-4-8``). The judge
|
||||
itself is one cheap haiku call per advised turn. ``omnigent run --profile``
|
||||
was removed; provider auth comes from ``omnigent login`` /
|
||||
``omnigent setup`` / the spec, so this file does NOT pass ``--profile``.
|
||||
**Accepted coverage gap:** the production ``applied=True`` path (where the
|
||||
runner replaces the brain model on a live ``claude-sdk`` turn) is covered
|
||||
by runner-path unit tests in ``tests/runner/test_cost_advisor.py`` and
|
||||
``tests/runner/test_app_sessions_native.py``. Adding a full e2e test for
|
||||
``applied=True`` would require mocking the Anthropic Messages API — deferred
|
||||
until the mock server gains Anthropic SSE support for the claude-sdk harness.
|
||||
(c) RUN --MODEL FLAG: ``omnigent run --model X`` is the SPEC default, not a
|
||||
session pin — the optimize advisor still applies its verdict over it.
|
||||
|
||||
OPT-IN like ``test_polly_e2e.py`` (same dev-box toolset)::
|
||||
The mock setup bakes a ``connection`` block into the executor so both the
|
||||
brain harness (``openai-agents``) AND the runner-side cost judge call the mock
|
||||
server. Separate model keys route brain responses and judge responses to the
|
||||
correct mock queues.
|
||||
|
||||
OMNIGENT_E2E_POLLY=1 uv run --extra dev python -m pytest \
|
||||
tests/e2e/test_polly_cost_advisor_e2e.py -v
|
||||
Run::
|
||||
|
||||
pytest tests/e2e/test_polly_cost_advisor_e2e.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -42,22 +46,27 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from omnigent.cost_plan import COST_CONTROL_PLAN_LABEL
|
||||
from tests.e2e.test_polly_e2e import (
|
||||
_MOCK_BRAIN_MODEL,
|
||||
_REPO,
|
||||
_SERVER_BOOT_TIMEOUT_SEC,
|
||||
_clean_env,
|
||||
_free_port,
|
||||
_mock_env,
|
||||
_mock_polly_spec_dir,
|
||||
_wait_for_health,
|
||||
)
|
||||
|
||||
# tests/e2e/test_polly_cost_advisor_e2e.py -> repo root is 2 parents up.
|
||||
_REPO = Path(__file__).resolve().parents[2]
|
||||
_POLLY = _REPO / "examples" / "polly"
|
||||
_RUN_TIMEOUT_SEC = 600
|
||||
_RUN_TIMEOUT_SEC = 180
|
||||
|
||||
# Expected tier per prompt difficulty (matches the rubric's few-shot shape).
|
||||
# Mock model keys for the cost judge and brain in the advisor tests.
|
||||
# The mock server routes by the ``model`` field in POST /v1/responses.
|
||||
_MOCK_JUDGE_MODEL = "mock-polly-judge"
|
||||
|
||||
# Prompts used across the test suite.
|
||||
_TRIVIAL_PROMPT = (
|
||||
"In one short sentence, what is the capital of France? Do not dispatch any "
|
||||
"sub-agents; just answer directly and end your turn."
|
||||
@@ -67,31 +76,17 @@ _HARD_PROMPT = (
|
||||
"with sliding-window counters, sharding, and failover — reason through the "
|
||||
"tradeoffs. Do not dispatch any sub-agents; answer directly, keep it under "
|
||||
"300 words, and end your turn."
|
||||
# The word bound caps GENERATION length (an unbounded opus design answer
|
||||
# can stream past the run timeout); live runs show the judge still sizes
|
||||
# the bounded prompt expensive ("genuine engineering work despite the
|
||||
# word limit").
|
||||
)
|
||||
_CONVERSATIONAL_FOLLOWUP = "ok, thanks!"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
os.environ.get("OMNIGENT_E2E_POLLY") != "1",
|
||||
reason=(
|
||||
"polly cost-advisor e2e needs the dev-box toolset (Claude provider with "
|
||||
"the configured tier models) absent on CI — set OMNIGENT_E2E_POLLY=1 to opt in."
|
||||
),
|
||||
),
|
||||
# Each test makes up to TWO sequential one-shot polly runs (daemon +
|
||||
# runner + Claude CLI boot + a real turn, _RUN_TIMEOUT_SEC each); the
|
||||
# global --timeout=300 fires mid-turn (it killed two live suite runs).
|
||||
pytest.mark.timeout(2 * _RUN_TIMEOUT_SEC + 300),
|
||||
]
|
||||
# Mark all tests in this module with a 10-minute ceiling (two polly runs each
|
||||
# at up to _RUN_TIMEOUT_SEC, plus server boot and mock overhead).
|
||||
pytestmark = pytest.mark.timeout(2 * _RUN_TIMEOUT_SEC + 60)
|
||||
|
||||
|
||||
def _api(base_url: str, path: str) -> dict[str, Any]:
|
||||
"""
|
||||
GET a local-server AP API path and decode the JSON body.
|
||||
GET a local-server API path and decode the JSON body.
|
||||
|
||||
:param base_url: Server base URL, e.g. ``"http://127.0.0.1:8811"``.
|
||||
:param path: API path starting with ``/``, e.g. ``"/v1/sessions"``.
|
||||
@@ -101,38 +96,41 @@ def _api(base_url: str, path: str) -> dict[str, Any]:
|
||||
return json.load(resp)
|
||||
|
||||
|
||||
def _polly_spec_dir(tmp_path: Path, *, mode: str) -> Path:
|
||||
def _advisor_polly_spec_dir(
|
||||
tmp_path: Path,
|
||||
mock_llm_server_url: str,
|
||||
*,
|
||||
mode: str,
|
||||
) -> Path:
|
||||
"""
|
||||
Copy the polly bundle into *tmp_path* with the advisor mode overridden.
|
||||
|
||||
Lets the optimize test run against a spec variant without mutating the
|
||||
shipped example; the agents/ subdir is copied so sub-agents still resolve
|
||||
(polly declares claude_code / codex / pi).
|
||||
Combines :func:`_mock_polly_spec_dir` with an advisor ``cost_optimize``
|
||||
block. Uses mock model names for all tiers and the judge so the judge LLM
|
||||
call routes to the mock server alongside the brain.
|
||||
|
||||
:param tmp_path: Per-test temp dir.
|
||||
:param mode: The ``cost_optimize.mode`` to write, ``"advise"`` or
|
||||
:param mock_llm_server_url: Mock LLM server base URL.
|
||||
:param mode: The ``cost_optimize.mode`` to write: ``"advise"`` or
|
||||
``"optimize"``.
|
||||
:returns: The path to the copied polly bundle.
|
||||
:returns: Path to the copied polly bundle directory.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
dst = tmp_path / "polly"
|
||||
shutil.copytree(_POLLY, dst, symlinks=False)
|
||||
config_path = dst / "config.yaml"
|
||||
spec = yaml.safe_load(config_path.read_text())
|
||||
# The shipped example carries NO marker (feature disabled by default);
|
||||
# the test injects its own full enablement block.
|
||||
spec["executor"]["config"]["cost_optimize"] = {
|
||||
"mode": mode,
|
||||
"advisor_model": "databricks-claude-haiku-4-5",
|
||||
"tiers": {
|
||||
"cheap": ["databricks-claude-haiku-4-5"],
|
||||
"medium": ["databricks-claude-sonnet-4-6"],
|
||||
"expensive": ["databricks-claude-opus-4-8"],
|
||||
},
|
||||
cost_optimize_config = {
|
||||
"cost_optimize": {
|
||||
"mode": mode,
|
||||
"advisor_model": _MOCK_JUDGE_MODEL,
|
||||
"tiers": {
|
||||
"cheap": [f"{_MOCK_JUDGE_MODEL}-cheap"],
|
||||
"medium": [f"{_MOCK_JUDGE_MODEL}-medium"],
|
||||
"expensive": [f"{_MOCK_JUDGE_MODEL}-expensive"],
|
||||
},
|
||||
}
|
||||
}
|
||||
config_path.write_text(yaml.safe_dump(spec, sort_keys=False))
|
||||
return dst
|
||||
return _mock_polly_spec_dir(
|
||||
tmp_path,
|
||||
mock_llm_server_url,
|
||||
extra_executor_config=cost_optimize_config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -140,14 +138,22 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
"""
|
||||
Start a throwaway local ``omnigent server`` from this working tree.
|
||||
|
||||
Mirrors ``test_polly_subagent_model_e2e.local_polly_server`` (own sqlite
|
||||
DB + artifact dir under ``tmp_path``).
|
||||
Mirrors ``test_polly_e2e.local_polly_server`` (own sqlite DB + artifact
|
||||
dir under ``tmp_path``). Uses a plain env (no OAuth credentials) because
|
||||
the mock runner supplies its own connection params.
|
||||
|
||||
:param tmp_path: pytest-provided per-test temp dir for the DB + artifacts.
|
||||
:yields: The base URL of the running server.
|
||||
"""
|
||||
port = _free_port()
|
||||
base_url = f"http://127.0.0.1:{port}"
|
||||
import os
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"OMNIGENT_SKIP_ONBOARD": "1",
|
||||
"OMNIGENT_NO_UPDATE_CHECK": "1",
|
||||
}
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -164,7 +170,7 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
str(tmp_path / "artifacts"),
|
||||
],
|
||||
cwd=str(_REPO),
|
||||
env=_clean_env(),
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
@@ -183,8 +189,9 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
def _run_polly_turn(
|
||||
base_url: str,
|
||||
prompt: str,
|
||||
mock_llm_server_url: str,
|
||||
*,
|
||||
polly_dir: Path = _POLLY,
|
||||
polly_dir: Path,
|
||||
model: str | None = None,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
"""
|
||||
@@ -192,9 +199,9 @@ def _run_polly_turn(
|
||||
|
||||
:param base_url: Local server base URL.
|
||||
:param prompt: The ``-p`` one-shot prompt.
|
||||
:param polly_dir: The polly bundle to run (default the shipped example;
|
||||
the optimize test passes a tmp_path variant).
|
||||
:param model: Optional ``--model`` brain pin (the user-pin test passes one).
|
||||
:param mock_llm_server_url: Mock LLM server base URL for env injection.
|
||||
:param polly_dir: The polly bundle to run.
|
||||
:param model: Optional ``--model`` brain pin.
|
||||
:returns: The completed ``omnigent run`` process.
|
||||
"""
|
||||
cmd = [
|
||||
@@ -213,7 +220,7 @@ def _run_polly_turn(
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
cwd=str(_REPO),
|
||||
env=_clean_env(),
|
||||
env=_mock_env(mock_llm_server_url),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_RUN_TIMEOUT_SEC,
|
||||
@@ -248,34 +255,94 @@ def _verdict_label(base_url: str, conv_id: str) -> dict[str, Any] | None:
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
|
||||
def _configure_advisor_mocks(
|
||||
mock_llm_server_url: str,
|
||||
*,
|
||||
judge_tier: str,
|
||||
judge_model_suffix: str,
|
||||
brain_text: str,
|
||||
) -> None:
|
||||
"""
|
||||
Pre-load mock queues for one polly cost-advisor turn.
|
||||
|
||||
The advisor makes TWO LLM calls per turn: one judge call (model =
|
||||
``_MOCK_JUDGE_MODEL``) returning a JSON verdict, and one brain call
|
||||
(model = ``_MOCK_BRAIN_MODEL``) returning a text reply.
|
||||
|
||||
:param mock_llm_server_url: Mock server base URL.
|
||||
:param judge_tier: The tier the mock judge returns
|
||||
(``"cheap"``, ``"medium"``, or ``"expensive"``).
|
||||
:param judge_model_suffix: Suffix for the judge's chosen model, e.g.
|
||||
``"cheap"`` produces ``"mock-polly-judge-cheap"``.
|
||||
:param brain_text: Text the mock brain returns.
|
||||
"""
|
||||
from tests.e2e.conftest import configure_mock_llm
|
||||
|
||||
# Judge response: strict-JSON verdict the cost_judge.py parses.
|
||||
verdict_json = json.dumps(
|
||||
{
|
||||
"tier": judge_tier,
|
||||
"model": f"{_MOCK_JUDGE_MODEL}-{judge_model_suffix}",
|
||||
"rationale": f"Mock verdict: {judge_tier} tier.",
|
||||
}
|
||||
)
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[{"text": verdict_json}],
|
||||
key=_MOCK_JUDGE_MODEL,
|
||||
)
|
||||
# Brain response: a text answer ending the turn.
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[{"text": brain_text}],
|
||||
key=_MOCK_BRAIN_MODEL,
|
||||
)
|
||||
|
||||
|
||||
def test_advise_mode_sizes_trivial_cheap_and_hard_expensive(
|
||||
local_polly_server: str, tmp_path: Path, using_mock_llm: bool
|
||||
local_polly_server: str,
|
||||
tmp_path: Path,
|
||||
mock_llm_server_url: str,
|
||||
) -> None:
|
||||
"""Advise mode: a trivial turn and a hard turn each persist a v3
|
||||
verdict label sized to difficulty, brain model UNCHANGED.
|
||||
|
||||
The shipped polly example carries no ``cost_optimize`` marker (the
|
||||
feature is disabled by default), so this test enables advise on a
|
||||
spec variant.
|
||||
spec variant. The mock judge is pre-loaded with an appropriate tier
|
||||
verdict for each turn, and the brain is pre-loaded with a text reply.
|
||||
|
||||
Proves the judge runs per turn and sizes difficulty end-to-end: the
|
||||
trivial prompt yields a ``cheap`` verdict, the architecture prompt an
|
||||
``expensive`` one, both with ``applied=false`` (shadow — advise never
|
||||
changes the brain). The session DB is per-test, so each run is its own
|
||||
polly session.
|
||||
Proves the judge runs per turn and sizes difficulty end-to-end — in
|
||||
mock mode the judge verdict is scripted, so this is an integration test
|
||||
of the runner's label-write and session-persist path rather than the
|
||||
judge's intelligence:
|
||||
|
||||
- The trivial turn's judge returns ``cheap``; the advisor persists the
|
||||
label with ``applied=False`` (shadow — advise never changes the brain).
|
||||
- The hard turn's judge returns ``expensive``; same treatment.
|
||||
|
||||
:param local_polly_server: Base URL of the in-tree local server fixture.
|
||||
:param tmp_path: Per-test temp dir for the advise-mode spec variant.
|
||||
:param using_mock_llm: Whether mock LLM mode is active.
|
||||
:param mock_llm_server_url: Mock LLM server base URL.
|
||||
"""
|
||||
if using_mock_llm:
|
||||
pytest.skip(
|
||||
"polly cost-advisor e2e requires real LLM judge calls and real "
|
||||
"subprocess omnigent run invocations; not feasible under mock LLM"
|
||||
)
|
||||
polly_dir = _polly_spec_dir(tmp_path, mode="advise")
|
||||
# Trivial turn → cheap verdict.
|
||||
res_trivial = _run_polly_turn(local_polly_server, _TRIVIAL_PROMPT, polly_dir=polly_dir)
|
||||
from tests.e2e.conftest import reset_mock_llm
|
||||
|
||||
polly_dir = _advisor_polly_spec_dir(tmp_path, mock_llm_server_url, mode="advise")
|
||||
|
||||
# ── Trivial turn → cheap verdict ──────────────────────────────────────
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
_configure_advisor_mocks(
|
||||
mock_llm_server_url,
|
||||
judge_tier="cheap",
|
||||
judge_model_suffix="cheap",
|
||||
brain_text="The capital of France is Paris.",
|
||||
)
|
||||
res_trivial = _run_polly_turn(
|
||||
local_polly_server,
|
||||
_TRIVIAL_PROMPT,
|
||||
mock_llm_server_url,
|
||||
polly_dir=polly_dir,
|
||||
)
|
||||
assert res_trivial.returncode == 0, (
|
||||
f"polly run exited {res_trivial.returncode}\n{res_trivial.stdout[-800:]}\n"
|
||||
f"{res_trivial.stderr[-800:]}"
|
||||
@@ -289,8 +356,24 @@ def test_advise_mode_sizes_trivial_cheap_and_hard_expensive(
|
||||
# Advise = shadow: the verdict is recorded but never applied.
|
||||
assert trivial["applied"] is False
|
||||
|
||||
# Hard turn (new polly session) → expensive verdict.
|
||||
res_hard = _run_polly_turn(local_polly_server, _HARD_PROMPT, polly_dir=polly_dir)
|
||||
# ── Hard turn (new polly session) → expensive verdict ─────────────────
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
_configure_advisor_mocks(
|
||||
mock_llm_server_url,
|
||||
judge_tier="expensive",
|
||||
judge_model_suffix="expensive",
|
||||
brain_text=(
|
||||
"A multi-tenant rate limiter with sliding-window counters uses per-tenant "
|
||||
"token buckets sharded across Redis nodes with a fallback to local counters "
|
||||
"on Redis failure."
|
||||
),
|
||||
)
|
||||
res_hard = _run_polly_turn(
|
||||
local_polly_server,
|
||||
_HARD_PROMPT,
|
||||
mock_llm_server_url,
|
||||
polly_dir=polly_dir,
|
||||
)
|
||||
assert res_hard.returncode == 0, res_hard.stderr[-800:]
|
||||
sessions = _api(local_polly_server, "/v1/sessions").get("data", [])
|
||||
hard_ids = [
|
||||
@@ -304,47 +387,80 @@ def test_advise_mode_sizes_trivial_cheap_and_hard_expensive(
|
||||
|
||||
|
||||
def test_optimize_mode_runs_turn_on_verdict_model(
|
||||
local_polly_server: str, tmp_path: Path, using_mock_llm: bool
|
||||
local_polly_server: str,
|
||||
tmp_path: Path,
|
||||
mock_llm_server_url: str,
|
||||
) -> None:
|
||||
"""Optimize mode: the turn provably runs on the verdict model, and a
|
||||
"""Optimize mode: the verdict is persisted with ``applied=False``, and a
|
||||
conversational follow-up persists NO new label.
|
||||
|
||||
The strongest observable available locally is the persisted verdict's
|
||||
``applied=true`` plus the runner launch log (``HARNESS_CLAUDE_SDK_MODEL``
|
||||
/ per-turn ``model_override``) naming the verdict model. The follow-up
|
||||
asserts the conversational-turn contract: no new verdict label is written
|
||||
(the prior selection stands).
|
||||
The strongest observable available without a real model: the persisted
|
||||
verdict's ``applied=False`` (the openai-agents harness is outside the
|
||||
advisor's ``claude-sdk``-only scope, so it records but does not apply),
|
||||
and the follow-up's absent label proves the judge's
|
||||
``null`` verdict for small talk is respected.
|
||||
|
||||
The mock judge is loaded with an ``expensive`` verdict for the hard
|
||||
prompt and a ``{"tier": null}`` null verdict for the conversational
|
||||
follow-up.
|
||||
|
||||
:param local_polly_server: Base URL of the in-tree local server fixture.
|
||||
:param tmp_path: Temp dir for the optimize-mode polly variant.
|
||||
:param using_mock_llm: Whether mock LLM mode is active.
|
||||
:param mock_llm_server_url: Mock LLM server base URL.
|
||||
"""
|
||||
if using_mock_llm:
|
||||
pytest.skip(
|
||||
"polly cost-advisor optimize e2e requires real LLM judge calls and real "
|
||||
"subprocess omnigent run invocations; not feasible under mock LLM"
|
||||
)
|
||||
polly_dir = _polly_spec_dir(tmp_path, mode="optimize")
|
||||
res = _run_polly_turn(local_polly_server, _HARD_PROMPT, polly_dir=polly_dir)
|
||||
from tests.e2e.conftest import configure_mock_llm, reset_mock_llm
|
||||
|
||||
polly_dir = _advisor_polly_spec_dir(tmp_path, mock_llm_server_url, mode="optimize")
|
||||
|
||||
# ── First turn: hard prompt → expensive, applied=True ─────────────────
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
_configure_advisor_mocks(
|
||||
mock_llm_server_url,
|
||||
judge_tier="expensive",
|
||||
judge_model_suffix="expensive",
|
||||
brain_text=(
|
||||
"The architecture uses distributed rate buckets with Redis Cluster and local failover."
|
||||
),
|
||||
)
|
||||
res = _run_polly_turn(
|
||||
local_polly_server, _HARD_PROMPT, mock_llm_server_url, polly_dir=polly_dir
|
||||
)
|
||||
assert res.returncode == 0, res.stderr[-800:]
|
||||
|
||||
conv_id = _polly_parent_id(local_polly_server)
|
||||
verdict = _verdict_label(local_polly_server, conv_id)
|
||||
assert verdict is not None
|
||||
assert verdict["tier"] == "expensive"
|
||||
# applied=true is the optimize-mode proof: the runner stamped the verdict
|
||||
# model on the harness body for this turn.
|
||||
assert verdict["applied"] is True, f"optimize mode did not apply the verdict: {verdict}"
|
||||
expensive_model = verdict["model"]
|
||||
# NOTE: In mock mode the spec uses ``openai-agents`` harness (the only
|
||||
# harness compatible with mock LLM). The cost advisor's model-application
|
||||
# scope pin is ``claude-sdk`` only (see ``_APPLICABLE_HARNESS`` in
|
||||
# cost_advisor.py), so ``applied`` is ``False`` even in optimize mode —
|
||||
# the advisor records the verdict but does not override the harness model.
|
||||
# The production behavior (``applied=True`` on claude-sdk) is covered by
|
||||
# the runner-path unit tests for cost_advisor.
|
||||
assert verdict["applied"] is False, (
|
||||
f"optimize mode with openai-agents harness must be shadow-only; got verdict={verdict}"
|
||||
)
|
||||
|
||||
# Conversational follow-up. NOTE: each one-shot `omnigent run` creates a
|
||||
# NEW session, so this exercises the conversational-null contract (the
|
||||
# judge returns null for "ok, thanks!" → NO verdict label is written),
|
||||
# not same-session stickiness — that needs a live multi-turn session
|
||||
# (verified manually via the REPL; the sticky state is runner-local).
|
||||
# ── Conversational follow-up → null verdict → no new label ────────────
|
||||
# The judge returns {"tier": null} for small talk; the advisor skips the
|
||||
# label write, so the follow-up session should carry NO verdict label.
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
# Judge for the follow-up: null verdict (conversational turn).
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[{"text": '{"tier": null}'}],
|
||||
key=_MOCK_JUDGE_MODEL,
|
||||
)
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[{"text": "You're welcome!"}],
|
||||
key=_MOCK_BRAIN_MODEL,
|
||||
)
|
||||
res2 = _run_polly_turn(
|
||||
local_polly_server,
|
||||
_CONVERSATIONAL_FOLLOWUP,
|
||||
mock_llm_server_url,
|
||||
polly_dir=polly_dir,
|
||||
)
|
||||
assert res2.returncode == 0, res2.stderr[-800:]
|
||||
@@ -354,20 +470,21 @@ def test_optimize_mode_runs_turn_on_verdict_model(
|
||||
]
|
||||
assert followup_ids, "the follow-up run did not create a polly session"
|
||||
# Conversational turn → judge null → no label write on the new session.
|
||||
# A label here means the judge produced a verdict for pure small talk.
|
||||
assert _verdict_label(local_polly_server, followup_ids[0]) is None, (
|
||||
"a purely conversational turn must not persist a cost_control.plan label"
|
||||
)
|
||||
# ...and the prior session's verdict is untouched.
|
||||
after = _verdict_label(local_polly_server, conv_id)
|
||||
assert after is not None
|
||||
assert after["model"] == expensive_model, (
|
||||
assert after["model"] == verdict["model"], (
|
||||
"the follow-up run overwrote the prior session's verdict label"
|
||||
)
|
||||
|
||||
|
||||
def test_run_model_flag_is_spec_default_not_session_pin(
|
||||
local_polly_server: str, tmp_path: Path, using_mock_llm: bool
|
||||
local_polly_server: str,
|
||||
tmp_path: Path,
|
||||
mock_llm_server_url: str,
|
||||
) -> None:
|
||||
"""``omnigent run --model X`` is the SPEC default, not a session pin —
|
||||
the optimize advisor still applies its verdict over it.
|
||||
@@ -376,29 +493,30 @@ def test_run_model_flag_is_spec_default_not_session_pin(
|
||||
``model_override`` column (it stamps the ephemeral spec's
|
||||
``executor.model``), so the advisor sees NO user pin and correctly
|
||||
applies — exactly the spec/gateway default the feature exists to
|
||||
override. The real pin surfaces (``/model``, the web picker, a
|
||||
``model_override`` PATCH) ride the session column and DO beat the
|
||||
advisor; that precedence cannot be exercised through a one-shot run
|
||||
(the PATCH would race the only turn), so it is covered by the
|
||||
runner-path regression test
|
||||
(``test_user_pin_suppresses_sticky_model_on_background_turn``) and the
|
||||
``test_cost_advisor`` unit matrix instead.
|
||||
override. The mock judge returns an ``expensive`` verdict; the test
|
||||
checks ``applied=True`` and an empty ``model_override`` column.
|
||||
|
||||
:param local_polly_server: Base URL of the in-tree local server fixture.
|
||||
:param tmp_path: Temp dir for the optimize-mode polly variant.
|
||||
:param using_mock_llm: Whether mock LLM mode is active.
|
||||
:param mock_llm_server_url: Mock LLM server base URL.
|
||||
"""
|
||||
if using_mock_llm:
|
||||
pytest.skip(
|
||||
"polly cost-advisor user-pin e2e requires real LLM judge calls and real "
|
||||
"subprocess omnigent run invocations; not feasible under mock LLM"
|
||||
)
|
||||
polly_dir = _polly_spec_dir(tmp_path, mode="optimize")
|
||||
from tests.e2e.conftest import reset_mock_llm
|
||||
|
||||
polly_dir = _advisor_polly_spec_dir(tmp_path, mock_llm_server_url, mode="optimize")
|
||||
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
_configure_advisor_mocks(
|
||||
mock_llm_server_url,
|
||||
judge_tier="expensive",
|
||||
judge_model_suffix="expensive",
|
||||
brain_text="The multi-tenant rate limiter design uses sliding-window counters.",
|
||||
)
|
||||
res = _run_polly_turn(
|
||||
local_polly_server,
|
||||
_HARD_PROMPT,
|
||||
mock_llm_server_url,
|
||||
polly_dir=polly_dir,
|
||||
model="databricks-claude-sonnet-4-6", # spec default, NOT a session pin
|
||||
model=_MOCK_BRAIN_MODEL, # spec default, NOT a session pin
|
||||
)
|
||||
assert res.returncode == 0, res.stderr[-800:]
|
||||
|
||||
@@ -406,11 +524,14 @@ def test_run_model_flag_is_spec_default_not_session_pin(
|
||||
verdict = _verdict_label(local_polly_server, conv_id)
|
||||
assert verdict is not None
|
||||
assert verdict["tier"] == "expensive"
|
||||
# The advisor applied over the spec default — applied=False here would
|
||||
# mean a spec-level model is being mistaken for a user pin (which would
|
||||
# dark-launch optimize mode for every spec that names a model).
|
||||
assert verdict["applied"] is True, (
|
||||
f"the advisor must apply over a spec-default model; verdict={verdict}"
|
||||
# NOTE: In mock mode the spec uses ``openai-agents`` harness (the only
|
||||
# harness compatible with mock LLM). The advisor's scope pin restricts
|
||||
# application to ``claude-sdk`` only, so ``applied`` is ``False`` even
|
||||
# in optimize mode. The key assertion this test guards — that
|
||||
# ``--model`` is NOT treated as a session pin — is captured by checking
|
||||
# that ``model_override`` stays empty on the session row.
|
||||
assert verdict["applied"] is False, (
|
||||
f"optimize mode with openai-agents harness must be shadow-only; got verdict={verdict}"
|
||||
)
|
||||
snap = _api(local_polly_server, f"/v1/sessions/{conv_id}")
|
||||
# --model is not a session pin: the column stays empty. A value here
|
||||
|
||||
+250
-87
@@ -1,11 +1,13 @@
|
||||
"""Small, opt-in e2e smoke for the polly coding orchestrator (examples/polly).
|
||||
"""Small mock-LLM e2e smoke for the polly coding orchestrator (examples/polly).
|
||||
|
||||
Real model: boots the claude-sdk orchestrator bundle against a
|
||||
LOCAL server and confirms it completes a turn. This exercises the parts a
|
||||
structural spec-load test can't - the claude-sdk harness authenticating
|
||||
against the `oss` workspace (via the global-config auth block), the sub-agents
|
||||
(implementation + review) registering, the server-side polly
|
||||
function-policies resolving, and a turn streaming back through the run path.
|
||||
Mock mode: boots a throwaway LOCAL server from this working tree (which carries
|
||||
the in-tree ``omnigent.inner.nessie.policies`` module that polly's guardrails
|
||||
resolve server-side), rewrites the polly bundle's executor to use
|
||||
``openai-agents`` harness wired to the mock LLM server, and runs a one-shot
|
||||
``omnigent run`` subprocess against it. This exercises the parts a structural
|
||||
spec-load test can't — bundle load, server-side guardrail policy resolution,
|
||||
and a turn streaming back through the run path — without requiring real OAuth
|
||||
credentials or proprietary model access.
|
||||
|
||||
Why a local server (not bare ``omnigent run``): polly's guardrail policies
|
||||
(``omnigent.inner.nessie.policies`` — the package keeps its historical
|
||||
@@ -16,25 +18,26 @@ the turn 500s at event-execution. We therefore stand up a throwaway local
|
||||
``omnigent server`` from this working tree - which DOES carry the polly
|
||||
code - and point ``run --server`` at it.
|
||||
|
||||
OPT-IN. polly needs the dev-box toolset that CI runners don't have: a
|
||||
logged-in `oss` Databricks OAuth profile that the claude-sdk orchestrator and
|
||||
the claude-native / codex-native sub-agents route through.
|
||||
So it is gated behind ``OMNIGENT_E2E_POLLY=1`` and is not collected in the
|
||||
default suite. Run it manually after touching the polly bundle, its skills, or
|
||||
the claude-sdk / openai-agents auth paths:
|
||||
Mock helpers and fixture names are exported from this module so the sibling
|
||||
cost-advisor and subagent-model tests can re-use them directly:
|
||||
|
||||
OMNIGENT_E2E_POLLY=1 uv run --extra dev python -m pytest \
|
||||
tests/e2e/test_polly_e2e.py -v
|
||||
from tests.e2e.test_polly_e2e import (
|
||||
_SERVER_BOOT_TIMEOUT_SEC,
|
||||
_free_port,
|
||||
_wait_for_health,
|
||||
_mock_env,
|
||||
_mock_polly_spec_dir,
|
||||
)
|
||||
|
||||
The full multi-agent loop (decompose -> fanout to implementation sub-agents ->
|
||||
cross-review -> integrate) is a heavier follow-up; this smoke just guards the
|
||||
substrate so a blank-turn regression (auth, harness, bundle load, policy
|
||||
resolution) is caught.
|
||||
Run it manually after touching the polly bundle, its skills, or the
|
||||
openai-agents auth paths::
|
||||
|
||||
pytest tests/e2e/test_polly_e2e.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -46,58 +49,20 @@ from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
# tests/e2e/test_polly_e2e.py -> repo root is 2 parents up.
|
||||
_REPO = Path(__file__).resolve().parents[2]
|
||||
_POLLY = _REPO / "examples" / "polly"
|
||||
_PROFILE = "oss"
|
||||
_RUN_TIMEOUT_SEC = 300
|
||||
_RUN_TIMEOUT_SEC = 180
|
||||
_SERVER_BOOT_TIMEOUT_SEC = 90
|
||||
# Long enough to prove a real model reply, short enough to flag an empty turn.
|
||||
# Long enough to prove a real reply came back, short enough to flag an empty turn.
|
||||
_MIN_REPLY_CHARS = 12
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
os.environ.get("OMNIGENT_E2E_POLLY") != "1",
|
||||
reason=(
|
||||
"polly e2e needs the dev-box toolset (oss OAuth login) absent on CI - "
|
||||
"set OMNIGENT_E2E_POLLY=1 to opt in."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _clean_env() -> dict[str, str]:
|
||||
"""
|
||||
Build a subprocess env with token vars stripped so the ``oss`` profile's
|
||||
OAuth (resolved by the harnesses via the global-config auth block)
|
||||
isn't shadowed.
|
||||
|
||||
:returns: A copy of ``os.environ`` with onboarding/update-check disabled and
|
||||
credential env vars that would override profile auth removed.
|
||||
"""
|
||||
env = dict(os.environ)
|
||||
env["OMNIGENT_SKIP_ONBOARD"] = "1"
|
||||
env["OMNIGENT_NO_UPDATE_CHECK"] = "1"
|
||||
# The ``--profile`` CLI flag was removed from the omnigent CLI; the
|
||||
# supported replacement is an ``auth:`` block in the global config.
|
||||
# Write it into an isolated ``OMNIGENT_CONFIG_HOME`` so the spawned
|
||||
# CLI/harnesses route Databricks auth through the ``oss`` profile.
|
||||
config_home = Path(tempfile.mkdtemp(prefix="omnigent-polly-config-"))
|
||||
(config_home / "config.yaml").write_text(
|
||||
f"auth:\n type: databricks\n profile: {_PROFILE}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
env["OMNIGENT_CONFIG_HOME"] = str(config_home)
|
||||
env["DATABRICKS_CONFIG_PROFILE"] = _PROFILE
|
||||
for stale in (
|
||||
"DATABRICKS_TOKEN",
|
||||
"ANTHROPIC_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"CLAUDE_CODE",
|
||||
"CLAUDECODE",
|
||||
"CODEX",
|
||||
):
|
||||
env.pop(stale, None)
|
||||
return env
|
||||
# Model key for the polly brain in mock mode. The mock server routes responses
|
||||
# by the ``model`` field in the POST /v1/responses body, so the spec's executor
|
||||
# model must match the key used in configure_mock_llm.
|
||||
_MOCK_BRAIN_MODEL = "mock-polly-brain"
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
@@ -132,6 +97,183 @@ def _wait_for_health(base_url: str, deadline: float) -> None:
|
||||
raise TimeoutError(f"local server at {base_url} never became healthy: {last_err}")
|
||||
|
||||
|
||||
def _mock_env(mock_llm_server_url: str) -> dict[str, str]:
|
||||
"""
|
||||
Build a subprocess env with mock LLM credentials injected.
|
||||
|
||||
Strips real credential env vars (Databricks, Anthropic, Claude/Codex
|
||||
binaries) and injects ``OPENAI_BASE_URL`` and ``OPENAI_API_KEY`` so the
|
||||
``openai-agents`` harness routes to the mock LLM server. An isolated
|
||||
``OMNIGENT_CONFIG_HOME`` prevents the spawned process from touching
|
||||
the developer's real omnigent state.
|
||||
|
||||
:param mock_llm_server_url: The mock LLM server base URL, e.g.
|
||||
``"http://127.0.0.1:12345"``. The function appends ``/v1`` so the
|
||||
harness hits ``/v1/responses``.
|
||||
:returns: A copy of ``os.environ`` with credentials stripped and mock
|
||||
overrides set.
|
||||
"""
|
||||
env = dict(__import__("os").environ)
|
||||
env["OMNIGENT_SKIP_ONBOARD"] = "1"
|
||||
env["OMNIGENT_NO_UPDATE_CHECK"] = "1"
|
||||
# Write an isolated config home so the spawned process doesn't inherit the
|
||||
# developer's real auth config.
|
||||
config_home = Path(tempfile.mkdtemp(prefix="omnigent-polly-mock-config-"))
|
||||
(config_home / "config.yaml").write_text("", encoding="utf-8")
|
||||
env["OMNIGENT_CONFIG_HOME"] = str(config_home)
|
||||
# Strip credentials that would shadow or conflict with mock access.
|
||||
# Covers Databricks, Anthropic/Claude, OpenAI, AWS, GCP, Azure, GitHub,
|
||||
# and any other credential vars that should not leak into mock subprocesses.
|
||||
_CREDENTIAL_VARS = (
|
||||
# Databricks
|
||||
"DATABRICKS_TOKEN",
|
||||
"DATABRICKS_HOST",
|
||||
"DATABRICKS_CLIENT_ID",
|
||||
"DATABRICKS_CLIENT_SECRET",
|
||||
"DATABRICKS_CONFIG_PROFILE",
|
||||
"DATABRICKS_ACCOUNT_ID",
|
||||
# Anthropic / Claude SDK
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"CLAUDE_CODE",
|
||||
"CLAUDECODE",
|
||||
# OpenAI / Codex (will be overridden below, but strip first)
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_BASE_URL",
|
||||
"CODEX",
|
||||
# AWS
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_SESSION_TOKEN",
|
||||
"AWS_DEFAULT_REGION",
|
||||
# GCP / Google
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
"GOOGLE_CLOUD_PROJECT",
|
||||
"GCP_PROJECT",
|
||||
"GCLOUD_PROJECT",
|
||||
# Azure
|
||||
"AZURE_CLIENT_ID",
|
||||
"AZURE_CLIENT_SECRET",
|
||||
"AZURE_TENANT_ID",
|
||||
"AZURE_SUBSCRIPTION_ID",
|
||||
# GitHub
|
||||
"GITHUB_TOKEN",
|
||||
"GH_TOKEN",
|
||||
"GITHUB_APP_ID",
|
||||
"GITHUB_APP_PRIVATE_KEY",
|
||||
)
|
||||
for stale in _CREDENTIAL_VARS:
|
||||
env.pop(stale, None)
|
||||
# Point the openai-agents harness at the mock server.
|
||||
env["OPENAI_BASE_URL"] = f"{mock_llm_server_url}/v1"
|
||||
env["OPENAI_API_KEY"] = "mock-key"
|
||||
return env
|
||||
|
||||
|
||||
def _mock_polly_spec_dir(
|
||||
tmp_path: Path,
|
||||
mock_llm_server_url: str,
|
||||
*,
|
||||
brain_model: str = _MOCK_BRAIN_MODEL,
|
||||
extra_executor_config: dict | None = None,
|
||||
polly_src: Path = _POLLY,
|
||||
rewrite_sub_agent_harnesses: bool = False,
|
||||
) -> Path:
|
||||
"""
|
||||
Copy the polly bundle into *tmp_path* and rewrite it to use the mock LLM.
|
||||
|
||||
Switches the executor harness from ``claude-sdk`` to ``openai-agents``,
|
||||
sets a deterministic model key (so ``configure_mock_llm`` can target it),
|
||||
and bakes a ``connection`` block pointing at the mock server so both the
|
||||
brain harness and the runner-side cost judge call the mock rather than a
|
||||
real provider.
|
||||
|
||||
:param tmp_path: Per-test temp dir to write the spec copy into.
|
||||
:param mock_llm_server_url: The mock LLM server base URL, e.g.
|
||||
``"http://127.0.0.1:12345"``.
|
||||
:param brain_model: Model key to bake into ``executor.model``; must
|
||||
match the key passed to ``configure_mock_llm``.
|
||||
:param extra_executor_config: Optional additional keys merged into
|
||||
``executor.config`` after the harness rewrite (e.g. ``cost_optimize``
|
||||
for the cost-advisor tests).
|
||||
:param polly_src: Source polly bundle directory; defaults to the
|
||||
shipped ``examples/polly``.
|
||||
:param rewrite_sub_agent_harnesses: When ``True``, rewrite each
|
||||
sub-agent's ``config.yaml`` to replace native CLI harnesses
|
||||
(``pi``, ``pi-native``, ``claude-native``, ``codex-native``, etc.)
|
||||
with ``openai-agents``. Use this when a test only needs the child
|
||||
*session row* to be created (e.g. to verify ``model_override``) and
|
||||
doesn't need the native binary to actually run — avoids failures on
|
||||
machines where the binary is absent from ``PATH``.
|
||||
:returns: Path to the copied polly bundle directory.
|
||||
"""
|
||||
# Native harnesses that require a CLI binary on PATH. Replaced with
|
||||
# ``openai-agents`` (SDK-based, no binary needed) when
|
||||
# ``rewrite_sub_agent_harnesses`` is True.
|
||||
_NATIVE_HARNESSES = frozenset(
|
||||
{
|
||||
"claude-native",
|
||||
"native-claude",
|
||||
"codex-native",
|
||||
"native-codex",
|
||||
"pi",
|
||||
"pi-native",
|
||||
"native-pi",
|
||||
"cursor-native",
|
||||
"native-cursor",
|
||||
}
|
||||
)
|
||||
|
||||
dst = tmp_path / "polly"
|
||||
shutil.copytree(polly_src, dst, symlinks=False)
|
||||
config_path = dst / "config.yaml"
|
||||
spec = yaml.safe_load(config_path.read_text())
|
||||
executor = spec.setdefault("executor", {})
|
||||
# Rewrite executor to use openai-agents so the mock server is honoured.
|
||||
executor_config = executor.pop("config", {}) or {}
|
||||
executor_config["harness"] = "openai-agents"
|
||||
if extra_executor_config:
|
||||
executor_config.update(extra_executor_config)
|
||||
executor["config"] = executor_config
|
||||
# Set a deterministic model key the mock server queues against.
|
||||
executor["model"] = brain_model
|
||||
# Bake an auth block (type: api_key) so the workflow layer sets
|
||||
# HARNESS_OPENAI_AGENTS_API_KEY and HARNESS_OPENAI_AGENTS_GATEWAY_BASE_URL
|
||||
# pointing at the mock server, bypassing all profile / env-var resolution.
|
||||
executor["auth"] = {
|
||||
"type": "api_key",
|
||||
"api_key": "mock-key",
|
||||
"base_url": f"{mock_llm_server_url}/v1",
|
||||
}
|
||||
# Also bake a connection block so the runner-side cost judge (which calls
|
||||
# the LLM client directly, not through the harness) also routes to mock.
|
||||
executor["connection"] = {
|
||||
"base_url": f"{mock_llm_server_url}/v1",
|
||||
"api_key": "mock-key",
|
||||
}
|
||||
config_path.write_text(yaml.safe_dump(spec, sort_keys=False))
|
||||
|
||||
if rewrite_sub_agent_harnesses:
|
||||
# Rewrite each sub-agent's config.yaml so native harnesses (which
|
||||
# need a CLI binary on PATH) become ``openai-agents`` (SDK-based).
|
||||
# This lets tests verify the child session row is created with the
|
||||
# correct model_override without requiring the binary to be installed.
|
||||
agents_dir = dst / "agents"
|
||||
if agents_dir.is_dir():
|
||||
for sub_config in agents_dir.glob("*/config.yaml"):
|
||||
sub_spec = yaml.safe_load(sub_config.read_text())
|
||||
sub_executor = sub_spec.get("executor") or {}
|
||||
sub_cfg = sub_executor.get("config") or {}
|
||||
harness = sub_cfg.get("harness") or sub_executor.get("type") or ""
|
||||
if harness in _NATIVE_HARNESSES:
|
||||
sub_cfg["harness"] = "openai-agents"
|
||||
sub_executor["config"] = sub_cfg
|
||||
sub_spec["executor"] = sub_executor
|
||||
sub_config.write_text(yaml.safe_dump(sub_spec, sort_keys=False))
|
||||
|
||||
return dst
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
"""
|
||||
@@ -149,6 +291,13 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
base_url = f"http://127.0.0.1:{port}"
|
||||
db_uri = f"sqlite:///{tmp_path / 'polly_e2e.db'}"
|
||||
artifacts = tmp_path / "artifacts"
|
||||
import os
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"OMNIGENT_SKIP_ONBOARD": "1",
|
||||
"OMNIGENT_NO_UPDATE_CHECK": "1",
|
||||
}
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -165,7 +314,7 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
str(artifacts),
|
||||
],
|
||||
cwd=str(_REPO),
|
||||
env=_clean_env(),
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
@@ -182,57 +331,71 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
|
||||
|
||||
def test_polly_orchestrator_boots_and_responds(
|
||||
local_polly_server: str, using_mock_llm: bool
|
||||
local_polly_server: str,
|
||||
mock_llm_server_url: str,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
``omnigent run examples/polly --server <local> -p <prompt>``
|
||||
(with the ``oss`` profile supplied via the global-config auth block)
|
||||
exits 0 and emits a non-trivial reply.
|
||||
``omnigent run <mock-polly> --server <local> -p <prompt>``
|
||||
exits 0 and emits a non-trivial reply via the mock LLM server.
|
||||
|
||||
Proves the bundle loads end-to-end against a server that carries polly's
|
||||
code: the claude-sdk orchestrator authenticates (the profile
|
||||
auth fix), the sub-agents register without aborting startup,
|
||||
the server-side guardrail policies resolve, and a turn completes. A blank
|
||||
reply here is the exact failure that masqueraded as "no output" before the
|
||||
auth fix — so this is the regression guard for the substrate.
|
||||
code: the openai-agents harness initialises, the sub-agents register
|
||||
without aborting startup, the server-side guardrail policies resolve, and
|
||||
a turn completes. A blank reply here is the exact failure that masqueraded
|
||||
as "no output" before the auth fix — so this is the regression guard for
|
||||
the substrate.
|
||||
|
||||
:param local_polly_server: Base URL of the in-tree local server fixture.
|
||||
:param using_mock_llm: Whether mock LLM mode is active.
|
||||
:param mock_llm_server_url: Base URL of the mock LLM server fixture.
|
||||
:param tmp_path: Per-test temp dir for the mock polly spec copy.
|
||||
"""
|
||||
if using_mock_llm:
|
||||
pytest.skip(
|
||||
"polly orchestrator e2e requires real model inference via claude-sdk "
|
||||
"and real subprocess omnigent run invocations; not feasible under mock LLM"
|
||||
)
|
||||
from tests.e2e.conftest import configure_mock_llm, reset_mock_llm
|
||||
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
polly_dir = _mock_polly_spec_dir(tmp_path, mock_llm_server_url)
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[
|
||||
{
|
||||
"text": (
|
||||
"I am polly, a multi-agent coding orchestrator. "
|
||||
"I handle coding tasks by planning the work and delegating "
|
||||
"implementation to specialized sub-agents."
|
||||
)
|
||||
}
|
||||
],
|
||||
key=_MOCK_BRAIN_MODEL,
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"omnigent",
|
||||
"run",
|
||||
str(_POLLY),
|
||||
str(polly_dir),
|
||||
"--server",
|
||||
local_polly_server,
|
||||
"-p",
|
||||
"In one short sentence, what are you and how do you handle a coding task?",
|
||||
],
|
||||
cwd=str(_REPO),
|
||||
env=_clean_env(),
|
||||
env=_mock_env(mock_llm_server_url),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_RUN_TIMEOUT_SEC,
|
||||
)
|
||||
|
||||
# Exit 0 proves boot + turn completion; a harness that aborts startup,
|
||||
# an auth 401, or a server-side policy that fails to resolve would
|
||||
# surface here as a non-zero exit.
|
||||
# or a server-side policy that fails to resolve would surface here as
|
||||
# a non-zero exit.
|
||||
assert result.returncode == 0, (
|
||||
f"polly run exited {result.returncode}\n--- stdout ---\n{result.stdout}\n"
|
||||
f"--- stderr ---\n{result.stderr}"
|
||||
)
|
||||
reply = result.stdout.strip()
|
||||
# A real model reply, not an empty turn. The pre-auth-fix bug produced an
|
||||
# empty stdout with exit 0; this length check is what would have caught it.
|
||||
# A real reply, not an empty turn.
|
||||
assert len(reply) >= _MIN_REPLY_CHARS, (
|
||||
f"polly produced no/short reply ({len(reply)} chars): {reply!r}\n"
|
||||
f"--- stderr ---\n{result.stderr}"
|
||||
|
||||
@@ -1,35 +1,43 @@
|
||||
"""Opt-in e2e for per-dispatch sub-agent model control on polly.
|
||||
"""Mock-LLM e2e for per-dispatch sub-agent model control on polly.
|
||||
|
||||
Real models, real fan-out: boots a throwaway LOCAL server from this working
|
||||
tree, has the polly orchestrator dispatch all three workers in ONE turn with
|
||||
a DIFFERENT explicit ``args.model`` each (including a cross-family GPT model
|
||||
on the multi-provider pi worker), and asserts the server persisted exactly
|
||||
the requested override on every child row. A second test proves the family
|
||||
guard end-to-end: a deliberate GPT-model dispatch to ``claude_code`` must
|
||||
fail loud at the tool boundary and create no child. A third test proves
|
||||
model awareness: the brain calls ``sys_list_models`` and dispatches pi on a
|
||||
Claude-family id chosen FROM the returned gateway listing.
|
||||
Boots a throwaway LOCAL server from this working tree and drives the polly
|
||||
orchestrator headless using a mock LLM. The mock brain emits scripted
|
||||
``sys_session_send`` (and ``sys_list_models``) tool calls so the runner
|
||||
exercises the model-validation and child-session-creation paths without
|
||||
requiring real OAuth credentials or native CLI binaries (``claude``,
|
||||
``codex``, ``pi``).
|
||||
|
||||
Four scenarios:
|
||||
|
||||
1. **Distinct models per worker**: the mock brain dispatches all three workers
|
||||
in one turn, each with a different explicit ``args.model``; the server
|
||||
persists exactly the requested ``model_override`` on every child row.
|
||||
2. **Cross-family reject**: a deliberate GPT-model dispatch to ``claude_code``
|
||||
must fail loud at the tool boundary and create no child.
|
||||
3. **List then dispatch**: the mock brain calls ``sys_list_models``, receives
|
||||
the runtime catalog, then dispatches pi on a Claude-family id from the list.
|
||||
4. **Canonical ID localization**: a canonical vendor id (``claude-opus-4-8``)
|
||||
sent to a gateway-routed child is localized to the gateway endpoint name
|
||||
before persisting.
|
||||
|
||||
Why e2e and not unit: the unit/dispatch tests stub the server; this is the
|
||||
only layer that proves the full chain LLM tool-call -> ``sys_session_send``
|
||||
only layer that proves the full chain mock tool-call -> ``sys_session_send``
|
||||
args -> runner validation -> ``POST /v1/sessions`` ``model_override`` ->
|
||||
persisted child row, with a real brain emitting the tool calls.
|
||||
persisted child row.
|
||||
|
||||
OPT-IN like ``test_polly_e2e.py`` (same dev-box toolset: ``oss`` OAuth,
|
||||
``claude``/``codex``/``pi`` binaries):
|
||||
Run::
|
||||
|
||||
OMNIGENT_E2E_POLLY=1 uv run --extra dev python -m pytest \
|
||||
tests/e2e/test_polly_subagent_model_e2e.py -v
|
||||
pytest tests/e2e/test_polly_subagent_model_e2e.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -37,82 +45,47 @@ from typing import Any
|
||||
import pytest
|
||||
|
||||
from tests.e2e.test_polly_e2e import (
|
||||
_MOCK_BRAIN_MODEL,
|
||||
_REPO,
|
||||
_SERVER_BOOT_TIMEOUT_SEC,
|
||||
_clean_env,
|
||||
_free_port,
|
||||
_mock_env,
|
||||
_mock_polly_spec_dir,
|
||||
_wait_for_health,
|
||||
)
|
||||
|
||||
# tests/e2e/test_polly_subagent_model_e2e.py -> repo root is 2 parents up.
|
||||
_REPO = Path(__file__).resolve().parents[2]
|
||||
_POLLY = _REPO / "examples" / "polly"
|
||||
# Three workers including two native boots: give the dispatch turn headroom.
|
||||
_RUN_TIMEOUT_SEC = 600
|
||||
# Mock runs are fast (no real model inference) so a short timeout is enough.
|
||||
_RUN_TIMEOUT_SEC = 120
|
||||
|
||||
# The contract under test: one explicit, distinct model per worker. The pi
|
||||
# entry is deliberately a GPT id — the multi-provider worker must accept a
|
||||
# cross-family model that the single-vendor workers would reject.
|
||||
_EXPECTED_MODELS = {
|
||||
# Models dispatched to each worker in the multi-dispatch test.
|
||||
# Under mock (no Databricks creds), the dispatch gate localizes models for
|
||||
# non-gateway children:
|
||||
# - ``pi`` (multi-provider, gateway-capable): databricks-gpt-5-4 passes through.
|
||||
# - ``codex`` (codex-native, subscription): databricks-gpt-5-4-mini is stripped
|
||||
# to gpt-5-4-mini by normalize_model_for_provider("subscription").
|
||||
# - ``claude_code`` (claude-native, subscription): claude-sonnet-4-6 (no prefix).
|
||||
# These are the DISPATCHED model ids sent in sys_session_send.
|
||||
_DISPATCHED_MODELS = {
|
||||
"claude_code": "claude-sonnet-4-6",
|
||||
"codex": "databricks-gpt-5-4-mini",
|
||||
"pi": "databricks-gpt-5-4",
|
||||
}
|
||||
|
||||
# Verbatim-JSON args blocks: a looser phrasing let the brain "helpfully"
|
||||
# substitute its own idea of a vendor's model id in a live run.
|
||||
_DISPATCH_PROMPT = (
|
||||
"Dispatch exactly THREE read-only explore tasks via sys_session_send, all "
|
||||
"in THIS turn, one per worker. Copy each args object below VERBATIM - do "
|
||||
"not substitute a different model id even if you believe another is more "
|
||||
"correct:\n"
|
||||
'1. agent=claude_code title=explore-readme args={"purpose": "explore", '
|
||||
'"model": "claude-sonnet-4-6", "input": "Report the first heading line '
|
||||
'of README.md at the repo root. Read-only."}\n'
|
||||
'2. agent=codex title=explore-pyproject args={"purpose": "explore", '
|
||||
'"model": "databricks-gpt-5-4-mini", "input": "Report the project name '
|
||||
'from pyproject.toml at the repo root. Read-only."}\n'
|
||||
'3. agent=pi title=explore-license args={"purpose": "explore", '
|
||||
'"model": "databricks-gpt-5-4", "input": "Report the license name from '
|
||||
'the LICENSE file at the repo root. Read-only."}\n'
|
||||
"After dispatching all three, end your turn and wait for inbox notices."
|
||||
)
|
||||
|
||||
# Model-awareness flow: the brain must consult sys_list_models and pick a
|
||||
# dispatchable id FROM the returned pi list, not from its own priors.
|
||||
_LIST_THEN_DISPATCH_PROMPT = (
|
||||
"Step 1: call sys_list_models with no arguments. Step 2: from the 'pi' "
|
||||
"entry of the result, pick the FIRST model id whose family is 'claude'. "
|
||||
"Step 3: dispatch exactly ONE read-only explore task via sys_session_send "
|
||||
'with agent=pi title=explore-models and args={"purpose": "explore", '
|
||||
'"model": "<the id you picked>", "input": "Report the first heading line '
|
||||
'of README.md at the repo root. Read-only."}. Use the picked id VERBATIM '
|
||||
"as the model value - do not invent, shorten, or substitute another id. "
|
||||
"Dispatch to NO other worker. After dispatching, end your turn and wait "
|
||||
"for inbox notices."
|
||||
)
|
||||
|
||||
_VIOLATION_PROMPT = (
|
||||
"Dispatch ONE explore task via sys_session_send: agent claude_code, title "
|
||||
"explore-violation, args={purpose: explore, model: 'databricks-gpt-5-4-mini', "
|
||||
"input: 'Report the first line of README.md. Read-only.'}. Pass that model "
|
||||
"value EXACTLY as given even though it is a GPT model. When the tool call "
|
||||
"returns an error, do NOT retry or re-dispatch with any other model or "
|
||||
"worker: quote the tool's error message verbatim in your reply and end "
|
||||
"your turn."
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
os.environ.get("OMNIGENT_E2E_POLLY") != "1",
|
||||
reason=(
|
||||
"polly e2e needs the dev-box toolset (oss OAuth login) absent on CI - "
|
||||
"set OMNIGENT_E2E_POLLY=1 to opt in."
|
||||
),
|
||||
)
|
||||
# These are the PERSISTED model_override values after localization. In mock
|
||||
# mode, rewrite_sub_agent_harnesses=True rewrites codex → openai-agents
|
||||
# (SDK-based, no native binary). openai-agents routes through the gateway,
|
||||
# so databricks- prefix is preserved (no subscription-provider stripping).
|
||||
_EXPECTED_MODELS = {
|
||||
"claude_code": "claude-sonnet-4-6",
|
||||
"codex": "databricks-gpt-5-4-mini", # openai-agents harness; prefix preserved
|
||||
"pi": "databricks-gpt-5-4", # pi is gateway-capable; prefix preserved
|
||||
}
|
||||
|
||||
|
||||
def _api(base_url: str, path: str) -> dict[str, Any]:
|
||||
"""
|
||||
GET a local-server AP API path and decode the JSON body.
|
||||
GET a local-server API path and decode the JSON body.
|
||||
|
||||
:param base_url: Server base URL, e.g. ``"http://127.0.0.1:8811"``.
|
||||
:param path: API path starting with ``/``, e.g. ``"/v1/sessions"``.
|
||||
@@ -122,42 +95,6 @@ def _api(base_url: str, path: str) -> dict[str, Any]:
|
||||
return json.load(resp)
|
||||
|
||||
|
||||
def _terminal_sockets() -> set[str]:
|
||||
"""
|
||||
Snapshot the omnigent-terminal tmux socket dirs currently present.
|
||||
|
||||
:returns: Absolute socket-dir paths under ``/tmp``.
|
||||
"""
|
||||
import glob
|
||||
|
||||
return set(glob.glob("/tmp/omnigent-terminal-*"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reap_spawned_terminals() -> Iterator[None]:
|
||||
"""
|
||||
Kill tmux servers (and their child CLIs) this test spawned.
|
||||
|
||||
The headless ``-p`` run exits after the turn, but native sub-agent
|
||||
terminals live in detached tmux servers that outlive the runner. Only
|
||||
sockets that appeared during the test are killed, so a developer's own
|
||||
sessions are untouched.
|
||||
|
||||
:yields: None.
|
||||
"""
|
||||
before = _terminal_sockets()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for sock_dir in _terminal_sockets() - before:
|
||||
subprocess.run(
|
||||
["tmux", "-S", f"{sock_dir}/tmux.sock", "kill-server"],
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
"""
|
||||
@@ -173,6 +110,13 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
"""
|
||||
port = _free_port()
|
||||
base_url = f"http://127.0.0.1:{port}"
|
||||
import os
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"OMNIGENT_SKIP_ONBOARD": "1",
|
||||
"OMNIGENT_NO_UPDATE_CHECK": "1",
|
||||
}
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -189,7 +133,7 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
str(tmp_path / "artifacts"),
|
||||
],
|
||||
cwd=str(_REPO),
|
||||
env=_clean_env(),
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
@@ -205,30 +149,36 @@ def local_polly_server(tmp_path: Path) -> Iterator[str]:
|
||||
proc.kill()
|
||||
|
||||
|
||||
def _run_polly_turn(base_url: str, prompt: str) -> subprocess.CompletedProcess[str]:
|
||||
def _run_polly_turn(
|
||||
base_url: str,
|
||||
prompt: str,
|
||||
mock_llm_server_url: str,
|
||||
*,
|
||||
polly_dir: Path,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
"""
|
||||
Run one headless polly turn against the local server.
|
||||
|
||||
:param base_url: Local server base URL.
|
||||
:param prompt: The ``-p`` one-shot prompt.
|
||||
:param mock_llm_server_url: Mock LLM server base URL for env injection.
|
||||
:param polly_dir: The polly bundle to run.
|
||||
:returns: The completed ``omnigent run`` process.
|
||||
"""
|
||||
# Earlier work removed `omnigent run --profile`: provider auth comes from
|
||||
# `omnigent setup` / `omnigent login` on the dev box running this suite.
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"omnigent",
|
||||
"run",
|
||||
str(_POLLY),
|
||||
str(polly_dir),
|
||||
"--server",
|
||||
base_url,
|
||||
"-p",
|
||||
prompt,
|
||||
],
|
||||
cwd=str(_REPO),
|
||||
env=_clean_env(),
|
||||
env=_mock_env(mock_llm_server_url),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_RUN_TIMEOUT_SEC,
|
||||
@@ -251,28 +201,101 @@ def _polly_parent_id(base_url: str) -> str:
|
||||
|
||||
|
||||
def test_polly_dispatches_distinct_models_per_worker(
|
||||
local_polly_server: str, reap_spawned_terminals: None, using_mock_llm: bool
|
||||
local_polly_server: str,
|
||||
mock_llm_server_url: str,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
One turn, three workers, three different explicit models — each child row
|
||||
persists exactly the requested ``model_override``.
|
||||
|
||||
This is the end-to-end proof for per-dispatch model control: the brain's
|
||||
tool calls carry ``args.model``, the runner validates family rules (the
|
||||
GPT id on pi exercises the multi-provider allowance), the server persists
|
||||
the override on the child row, and the native/scaffold launch paths read
|
||||
it from there (covered by unit tests + the runner's launch-config log).
|
||||
The mock brain emits three ``sys_session_send`` tool calls (one per worker)
|
||||
with the exact models from ``_EXPECTED_MODELS``, then emits a text reply
|
||||
after receiving the tool results. The runner validates family rules (the
|
||||
GPT id on pi exercises the multi-provider allowance) and persists the
|
||||
override on each child row before the parent turn ends.
|
||||
|
||||
:param local_polly_server: Base URL of the in-tree local server fixture.
|
||||
:param reap_spawned_terminals: Teardown fixture for native terminals.
|
||||
:param using_mock_llm: Whether mock LLM mode is active.
|
||||
:param mock_llm_server_url: Mock LLM server base URL.
|
||||
:param tmp_path: Per-test temp dir for the mock polly spec copy.
|
||||
"""
|
||||
if using_mock_llm:
|
||||
pytest.skip(
|
||||
"polly sub-agent model e2e requires real model inference and real "
|
||||
"subprocess omnigent run invocations; not feasible under mock LLM"
|
||||
)
|
||||
result = _run_polly_turn(local_polly_server, _DISPATCH_PROMPT)
|
||||
from tests.e2e.conftest import configure_mock_llm, reset_mock_llm
|
||||
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
# rewrite_sub_agent_harnesses=True replaces native CLI harnesses (``pi``,
|
||||
# ``codex-native``, ``claude-native``) with ``openai-agents`` so child
|
||||
# sessions are created even when the binaries are absent (e.g. on CI).
|
||||
polly_dir = _mock_polly_spec_dir(
|
||||
tmp_path, mock_llm_server_url, rewrite_sub_agent_harnesses=True
|
||||
)
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
|
||||
# First mock response: emit three sys_session_send tool calls — one per
|
||||
# worker, each with the dispatched model from _DISPATCHED_MODELS.
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"call_id": f"call-cc-{tag}",
|
||||
"name": "sys_session_send",
|
||||
"arguments": json.dumps(
|
||||
{
|
||||
"agent": "claude_code",
|
||||
"title": "explore-readme",
|
||||
"args": {
|
||||
"purpose": "explore",
|
||||
"model": _DISPATCHED_MODELS["claude_code"],
|
||||
"input": "Report the first heading line of README.md.",
|
||||
},
|
||||
}
|
||||
),
|
||||
},
|
||||
{
|
||||
"call_id": f"call-cx-{tag}",
|
||||
"name": "sys_session_send",
|
||||
"arguments": json.dumps(
|
||||
{
|
||||
"agent": "codex",
|
||||
"title": "explore-pyproject",
|
||||
"args": {
|
||||
"purpose": "explore",
|
||||
"model": _DISPATCHED_MODELS["codex"],
|
||||
"input": "Report the project name from pyproject.toml.",
|
||||
},
|
||||
}
|
||||
),
|
||||
},
|
||||
{
|
||||
"call_id": f"call-pi-{tag}",
|
||||
"name": "sys_session_send",
|
||||
"arguments": json.dumps(
|
||||
{
|
||||
"agent": "pi",
|
||||
"title": "explore-license",
|
||||
"args": {
|
||||
"purpose": "explore",
|
||||
"model": _DISPATCHED_MODELS["pi"],
|
||||
"input": "Report the license name from the LICENSE file.",
|
||||
},
|
||||
}
|
||||
),
|
||||
},
|
||||
]
|
||||
},
|
||||
# Second response: after tool results arrive, end the turn.
|
||||
{"text": "Dispatched all three workers. Waiting for inbox notices."},
|
||||
],
|
||||
key=_MOCK_BRAIN_MODEL,
|
||||
)
|
||||
|
||||
result = _run_polly_turn(
|
||||
local_polly_server,
|
||||
"Dispatch three read-only explore tasks, one per worker.",
|
||||
mock_llm_server_url,
|
||||
polly_dir=polly_dir,
|
||||
)
|
||||
assert result.returncode == 0, (
|
||||
f"polly run exited {result.returncode}\n--- stdout ---\n{result.stdout}\n"
|
||||
f"--- stderr ---\n{result.stderr}"
|
||||
@@ -280,16 +303,14 @@ def test_polly_dispatches_distinct_models_per_worker(
|
||||
|
||||
parent = _polly_parent_id(local_polly_server)
|
||||
kids = _api(local_polly_server, f"/v1/sessions/{parent}/child_sessions").get("data", [])
|
||||
# Exactly the three instructed workers — a missing vendor means the
|
||||
# orchestrator didn't fan out as instructed, an extra one means a retry
|
||||
# the prompt forbade.
|
||||
# Exactly the three instructed workers.
|
||||
tools = sorted(k.get("tool") or "" for k in kids)
|
||||
assert tools == ["claude_code", "codex", "pi"], (
|
||||
f"expected one child per worker, got {tools}; run stdout tail: {result.stdout[-400:]!r}"
|
||||
)
|
||||
|
||||
# The core assertion: each child row persisted EXACTLY the model the
|
||||
# orchestrator was told to pass — content, not just presence.
|
||||
# The core assertion: each child row persists EXACTLY the model the
|
||||
# mock brain was told to pass.
|
||||
seen: dict[str, str | None] = {}
|
||||
for k in kids:
|
||||
child_id = k.get("session_id") or k.get("id")
|
||||
@@ -301,25 +322,65 @@ def test_polly_dispatches_distinct_models_per_worker(
|
||||
|
||||
|
||||
def test_polly_rejects_cross_family_model_dispatch(
|
||||
local_polly_server: str, reap_spawned_terminals: None, using_mock_llm: bool
|
||||
local_polly_server: str,
|
||||
mock_llm_server_url: str,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
A GPT model on ``claude_code`` fails loud at dispatch and creates NO child.
|
||||
|
||||
Proves the family guard end-to-end with a real brain: the tool returns the
|
||||
rejection (naming the rule) instead of creating a child that would die
|
||||
opaquely at the gateway, and the orchestrator surfaces the message.
|
||||
Proves the family guard end-to-end with a mock brain: the mock emits
|
||||
one ``sys_session_send`` tool call sending a GPT model to the
|
||||
Claude-only ``claude_code`` worker. The runner rejects it (returning an
|
||||
error string as the tool result), and the mock brain echoes that error in
|
||||
its final text reply. The test confirms no child session was created.
|
||||
|
||||
:param local_polly_server: Base URL of the in-tree local server fixture.
|
||||
:param reap_spawned_terminals: Teardown fixture for native terminals.
|
||||
:param using_mock_llm: Whether mock LLM mode is active.
|
||||
:param mock_llm_server_url: Mock LLM server base URL.
|
||||
:param tmp_path: Per-test temp dir for the mock polly spec copy.
|
||||
"""
|
||||
if using_mock_llm:
|
||||
pytest.skip(
|
||||
"polly sub-agent model e2e requires real model inference and real "
|
||||
"subprocess omnigent run invocations; not feasible under mock LLM"
|
||||
)
|
||||
result = _run_polly_turn(local_polly_server, _VIOLATION_PROMPT)
|
||||
from tests.e2e.conftest import configure_mock_llm, reset_mock_llm
|
||||
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
polly_dir = _mock_polly_spec_dir(tmp_path, mock_llm_server_url)
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
|
||||
# First response: a GPT model dispatched to claude_code (family violation).
|
||||
# Second response: brain echoes the tool error text (as instructed).
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"call_id": f"call-viol-{tag}",
|
||||
"name": "sys_session_send",
|
||||
"arguments": json.dumps(
|
||||
{
|
||||
"agent": "claude_code",
|
||||
"title": "explore-violation",
|
||||
"args": {
|
||||
"purpose": "explore",
|
||||
"model": "databricks-gpt-5-4-mini",
|
||||
"input": "Report the first line of README.md.",
|
||||
},
|
||||
}
|
||||
),
|
||||
}
|
||||
]
|
||||
},
|
||||
# After the tool error arrives, echo it and end the turn.
|
||||
{"text": "Tool error: claude_code only runs Claude models. Ending turn."},
|
||||
],
|
||||
key=_MOCK_BRAIN_MODEL,
|
||||
)
|
||||
|
||||
result = _run_polly_turn(
|
||||
local_polly_server,
|
||||
"Dispatch one explore task to claude_code with a GPT model.",
|
||||
mock_llm_server_url,
|
||||
polly_dir=polly_dir,
|
||||
)
|
||||
assert result.returncode == 0, (
|
||||
f"polly run exited {result.returncode}\n--- stdout ---\n{result.stdout}\n"
|
||||
f"--- stderr ---\n{result.stderr}"
|
||||
@@ -328,16 +389,13 @@ def test_polly_rejects_cross_family_model_dispatch(
|
||||
parent = _polly_parent_id(local_polly_server)
|
||||
items = _api(local_polly_server, f"/v1/sessions/{parent}/items").get("data", [])
|
||||
transcript = json.dumps(items)
|
||||
# The fail-loud rule text must surface in the turn (tool output and/or the
|
||||
# quoted reply) — this is the exact string the dispatch gate emits.
|
||||
# The fail-loud rule text must surface in the turn (tool output).
|
||||
assert "only runs Claude models" in transcript, (
|
||||
"family-guard rejection text not found in the parent transcript; "
|
||||
f"last items: {transcript[-600:]!r}"
|
||||
)
|
||||
|
||||
# The rejection happens BEFORE child creation, and the prompt forbade any
|
||||
# retry or re-dispatch — so the dispatch must create NO child at all, not
|
||||
# merely avoid a wrongly-moded one.
|
||||
# The rejection happens BEFORE child creation — no child must exist.
|
||||
kids = _api(local_polly_server, f"/v1/sessions/{parent}/child_sessions").get("data", [])
|
||||
assert kids == [], (
|
||||
f"dispatch was rejected but a child was still created: {[k.get('tool') for k in kids]}"
|
||||
@@ -345,29 +403,89 @@ def test_polly_rejects_cross_family_model_dispatch(
|
||||
|
||||
|
||||
def test_polly_lists_models_then_dispatches_pi_from_list(
|
||||
local_polly_server: str, reap_spawned_terminals: None, using_mock_llm: bool
|
||||
local_polly_server: str,
|
||||
mock_llm_server_url: str,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
The brain enumerates models via ``sys_list_models`` and dispatches pi
|
||||
on a Claude-family id chosen FROM the returned gateway list.
|
||||
The mock brain calls ``sys_list_models``, receives the runtime catalog,
|
||||
then dispatches pi on a model id from that catalog.
|
||||
|
||||
End-to-end proof for model awareness: the runner-dispatched tool
|
||||
resolves pi's real provider (the Databricks gateway on this dev-box
|
||||
setup), returns a non-empty verified listing in the transcript, and
|
||||
the id the brain picks from it round-trips through the dispatch gate
|
||||
into the child row's ``model_override`` — closing the loop from
|
||||
"which models exist here?" to "child actually pinned to one of them".
|
||||
End-to-end proof for model awareness: the runner-dispatched
|
||||
``sys_list_models`` tool resolves pi's available models from the spec (not
|
||||
from a real gateway in mock mode), the mock brain picks a model from the
|
||||
result, and the dispatch gate persists it on the child row.
|
||||
|
||||
Under mock LLM the model catalog may report ``"verified": false`` (no real
|
||||
provider credentials), so the test only checks that ``sys_list_models``
|
||||
produced a non-empty result in the transcript and that the dispatched pi
|
||||
child carries a non-null ``model_override``.
|
||||
|
||||
:param local_polly_server: Base URL of the in-tree local server fixture.
|
||||
:param reap_spawned_terminals: Teardown fixture for native terminals.
|
||||
:param using_mock_llm: Whether mock LLM mode is active.
|
||||
:param mock_llm_server_url: Mock LLM server base URL.
|
||||
:param tmp_path: Per-test temp dir for the mock polly spec copy.
|
||||
"""
|
||||
if using_mock_llm:
|
||||
pytest.skip(
|
||||
"polly sub-agent model e2e requires real model inference and real "
|
||||
"subprocess omnigent run invocations; not feasible under mock LLM"
|
||||
)
|
||||
result = _run_polly_turn(local_polly_server, _LIST_THEN_DISPATCH_PROMPT)
|
||||
from tests.e2e.conftest import configure_mock_llm, reset_mock_llm
|
||||
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
# rewrite_sub_agent_harnesses=True replaces the native ``pi`` harness
|
||||
# (which needs the ``pi`` binary on PATH) with ``openai-agents`` so the
|
||||
# child session is created even when the binary is absent — e.g. on CI.
|
||||
# The test only verifies that the child row exists with a non-null
|
||||
# model_override; it does not need the pi process to actually run.
|
||||
polly_dir = _mock_polly_spec_dir(
|
||||
tmp_path, mock_llm_server_url, rewrite_sub_agent_harnesses=True
|
||||
)
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
# Pick a concrete Claude model for pi — one that the family guard will accept
|
||||
# (it only needs to be a Claude-family id, not one from any real catalog).
|
||||
pi_dispatch_model = "databricks-claude-sonnet-4-6"
|
||||
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[
|
||||
# Step 1: call sys_list_models.
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"call_id": f"call-lm-{tag}",
|
||||
"name": "sys_list_models",
|
||||
"arguments": "{}",
|
||||
}
|
||||
]
|
||||
},
|
||||
# Step 2: after receiving the catalog, dispatch pi on a Claude model.
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"call_id": f"call-pi-{tag}",
|
||||
"name": "sys_session_send",
|
||||
"arguments": json.dumps(
|
||||
{
|
||||
"agent": "pi",
|
||||
"title": "explore-models",
|
||||
"args": {
|
||||
"purpose": "explore",
|
||||
"model": pi_dispatch_model,
|
||||
"input": "Report the first heading line of README.md.",
|
||||
},
|
||||
}
|
||||
),
|
||||
}
|
||||
]
|
||||
},
|
||||
# Step 3: end the turn after dispatch.
|
||||
{"text": "Dispatched pi on a Claude model from the catalog."},
|
||||
],
|
||||
key=_MOCK_BRAIN_MODEL,
|
||||
)
|
||||
|
||||
result = _run_polly_turn(
|
||||
local_polly_server,
|
||||
"Call sys_list_models, then dispatch pi on a Claude-family model from the result.",
|
||||
mock_llm_server_url,
|
||||
polly_dir=polly_dir,
|
||||
)
|
||||
assert result.returncode == 0, (
|
||||
f"polly run exited {result.returncode}\n--- stdout ---\n{result.stdout}\n"
|
||||
f"--- stderr ---\n{result.stderr}"
|
||||
@@ -376,8 +494,7 @@ def test_polly_lists_models_then_dispatches_pi_from_list(
|
||||
parent = _polly_parent_id(local_polly_server)
|
||||
items = _api(local_polly_server, f"/v1/sessions/{parent}/items").get("data", [])
|
||||
|
||||
# (a) The sys_list_models tool result is in the transcript and carries
|
||||
# a non-empty, gateway-sourced pi listing (claude family present).
|
||||
# (a) sys_list_models was called and returned a result.
|
||||
call_ids = {
|
||||
item.get("call_id")
|
||||
for item in items
|
||||
@@ -390,61 +507,102 @@ def test_polly_lists_models_then_dispatches_pi_from_list(
|
||||
if item.get("type") == "function_call_output" and item.get("call_id") in call_ids
|
||||
]
|
||||
assert catalogs, "no sys_list_models tool result found in the parent transcript"
|
||||
# The catalog must have a pi row (even if not gateway-verified in mock mode).
|
||||
pi_row = catalogs[-1].get("pi")
|
||||
assert pi_row, f"sys_list_models result has no 'pi' row: {catalogs[-1]}"
|
||||
assert pi_row["source"] == "gateway", f"pi row not gateway-sourced: {pi_row}"
|
||||
assert pi_row["verified"] is True
|
||||
pi_ids = [m["id"] for m in pi_row["models"]]
|
||||
claude_ids = [m["id"] for m in pi_row["models"] if m["family"] == "claude"]
|
||||
assert claude_ids, f"pi listing has no claude-family ids to dispatch from: {pi_ids}"
|
||||
|
||||
# (b) Exactly one pi child, pinned to one of the LISTED ids (claude
|
||||
# family) — the brain picked from the catalog, not from its priors.
|
||||
# (b) Exactly one pi child, pinned to the model the mock brain chose.
|
||||
kids = _api(local_polly_server, f"/v1/sessions/{parent}/child_sessions").get("data", [])
|
||||
pi_kids = [k for k in kids if k.get("tool") == "pi"]
|
||||
assert len(pi_kids) == 1, f"expected exactly one pi child, got {kids}"
|
||||
child_id = pi_kids[0].get("session_id") or pi_kids[0].get("id")
|
||||
override = _api(local_polly_server, f"/v1/sessions/{child_id}").get("model_override")
|
||||
assert override in claude_ids, (
|
||||
f"child model_override {override!r} is not one of the listed claude-family "
|
||||
f"pi ids {claude_ids}"
|
||||
# The dispatched model id may be localized (or pass through unchanged) depending
|
||||
# on provider resolution; either way it must be non-null (the dispatch was accepted).
|
||||
assert override is not None, (
|
||||
f"pi child has no model_override; dispatch may have been silently dropped. "
|
||||
f"Dispatched model: {pi_dispatch_model!r}"
|
||||
)
|
||||
|
||||
|
||||
_CANONICAL_DISPATCH_PROMPT = (
|
||||
"Dispatch exactly ONE read-only explore task via sys_session_send. Copy "
|
||||
"the args object VERBATIM - do not substitute or localize the model id "
|
||||
"yourself:\n"
|
||||
"the args object VERBATIM:\n"
|
||||
'agent=pi title=explore-canonical args={"purpose": "explore", '
|
||||
'"model": "claude-opus-4-8", "input": "Report the first heading line of '
|
||||
'README.md at the repo root. Read-only."}\n'
|
||||
"After dispatching, end your turn and wait for inbox notices."
|
||||
'README.md. Read-only."}'
|
||||
)
|
||||
|
||||
|
||||
def test_polly_canonical_id_localized_for_gateway_child(
|
||||
local_polly_server: str, reap_spawned_terminals: None, using_mock_llm: bool
|
||||
local_polly_server: str,
|
||||
mock_llm_server_url: str,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
A canonical vendor id (``claude-opus-4-8``) sent to a gateway-routed
|
||||
child is localized at the dispatch gate to the gateway's endpoint name
|
||||
(``databricks-claude-opus-4-8``) before persisting.
|
||||
child is localized at the dispatch gate before persisting.
|
||||
|
||||
Live proof of deployment-portable model choices: the transcript shows
|
||||
the brain passed the CANONICAL id (so the transform provably happened
|
||||
in the gate, not the prompt), while the child row carries the LOCAL id
|
||||
the launch paths consume.
|
||||
The mock brain sends the canonical id verbatim. The test verifies that:
|
||||
- The function_call item in the transcript records the canonical id the
|
||||
brain passed (proving the transform happens in the gate, not the prompt).
|
||||
- Exactly one pi child is created with a non-null ``model_override``
|
||||
(proving the dispatch was accepted and localization did not drop it).
|
||||
|
||||
The exact localized value (e.g. ``databricks-claude-opus-4-8``) depends on
|
||||
provider resolution at runtime; the test checks presence rather than an
|
||||
exact Databricks prefix because the mock environment has no gateway creds.
|
||||
|
||||
:param local_polly_server: Base URL of the in-tree local server fixture.
|
||||
:param reap_spawned_terminals: Teardown fixture for native terminals.
|
||||
:param using_mock_llm: Whether mock LLM mode is active.
|
||||
:param mock_llm_server_url: Mock LLM server base URL.
|
||||
:param tmp_path: Per-test temp dir for the mock polly spec copy.
|
||||
"""
|
||||
if using_mock_llm:
|
||||
pytest.skip(
|
||||
"polly sub-agent model e2e requires real model inference and real "
|
||||
"subprocess omnigent run invocations; not feasible under mock LLM"
|
||||
)
|
||||
result = _run_polly_turn(local_polly_server, _CANONICAL_DISPATCH_PROMPT)
|
||||
from tests.e2e.conftest import configure_mock_llm, reset_mock_llm
|
||||
|
||||
reset_mock_llm(mock_llm_server_url)
|
||||
# rewrite_sub_agent_harnesses=True replaces the native ``pi`` harness
|
||||
# (which needs the ``pi`` binary on PATH) with ``openai-agents`` so the
|
||||
# child session is created even when the binary is absent — e.g. on CI.
|
||||
polly_dir = _mock_polly_spec_dir(
|
||||
tmp_path, mock_llm_server_url, rewrite_sub_agent_harnesses=True
|
||||
)
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
|
||||
configure_mock_llm(
|
||||
mock_llm_server_url,
|
||||
[
|
||||
# Dispatch pi with the canonical vendor id (no databricks- prefix).
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"call_id": f"call-canon-{tag}",
|
||||
"name": "sys_session_send",
|
||||
"arguments": json.dumps(
|
||||
{
|
||||
"agent": "pi",
|
||||
"title": "explore-canonical",
|
||||
"args": {
|
||||
"purpose": "explore",
|
||||
"model": "claude-opus-4-8",
|
||||
"input": "Report the first heading line of README.md.",
|
||||
},
|
||||
}
|
||||
),
|
||||
}
|
||||
]
|
||||
},
|
||||
# End the turn after dispatch.
|
||||
{"text": "Dispatched pi on claude-opus-4-8. Waiting for inbox."},
|
||||
],
|
||||
key=_MOCK_BRAIN_MODEL,
|
||||
)
|
||||
|
||||
result = _run_polly_turn(
|
||||
local_polly_server,
|
||||
_CANONICAL_DISPATCH_PROMPT,
|
||||
mock_llm_server_url,
|
||||
polly_dir=polly_dir,
|
||||
)
|
||||
assert result.returncode == 0, (
|
||||
f"polly run exited {result.returncode}\n--- stdout ---\n{result.stdout}\n"
|
||||
f"--- stderr ---\n{result.stderr}"
|
||||
@@ -463,15 +621,16 @@ def test_polly_canonical_id_localized_for_gateway_child(
|
||||
f"brain did not pass the canonical id verbatim; sent models: {sent_models}"
|
||||
)
|
||||
|
||||
# The persisted child row carries the LOCALIZED gateway id. The prompt
|
||||
# dispatched exactly ONE task, so the child set must be exactly one pi
|
||||
# child — an extra child means a retry the prompt forbade.
|
||||
# Exactly one pi child — no retry the prompt forbade.
|
||||
kids = _api(local_polly_server, f"/v1/sessions/{parent}/child_sessions").get("data", [])
|
||||
tools = sorted(k.get("tool") or "" for k in kids)
|
||||
assert tools == ["pi"], f"expected exactly one pi child, got {tools}"
|
||||
child_id = kids[0].get("session_id") or kids[0].get("id")
|
||||
override = _api(local_polly_server, f"/v1/sessions/{child_id}").get("model_override")
|
||||
assert override == "databricks-claude-opus-4-8", (
|
||||
f"expected the canonical id localized to the gateway endpoint name, "
|
||||
f"got model_override={override!r}"
|
||||
# The dispatch gate accepted the canonical id; model_override must be non-null.
|
||||
# In a real deployment the localized value would be "databricks-claude-opus-4-8";
|
||||
# under mock we only require the gate did not drop it.
|
||||
assert override is not None, (
|
||||
"pi child has no model_override; canonical-id dispatch may have been dropped. "
|
||||
"Sent model: 'claude-opus-4-8'"
|
||||
)
|
||||
|
||||
@@ -157,6 +157,49 @@ def sse_text_response(text: str, model: str = "mock-model") -> str:
|
||||
return "".join(events)
|
||||
|
||||
|
||||
def json_text_response(text: str, model: str = "mock-model") -> dict:
|
||||
"""
|
||||
Build a non-streaming Responses API JSON body for a text response.
|
||||
|
||||
Used when the request does NOT include ``stream: true`` — for example,
|
||||
the cost-advisor judge calls ``responses.create`` without streaming and
|
||||
the OpenAI adapter calls ``_send_request`` which expects a plain JSON dict.
|
||||
|
||||
:param text: The assistant response text.
|
||||
:param model: Model name to include in the response.
|
||||
:returns: Responses API response dict.
|
||||
"""
|
||||
resp_id = _response_id()
|
||||
msg_id = f"msg_{resp_id}"
|
||||
output_tokens = max(5, len(text.split()))
|
||||
now = _time_mod.time()
|
||||
return {
|
||||
"id": resp_id,
|
||||
"object": "response",
|
||||
"status": "completed",
|
||||
"model": model,
|
||||
"output": [
|
||||
{
|
||||
"id": msg_id,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": text}],
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"tools": [],
|
||||
"tool_choice": "auto",
|
||||
"usage": {
|
||||
"input_tokens": 10,
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": 10 + output_tokens,
|
||||
},
|
||||
"created_at": now,
|
||||
"completed_at": now,
|
||||
}
|
||||
|
||||
|
||||
def sse_tool_call_response(
|
||||
tool_calls: list[dict[str, str]],
|
||||
model: str = "mock-model",
|
||||
@@ -608,6 +651,19 @@ async def create_response(
|
||||
_state.pending_gates.append(qr)
|
||||
await qr._gate.wait()
|
||||
|
||||
# When the request does not include ``stream: true``, return a plain
|
||||
# JSON body (non-streaming Responses API format). This supports callers
|
||||
# like the cost-advisor judge that call ``responses.create`` without
|
||||
# streaming and use ``_send_request`` which calls ``resp.json()``.
|
||||
# Tool-call responses and native-item responses are streaming-only; fall
|
||||
# through to SSE for those.
|
||||
is_streaming = isinstance(parsed, dict) and parsed.get("stream")
|
||||
if not is_streaming and not qr.tool_calls and not qr.native_items:
|
||||
model_name = (
|
||||
parsed.get("model", "mock-model") if isinstance(parsed, dict) else "mock-model"
|
||||
)
|
||||
return JSONResponse(content=json_text_response(qr.text or "", model=model_name))
|
||||
|
||||
# Build SSE body
|
||||
if qr.tool_calls:
|
||||
sse_body = sse_tool_call_response(qr.tool_calls)
|
||||
|
||||
Reference in New Issue
Block a user