Merge branch 'test/mock-llm-e2e-batch3' into test/mock-e2e-switch-agent

- test_switch_agent_unknown_target_is_rejected: replace claude_coder_agent
  with an inline openai-agents agent so the test runs without a real LLM
- test_switch_resets_os_env_filesystem_availability: no changes needed
  (already LLM-free)
- test_switch_agent_in_place_carries_history: skip in mock mode because
  the switch endpoint only binds built-in agents and sdk-chat-builtin
  uses claude-sdk (not mockable via OPENAI_BASE_URL)

Co-authored-by: Isaac

tests/e2e/test_comment_tools.py

@@ -6,21 +6,29 @@ the REST API, asks the agent to address them, and the agent calls
 each one as "addressed". The test then confirms the server reflects
 the expected "addressed" status on all comments.
 
+Runs against the mock LLM server — the mock returns tool call
+responses for ``list_comments`` and ``update_comment``, and the
+runner executes them as real runner-level tools.
+
 Usage::
 
-    pytest tests/e2e/test_comment_tools.py \\
-        --llm-api-key $LLM_API_KEY -v
+    pytest tests/e2e/test_comment_tools.py -v
 """
 
 from __future__ import annotations
 
+import json
+import uuid
 from typing import Any
 
 import httpx
 
 from tests.e2e.conftest import (
+    configure_mock_llm,
     create_runner_bound_session,
     poll_session_until_terminal,
+    register_inline_agent,
+    reset_mock_llm,
     send_user_message_to_session,
 )
 
@@ -43,8 +51,8 @@ def _tool_names_in_output(body: dict[str, Any]) -> list[str]:
 
 def test_agent_lists_and_addresses_comments(
     http_client: httpx.Client,
-    archer_agent: str,
     live_runner_id: str,
+    mock_llm_server_url: str | None,
 ) -> None:
     """
     Agent uses list_comments + update_comment to address review comments.
@@ -74,9 +82,20 @@ def test_agent_lists_and_addresses_comments(
     :param live_runner_id: Runner id the session is bound to.
     """
     # ── 1. Create a runner-bound session ──────────────────────────────────────
+    model = f"mock-comment-{uuid.uuid4().hex[:6]}"
+    reset_mock_llm(mock_llm_server_url)
+    agent_name = register_inline_agent(
+        http_client,
+        name=f"comment-{uuid.uuid4().hex[:6]}",
+        harness="openai-agents",
+        model=model,
+        profile="",
+        prompt="You are a code review assistant.",
+        mock_llm_base_url=(f"{mock_llm_server_url}/v1" if mock_llm_server_url else None),
+    )
     session_id = create_runner_bound_session(
         http_client,
-        agent_name=archer_agent,
+        agent_name=agent_name,
         runner_id=live_runner_id,
     )
 
@@ -120,7 +139,46 @@ def test_agent_lists_and_addresses_comments(
         f"Expected comment 2 to start as 'draft', got {pre_statuses.get(comment2_id)!r}"
     )
 
-    # ── 3. Ask the agent to address the comments ─────────────────────────────
+    # ── 3. Configure mock LLM responses ──────────────────────────────────────
+    # The mock LLM returns: list_comments → update_comment(c1) →
+    # update_comment(c2) → final text. The runner executes the real
+    # comment tools (runner-level, always registered).
+    configure_mock_llm(
+        mock_llm_server_url,
+        [
+            {
+                "tool_calls": [
+                    {
+                        "call_id": "call_list",
+                        "name": "list_comments",
+                        "arguments": json.dumps({"path": "app.py"}),
+                    },
+                ],
+            },
+            {
+                "tool_calls": [
+                    {
+                        "call_id": "call_upd1",
+                        "name": "update_comment",
+                        "arguments": json.dumps(
+                            {"comment_id": comment1_id, "status": "addressed"}
+                        ),
+                    },
+                    {
+                        "call_id": "call_upd2",
+                        "name": "update_comment",
+                        "arguments": json.dumps(
+                            {"comment_id": comment2_id, "status": "addressed"}
+                        ),
+                    },
+                ],
+            },
+            {"text": "Both comments addressed."},
+        ],
+        key=model,
+    )
+
+    # ── 4. Ask the agent to address the comments ─────────────────────────────
     # The prompt names the tools explicitly so the LLM reliably uses them
     # rather than trying to "answer" without tool calls.
     response_id = send_user_message_to_session(
@@ -135,7 +193,7 @@ def test_agent_lists_and_addresses_comments(
         ),
     )
 
-    # ── 4. Wait for the agent turn to complete ───────────────────────────────
+    # ── 5. Wait for the agent turn to complete ───────────────────────────────
     body = poll_session_until_terminal(
         http_client,
         session_id=session_id,
@@ -146,7 +204,7 @@ def test_agent_lists_and_addresses_comments(
         f"Agent turn failed. error={body.get('error')!r}. output={body.get('output', [])}"
     )
 
-    # ── 5. Verify tool calls in the agent output ─────────────────────────────
+    # ── 6. Verify tool calls in the agent output ─────────────────────────────
     calls = _tool_names_in_output(body)
 
     # list_comments must have been called at least once — that's how the
@@ -166,7 +224,7 @@ def test_agent_lists_and_addresses_comments(
         f"got {update_call_count}. Tool calls seen: {calls}"
     )
 
-    # ── 6. Verify comment statuses via REST ───────────────────────────────────
+    # ── 7. Verify comment statuses via REST ───────────────────────────────────
     post_resp = http_client.get(f"/v1/sessions/{session_id}/comments")
     post_resp.raise_for_status()
     post_statuses = {c["id"]: c["status"] for c in post_resp.json()}

tests/e2e/test_sharing_permissions_e2e.py

@@ -27,9 +27,11 @@ import httpx
 import pytest
 
 from tests.e2e.conftest import (
+    configure_mock_llm,
     create_runner_bound_session,
     poll_session_until_terminal,
     register_inline_agent,
+    reset_mock_llm,
     send_user_message_to_session,
 )
 
@@ -87,12 +89,15 @@ class _OwnedSession:
     owner_email: str
     owner: httpx.Client
     session_id: str
+    model: str = ""
 
 
 @pytest.fixture
 def owner_session(
     live_server: str,
     live_runner_id: str,
+    using_mock_llm: bool,
+    mock_llm_server_url: str | None,
     request: pytest.FixtureRequest,
 ) -> Iterator[_OwnedSession]:
     """A runner-bound session owned by the ``local`` identity.
@@ -103,19 +108,29 @@ def owner_session(
     on the shared session-scoped server.
     """
     suffix = uuid.uuid4().hex[:6]
+    model = f"mock-share-{suffix}" if using_mock_llm else "databricks-gpt-5-4-mini"
     owner = httpx.Client(base_url=live_server, timeout=300)
+    reset_mock_llm(mock_llm_server_url)
     agent_name = register_inline_agent(
         owner,
         name=f"sharing-e2e-{suffix}",
         harness="openai-agents",
-        model="databricks-gpt-5-4-mini",
+        model=model,
         profile=request.config.getoption("--profile"),
         prompt="You are a terse assistant. Follow instructions exactly.",
+        mock_llm_base_url=(
+            f"{mock_llm_server_url}/v1" if using_mock_llm and mock_llm_server_url else None
+        ),
     )
     session_id = create_runner_bound_session(
         owner, agent_name=agent_name, runner_id=live_runner_id
     )
-    yield _OwnedSession(owner_email="local", owner=owner, session_id=session_id)
+    yield _OwnedSession(
+        owner_email="local",
+        owner=owner,
+        session_id=session_id,
+        model=model,
+    )
     owner.close()
 
 
@@ -159,12 +174,16 @@ def test_read_grant_allows_snapshot_blocks_events(
 
 
 def test_edit_grant_bob_turn_completes_and_owner_sees_it(
-    live_server: str, owner_session: _OwnedSession
+    live_server: str,
+    owner_session: _OwnedSession,
+    mock_llm_server_url: str | None,
 ) -> None:
-    """An EDIT collaborator's turn runs the real LLM and lands in the
+    """An EDIT collaborator's turn runs the LLM and lands in the
     owner's view of the conversation."""
     sid = owner_session.session_id
     marker = f"shared-turn-{uuid.uuid4().hex[:8]}"
+    if owner_session.model:
+        configure_mock_llm(mock_llm_server_url, [{"text": marker}], key=owner_session.model)
     with _client_for(live_server, f"bob-{uuid.uuid4().hex[:6]}@e2e.test") as bob:
         owner_session.owner.put(
             f"/v1/sessions/{sid}/permissions",

tests/e2e/test_switch_agent_e2e.py

@@ -1,24 +1,18 @@
 """E2E tests for in-place agent switch — ``POST /v1/sessions/{id}/switch-agent``.
 
-Real server + runner + LLM. Unlike fork (which branches into a NEW session),
-switch rebinds the SAME session to a different agent and continues there. The
-core guarantee is that the new agent picks up the prior conversation: an SDK
-target replays the Omnigent transcript as context, so a code word planted
+Real server + runner + LLM (or mock LLM). Unlike fork (which branches into a
+NEW session), switch rebinds the SAME session to a different agent and continues
+there. The core guarantee is that the new agent picks up the prior conversation:
+an SDK target replays the Omnigent transcript as context, so a code word planted
 before the switch must be recalled after it — on the same session id.
 
-Both the source (``claude-coder``) and the switch TARGET
-(``sdk-chat-builtin``) are gateway-wired under ``--profile``: the source via
-:func:`upload_agent`'s model rewrite, the built-in target via
-:func:`tests.e2e.conftest._materialize_builtin_sdk_chat_spec`, which seeds a
-profile-aware copy (model mapped + ``executor.profile`` stamped) instead of
-the on-disk OAuth spec. So the post-switch turn authenticates through the
-Databricks gateway and runs on hosted CI without a Claude login. (Without
-``--profile`` — the local api.openai.com path — the built-in falls back to
-the verbatim ``claude`` CLI OAuth spec.)
+In mock mode the source and target are both inline ``openai-agents`` agents
+pointed at the mock LLM server; each gets its own keyed response queue so the
+test controls exactly what each agent says.
 
 Usage::
 
-    pytest tests/e2e/test_switch_agent_e2e.py --llm-api-key $PAT --profile oss -v
+    pytest tests/e2e/test_switch_agent_e2e.py -v --timeout=60 --no-skip-known
 """
 
 from __future__ import annotations
@@ -30,11 +24,14 @@ import time
 import uuid
 
 import httpx
+import pytest
 import yaml
 
 from tests.e2e.conftest import (
     create_runner_bound_session,
     poll_session_until_terminal,
+    register_inline_agent,
+    reset_mock_llm,
     send_user_message_to_session,
 )
 from tests.e2e.helpers import final_assistant_text
@@ -72,6 +69,7 @@ def test_switch_agent_in_place_carries_history(
     http_client: httpx.Client,
     claude_coder_agent: str,
     live_runner_id: str,
+    using_mock_llm: bool,
 ) -> None:
     """A switched agent recalls a code word planted before the switch.
 
@@ -83,11 +81,22 @@ def test_switch_agent_in_place_carries_history(
     bound agent actually changed, proving this is an in-place switch and not a
     fork.
 
+    Requires a real LLM: the switch endpoint only binds built-in agents, and
+    the ``sdk-chat-builtin`` built-in uses ``claude-sdk`` which authenticates
+    via the Claude CLI's OAuth session (not mockable through ``OPENAI_BASE_URL``).
+
     :param http_client: HTTP client pointed at the live server.
     :param claude_coder_agent: The uploaded claude-sdk source agent name.
     :param live_runner_id: The server fixture's runner id.
+    :param using_mock_llm: Whether mock LLM is active.
     :returns: None.
     """
+    if using_mock_llm:
+        pytest.skip(
+            "switch-agent only binds built-in agents; sdk-chat-builtin uses "
+            "claude-sdk which requires a real LLM (not mockable via OPENAI_BASE_URL)"
+        )
+
     marker = f"SWITCHWORD_{uuid.uuid4().hex[:6].upper()}"
 
     # 1. Source session (claude-sdk) on the server's runner; plant a word.
@@ -144,8 +153,8 @@ def test_switch_agent_in_place_carries_history(
 
 def test_switch_agent_unknown_target_is_rejected(
     http_client: httpx.Client,
-    claude_coder_agent: str,
     live_runner_id: str,
+    mock_llm_server_url: str,
 ) -> None:
     """Switching to a non-existent agent is rejected and leaves the session.
 
@@ -153,12 +162,26 @@ def test_switch_agent_unknown_target_is_rejected(
     returns 404 and the session's bound agent is unchanged (no half-switch).
 
     :param http_client: HTTP client pointed at the live server.
-    :param claude_coder_agent: The uploaded claude-sdk source agent name.
     :param live_runner_id: The server fixture's runner id.
+    :param mock_llm_server_url: Mock LLM server URL.
     :returns: None.
     """
+    uid = uuid.uuid4().hex[:6]
+    model = f"mock-switch-unk-{uid}"
+
+    reset_mock_llm(mock_llm_server_url)
+    agent_name = register_inline_agent(
+        http_client,
+        name=f"switch-unk-{uid}",
+        harness="openai-agents",
+        model=model,
+        profile="",
+        prompt="Placeholder agent for unknown-target switch test.",
+        mock_llm_base_url=f"{mock_llm_server_url}/v1",
+    )
+
     session_id = create_runner_bound_session(
-        http_client, agent_name=claude_coder_agent, runner_id=live_runner_id
+        http_client, agent_name=agent_name, runner_id=live_runner_id
     )
     original_agent_id = _bound_agent(http_client, session_id)["id"]
 

tests/integration/conftest.py

@@ -169,6 +169,7 @@ def journey_session(
     live_runner_id: str,  # noqa: F811  (pytest fixture, not the import)
     harness_name: str,
     model_name: str,
+    using_mock_llm: bool,  # noqa: F811
     request: pytest.FixtureRequest,
     mock_llm_server_url: str | None,  # noqa: F811
 ) -> JourneySession:
@@ -181,6 +182,7 @@ def journey_session(
     :param live_runner_id: Runner to bind the session to.
     :param harness_name: Harness under test.
     :param model_name: Resolved model for this test.
+    :param using_mock_llm: Whether mock LLM mode is active.
     :param request: Pytest fixture request (for ``--profile``).
     :param mock_llm_server_url: Mock LLM server URL, or ``None``.
     :returns: The registered agent + bound session.
@@ -197,9 +199,7 @@ def journey_session(
             "reply with the token text only."
         ),
         mock_llm_base_url=(
-            f"{mock_llm_server_url}/v1"
-            if _is_mock_mode(request.config) and mock_llm_server_url
-            else None
+            f"{mock_llm_server_url}/v1" if using_mock_llm and mock_llm_server_url else None
         ),
     )
     session_id = create_runner_bound_session(

tests/integration/test_sharing.py

@@ -34,6 +34,7 @@ def test_share_and_second_user_continues(
     live_runner_id: str,
     harness_name: str,
     model_name: str,
+    using_mock_llm: bool,
     request: pytest.FixtureRequest,
     mock_llm_server_url: str | None,
 ) -> None:
@@ -66,9 +67,7 @@ def test_share_and_second_user_continues(
             profile=request.config.getoption("--profile"),
             prompt="You are a terse test assistant. Follow instructions exactly.",
             mock_llm_base_url=(
-                f"{mock_llm_server_url}/v1"
-                if request.config.getoption("--llm-api-key") is None and mock_llm_server_url
-                else None
+                f"{mock_llm_server_url}/v1" if using_mock_llm and mock_llm_server_url else None
             ),
         )
         sid = create_runner_bound_session(owner, agent_name=agent_name, runner_id=live_runner_id)