feat: add scripted model test utilities (#4362)

This commit is contained in:
Kazuhiro Sera
2026-08-13 11:20:33 +09:00
committed by GitHub
parent 0fc268e3e2
commit 05d6850da5
115 changed files with 11253 additions and 4953 deletions
+2
View File
@@ -188,6 +188,8 @@ The OpenAI Agents Python repository provides the Python Agents SDK, examples, an
Before submitting changes, ensure relevant checks pass and extend tests when you touch code.
For provider-neutral agent workflow tests, prefer `ScriptedModel` from `agents.testing` over adding a new mock or fake `Model`. Prefer `ScriptedRealtimeModel` from `agents.realtime.testing` for Realtime session tests, the scripted utilities from `agents.voice.testing` for Voice pipeline tests, and `scripted_sandbox_session()` from `agents.testing` for deterministic Sandbox session calls. Keep a specialized test double only when the test specifically requires provider-wire conversion, malformed streams, controlled suspension or concurrency, or an exact cancellation or lifecycle boundary that the scripted utilities cannot preserve; document that boundary in the test.
Before adding or changing async, retry, timeout, subprocess, PTY, warning, or xdist-sensitive tests, read [Performance and determinism](tests/README.md#performance-and-determinism) and preserve the applicable behavioral and lifecycle coverage while optimizing execution.
When `$code-change-verification` applies, run it to execute the required verification stack from the repository root. Rerun the full stack after applying fixes.
+4 -2
View File
@@ -1712,7 +1712,7 @@ async def validate_historical_resume_behavior(
from agents import Agent, Runner, RunState, function_tool
from agents.items import ToolCallOutputItem, TResponseOutputItem
from integration_tests._fake_model import QueuedFakeModel
from agents.testing import ModelStep, ScriptedModel
invocation_count = 0
if feature == "canonical_invocation_identity":
@@ -1771,7 +1771,9 @@ async def validate_historical_resume_behavior(
],
)
model_turns.append([final_message])
model = QueuedFakeModel(model_turns)
model = ScriptedModel(
[ModelStep(output=turn, response_id="queued-fake-response") for turn in model_turns]
)
agent = Agent(name="compat-agent", model=model, tools=[tool])
payload = json.loads(path.read_text(encoding="utf-8"))
restored = await RunState.from_json(agent, payload)
-120
View File
@@ -1,120 +0,0 @@
from __future__ import annotations
from collections.abc import AsyncIterator, Sequence
from copy import deepcopy
from typing import Any, cast
from openai.types.responses.response_prompt_param import ResponsePromptParam
from agents.agent_output import AgentOutputSchemaBase
from agents.handoffs import Handoff
from agents.items import (
ModelResponse,
TResponseInputItem,
TResponseOutputItem,
TResponseStreamEvent,
)
from agents.model_settings import ModelSettings
from agents.models.interface import Model, ModelTracing
from agents.tool import Tool
from agents.usage import Usage
class QueuedFakeModel(Model):
"""Deterministic non-streaming model for installed-distribution contracts."""
def __init__(self, turns: Sequence[Sequence[TResponseOutputItem]]) -> None:
self._turns = [list(turn) for turn in turns]
self.requests: list[dict[str, Any]] = []
def _record_request(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
previous_response_id: str | None,
conversation_id: str | None,
prompt: ResponsePromptParam | None,
) -> None:
self.requests.append(
{
"system_instructions": system_instructions,
"input": deepcopy(input),
"model_settings": model_settings,
"tools": list(tools),
"output_schema": output_schema,
"handoffs": list(handoffs),
"tracing": tracing,
"previous_response_id": previous_response_id,
"conversation_id": conversation_id,
"prompt": deepcopy(prompt),
}
)
async def get_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: ResponsePromptParam | None,
) -> ModelResponse:
self._record_request(
system_instructions,
input,
model_settings,
tools,
output_schema,
handoffs,
tracing,
previous_response_id,
conversation_id,
prompt,
)
if not self._turns:
raise AssertionError("QueuedFakeModel received an unexpected model request")
return ModelResponse(
output=self._turns.pop(0),
usage=Usage(requests=1),
response_id="queued-fake-response",
)
async def stream_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: ResponsePromptParam | None,
) -> AsyncIterator[TResponseStreamEvent]:
self._record_request(
system_instructions,
input,
model_settings,
tools,
output_schema,
handoffs,
tracing,
previous_response_id,
conversation_id,
prompt,
)
if False:
yield cast(TResponseStreamEvent, None)
raise AssertionError("QueuedFakeModel does not support streaming")
@@ -28,8 +28,8 @@ from agents.sandbox.session import (
SandboxSessionEvent,
)
from agents.sandbox.snapshot import NoopSnapshotSpec
from agents.testing import ModelStep, ScriptedModel, UnexpectedModelCall
from integration_tests._contract_support import _redaction_observables
from integration_tests._fake_model import QueuedFakeModel
from integration_tests.conftest import skip_or_fail
pytestmark = pytest.mark.security
@@ -204,7 +204,9 @@ async def test_runner_owned_local_sandbox_cannot_inspect_trusted_client_credenti
turns: list[list[TResponseOutputItem]] = [[tool_call]]
if not fail_after_inspection:
turns.append([final_message])
model = QueuedFakeModel(turns)
model = ScriptedModel(
[ModelStep(output=turn, response_id="queued-fake-response") for turn in turns]
)
client = _CredentialOwningDockerClient(trusted_credential=sentinel)
nested_mount_source = tmp_path / "nested-mount-probe"
nested_mount_source.mkdir()
@@ -234,8 +236,8 @@ async def test_runner_owned_local_sandbox_cannot_inspect_trusted_client_credenti
with caplog.at_level(logging.DEBUG):
if fail_after_inspection:
with pytest.raises(
AssertionError,
match="QueuedFakeModel received an unexpected model request",
UnexpectedModelCall,
match="no scripted steps remain",
) as exc_info:
await Runner.run(
agent,
@@ -277,24 +279,12 @@ async def test_runner_owned_local_sandbox_cannot_inspect_trusted_client_credenti
pass
docker_client.close()
expected_model_request_fields = {
"system_instructions",
"input",
"model_settings",
"tools",
"output_schema",
"handoffs",
"tracing",
"previous_response_id",
"conversation_id",
"prompt",
}
assert model.requests
assert all(set(request) == expected_model_request_fields for request in model.requests)
model_requests = repr(model.requests)
assert model.calls
assert all(call.streamed is False for call in model.calls)
model_requests = repr(model.calls)
model_visible_tool_outputs: list[object] = []
for request in model.requests:
model_input = request["input"]
for call in model.calls:
model_input = call.input
if not isinstance(model_input, list):
continue
model_visible_tool_outputs.extend(
+5 -13
View File
@@ -217,22 +217,11 @@ async def test_voice_pipeline_surfaces_tts_failures_without_hanging(
from agents.voice import (
AudioInput,
SingleAgentVoiceWorkflow,
TTSModel,
TTSModelSettings,
VoicePipeline,
VoiceStreamEventLifecycle,
)
from agents.voice.events import VoiceStreamEventError
class FailingTTSModel(TTSModel):
@property
def model_name(self) -> str:
return "failing-packaged-tts"
async def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]:
del text, settings
raise RuntimeError("Packaged TTS synthesis failed.")
yield b"" # pragma: no cover
from agents.voice.testing import ScriptedTTSModel
agent: Agent[Any] = Agent(
name="Packaged failing voice workflow agent",
@@ -242,7 +231,10 @@ async def test_voice_pipeline_surfaces_tts_failures_without_hanging(
)
pipeline = VoicePipeline(
workflow=SingleAgentVoiceWorkflow(agent),
tts_model=FailingTTSModel(),
tts_model=ScriptedTTSModel(
[RuntimeError("Packaged TTS synthesis failed.")],
model_name="failing-packaged-tts",
),
config={"tracing_disabled": True},
)
audio = np.frombuffer(integration_pcm_audio, dtype=np.int16).copy()
+478
View File
@@ -0,0 +1,478 @@
"""Deterministic Realtime model transport for session tests."""
from __future__ import annotations
import asyncio
import copy
from collections import deque
from collections.abc import Callable, Iterable, Sequence
from dataclasses import dataclass, field
from typing import TypeAlias, cast
from typing_extensions import Required, TypedDict
from ..models._trace import sanitize_url_for_trace
from .config import RealtimeSessionModelSettings
from .model import (
RealtimeModel,
RealtimeModelConfig,
RealtimeModelListener,
RealtimePlaybackTracker,
)
from .model_events import RealtimeModelErrorEvent, RealtimeModelEvent, RealtimeModelExceptionEvent
from .model_inputs import RealtimeModelSendEvent, RealtimeModelSendSessionUpdate
class RealtimeScriptError(Exception):
"""Base exception for an invalid or incompletely consumed Realtime script."""
class UnexpectedRealtimeSend(RealtimeScriptError):
"""Raised when an outbound event does not match the next scripted step."""
def __init__(
self,
message: str,
*,
actual: RealtimeModelSendEvent,
expected: RealtimeSendMatcher | None,
) -> None:
super().__init__(message)
self.actual = actual
self.expected = expected
class UnconsumedRealtimeSteps(RealtimeScriptError):
"""Raised when a test finishes before consuming every configured send step."""
def __init__(self, message: str, *, remaining_steps: int) -> None:
super().__init__(message)
self.remaining_steps = remaining_steps
RealtimeSendMatcher: TypeAlias = (
RealtimeModelSendEvent | type[RealtimeModelSendEvent] | Callable[[RealtimeModelSendEvent], bool]
)
@dataclass(frozen=True)
class RealtimeStep:
"""One expected outbound event and the normalized inbound events it triggers."""
expect: RealtimeSendMatcher
emit: Sequence[RealtimeModelEvent] = field(default_factory=tuple)
error: Exception | None = None
def __post_init__(self) -> None:
frozen_emit = tuple(self.emit)
if frozen_emit and self.error is not None:
raise ValueError("A RealtimeStep cannot define both emit events and an error.")
object.__setattr__(self, "emit", frozen_emit)
class RealtimeConnectCall(TypedDict, total=False):
"""A credential-free snapshot of one Realtime connection call."""
api_key_provided: Required[bool]
headers_provided: Required[bool]
url: str
initial_model_settings: RealtimeSessionModelSettings
playback_tracker: RealtimePlaybackTracker
call_id: str
@dataclass
class _RealtimeDeliveryResult:
done: asyncio.Future[BaseException | None]
pending_deliveries: int = 1
error: BaseException | None = None
@dataclass(frozen=True)
class _QueuedRealtimeDelivery:
events: Sequence[RealtimeModelEvent]
error: Exception | None
result: _RealtimeDeliveryResult
committed_close_calls: int
class ScriptedRealtimeModel(RealtimeModel):
"""An in-memory, listener-based Realtime transport with deterministic send steps."""
def __init__(
self,
steps: Iterable[RealtimeStep] = (),
*,
connect_events: Iterable[RealtimeModelEvent] = (),
connect_error: Exception | None = None,
close_error: Exception | None = None,
strict: bool = True,
) -> None:
connect_event_values = tuple(connect_events)
if connect_event_values and connect_error is not None:
raise ValueError(
"A ScriptedRealtimeModel cannot define both connect events and a connect error."
)
self._steps = [_snapshot_realtime_step(step) for step in steps]
self._connect_events = tuple(_snapshot_model_event(event) for event in connect_event_values)
self._connect_error = connect_error
self._close_error = close_error
self._strict = strict
self._listeners: list[RealtimeModelListener] = []
self._send_lock = asyncio.Lock()
self._delivery_queue: deque[_QueuedRealtimeDelivery] = deque()
self._delivery_worker: asyncio.Task[None] | None = None
self._active_delivery_result: _RealtimeDeliveryResult | None = None
self._connect_calls: list[RealtimeConnectCall] = []
self._sent_events: list[RealtimeModelSendEvent] = []
self.connected = False
self.closed = False
self.close_calls = 0
@property
def listeners(self) -> tuple[RealtimeModelListener, ...]:
"""Return the currently registered listeners."""
return tuple(self._listeners)
@property
def connect_calls(self) -> tuple[RealtimeConnectCall, ...]:
"""Return detached snapshots of recorded connection calls."""
return tuple(_clone_connect_call(call) for call in self._connect_calls)
@property
def sent_events(self) -> tuple[RealtimeModelSendEvent, ...]:
"""Return detached snapshots of recorded outbound events."""
return tuple(self._snapshot_send_event(event) for event in self._sent_events)
@property
def remaining_steps(self) -> int:
"""Return the number of expected outbound sends that remain."""
return len(self._steps)
async def connect(self, options: RealtimeModelConfig) -> None:
if self.connected or self._delivery_worker is not None:
raise AssertionError("Already connected")
self._connect_calls.append(_snapshot_connect_call(options))
if self._connect_error is not None:
raise self._connect_error
self.connected = True
self.closed = False
try:
async with self._send_lock:
self._ensure_emit_allowed()
event_snapshots = tuple(
_snapshot_model_event(event) for event in self._connect_events
)
result, reentrant = self._queue_delivery_locked(events=event_snapshots)
await self._finish_queued_delivery(result, reentrant)
except BaseException:
self.connected = False
self.closed = True
raise
def add_listener(self, listener: RealtimeModelListener) -> None:
if listener not in self._listeners:
self._listeners.append(listener)
def remove_listener(self, listener: RealtimeModelListener) -> None:
if listener in self._listeners:
self._listeners.remove(listener)
async def send_event(self, event: RealtimeModelSendEvent) -> None:
result, reentrant = await self._commit_send(event)
await self._finish_queued_delivery(result, reentrant)
async def send_event_if(
self,
event: RealtimeModelSendEvent,
send_if: Callable[[], bool],
) -> bool:
async with self._send_lock:
self._ensure_sendable()
if not send_if():
return False
result, reentrant = self._commit_send_locked(event)
await self._finish_queued_delivery(result, reentrant)
return True
async def emit(self, *events: RealtimeModelEvent) -> None:
"""Deliver normalized model events to all current listeners in order."""
async with self._send_lock:
self._ensure_emit_allowed()
event_snapshots = tuple(_snapshot_model_event(event) for event in events)
result, reentrant = self._queue_delivery_locked(events=event_snapshots)
await self._finish_queued_delivery(result, reentrant)
async def _broadcast_events(
self,
events: Sequence[RealtimeModelEvent],
*,
committed_close_calls: int,
) -> None:
for event in events:
listeners = tuple(self._listeners)
for listener in listeners:
if self.closed or self.close_calls != committed_close_calls:
return
await listener.on_event(event)
async def close(self) -> None:
self.close_calls += 1
if self.closed:
return
self.connected = False
self.closed = True
if self._close_error is not None:
raise self._close_error
def assert_complete(self) -> None:
"""Raise when expected outbound send steps remain unconsumed."""
if self._steps:
raise UnconsumedRealtimeSteps(
f"{len(self._steps)} scripted Realtime step(s) were not consumed.",
remaining_steps=len(self._steps),
)
async def _commit_send(
self, event: RealtimeModelSendEvent
) -> tuple[_RealtimeDeliveryResult, bool]:
async with self._send_lock:
self._ensure_sendable()
return self._commit_send_locked(event)
def _commit_send_locked(
self, event: RealtimeModelSendEvent
) -> tuple[_RealtimeDeliveryResult, bool]:
event_snapshot = self._snapshot_send_event(event)
step = self._pop_matching_step(event, actual_snapshot=event_snapshot)
self._sent_events.append(event_snapshot)
return self._queue_delivery_locked(
events=step.emit if step is not None else (),
error=step.error if step is not None else None,
)
def _queue_delivery_locked(
self,
*,
events: Sequence[RealtimeModelEvent],
error: Exception | None = None,
) -> tuple[_RealtimeDeliveryResult, bool]:
current_task = asyncio.current_task()
if current_task is None:
raise RuntimeError("A scripted Realtime send requires an active asyncio task.")
active_result = self._active_delivery_result
if current_task is self._delivery_worker and active_result is not None:
reentrant = True
result = active_result
result.pending_deliveries += 1
else:
reentrant = False
result = _RealtimeDeliveryResult(done=asyncio.get_running_loop().create_future())
self._delivery_queue.append(
_QueuedRealtimeDelivery(
events=tuple(events),
error=error,
result=result,
committed_close_calls=self.close_calls,
)
)
self._ensure_delivery_worker_locked()
return result, reentrant
async def _finish_queued_delivery(
self,
result: _RealtimeDeliveryResult,
reentrant: bool,
) -> None:
if reentrant:
return
error = await asyncio.shield(result.done)
if error is not None:
raise error
def _ensure_delivery_worker_locked(self) -> None:
if self._delivery_worker is None or self._delivery_worker.done():
self._delivery_worker = asyncio.create_task(self._drain_committed_sends())
async def _drain_committed_sends(self) -> None:
while True:
async with self._send_lock:
if not self._delivery_queue:
self._active_delivery_result = None
self._delivery_worker = None
return
delivery = self._delivery_queue.popleft()
self._active_delivery_result = delivery.result
error = await self._deliver_queued_events(delivery)
result = delivery.result
if error is not None and result.error is None:
result.error = error
result.pending_deliveries -= 1
if result.pending_deliveries == 0 and not result.done.done():
result.done.set_result(result.error)
async def _deliver_queued_events(
self, delivery: _QueuedRealtimeDelivery
) -> BaseException | None:
try:
if delivery.error is not None:
raise delivery.error
self._ensure_emit_allowed(delivery.committed_close_calls)
await self._broadcast_events(
delivery.events,
committed_close_calls=delivery.committed_close_calls,
)
except BaseException as error:
return error
return None
def _ensure_sendable(self) -> None:
if not self.connected or self.closed:
raise RealtimeScriptError(
"Cannot send an event while the scripted model is disconnected."
)
def _ensure_emit_allowed(
self,
committed_close_calls: int | None = None,
) -> None:
if (
not self.connected
or self.closed
or (committed_close_calls is not None and self.close_calls != committed_close_calls)
):
raise RealtimeScriptError(
"Cannot emit events while the scripted model is disconnected."
)
def _pop_matching_step(
self,
event: RealtimeModelSendEvent,
*,
actual_snapshot: RealtimeModelSendEvent,
) -> RealtimeStep | None:
if not self._steps:
if not self._strict:
return None
raise UnexpectedRealtimeSend(
"Unexpected Realtime send: no scripted steps remain.",
actual=actual_snapshot,
expected=None,
)
step = self._steps[0]
if not _matches(step.expect, event):
if not self._strict:
return None
raise UnexpectedRealtimeSend(
"Unexpected Realtime send: event did not match the next scripted expectation.",
actual=actual_snapshot,
expected=_snapshot_realtime_expectation(step.expect),
)
return self._steps.pop(0)
@staticmethod
def _snapshot_send_event(event: RealtimeModelSendEvent) -> RealtimeModelSendEvent:
return _snapshot_send_event(event)
def _snapshot_realtime_step(step: RealtimeStep) -> RealtimeStep:
return RealtimeStep(
expect=_snapshot_realtime_expectation(step.expect),
emit=tuple(_snapshot_model_event(event) for event in step.emit),
error=step.error,
)
def _snapshot_realtime_expectation(expectation: RealtimeSendMatcher) -> RealtimeSendMatcher:
if isinstance(expectation, type) or callable(expectation):
return expectation
return _snapshot_send_event(expectation)
def _snapshot_send_event(event: RealtimeModelSendEvent) -> RealtimeModelSendEvent:
if isinstance(event, RealtimeModelSendSessionUpdate):
return RealtimeModelSendSessionUpdate(
session_settings=_snapshot_model_settings(event.session_settings)
)
return copy.deepcopy(event)
def _snapshot_model_event(event: RealtimeModelEvent) -> RealtimeModelEvent:
if isinstance(event, RealtimeModelExceptionEvent):
return RealtimeModelExceptionEvent(
exception=event.exception,
context=copy.deepcopy(event.context),
)
if isinstance(event, RealtimeModelErrorEvent) and isinstance(event.error, Exception):
return RealtimeModelErrorEvent(error=event.error)
return copy.deepcopy(event)
def _matches(expectation: RealtimeSendMatcher, event: RealtimeModelSendEvent) -> bool:
if isinstance(expectation, type):
return isinstance(event, expectation)
if callable(expectation):
return expectation(_snapshot_send_event(event))
return event == expectation
def _snapshot_connect_call(options: RealtimeModelConfig) -> RealtimeConnectCall:
snapshot: RealtimeConnectCall = {
"api_key_provided": "api_key" in options,
"headers_provided": "headers" in options,
}
if "url" in options:
snapshot["url"] = sanitize_url_for_trace(options["url"])
if "initial_model_settings" in options:
snapshot["initial_model_settings"] = _snapshot_model_settings(
options["initial_model_settings"]
)
if "playback_tracker" in options:
snapshot["playback_tracker"] = options["playback_tracker"]
if "call_id" in options:
snapshot["call_id"] = options["call_id"]
return snapshot
def _clone_connect_call(call: RealtimeConnectCall) -> RealtimeConnectCall:
snapshot: RealtimeConnectCall = {
"api_key_provided": call["api_key_provided"],
"headers_provided": call["headers_provided"],
}
if "url" in call:
snapshot["url"] = call["url"]
if "initial_model_settings" in call:
snapshot["initial_model_settings"] = _snapshot_model_settings(
call["initial_model_settings"]
)
if "playback_tracker" in call:
snapshot["playback_tracker"] = call["playback_tracker"]
if "call_id" in call:
snapshot["call_id"] = call["call_id"]
return snapshot
def _snapshot_model_settings(
settings: RealtimeSessionModelSettings,
) -> RealtimeSessionModelSettings:
snapshot = cast(
RealtimeSessionModelSettings,
copy.deepcopy(
{key: value for key, value in settings.items() if key not in {"tools", "handoffs"}}
),
)
if "tools" in settings:
snapshot["tools"] = list(settings["tools"])
if "handoffs" in settings:
snapshot["handoffs"] = list(settings["handoffs"])
return snapshot
__all__ = [
"RealtimeConnectCall",
"RealtimeScriptError",
"RealtimeStep",
"ScriptedRealtimeModel",
"UnconsumedRealtimeSteps",
"UnexpectedRealtimeSend",
]
+45
View File
@@ -0,0 +1,45 @@
"""Deterministic test doubles for Agents SDK workflows."""
from .model import (
InvalidModelStep,
ModelCall,
ModelScriptError,
ModelStep,
ModelStepSpec,
ScriptedModel,
UnconsumedModelSteps,
UnexpectedModelCall,
assistant_message,
function_call,
)
from .sandbox import (
InvalidSandboxStep,
SandboxCall,
SandboxCallMatcherError,
SandboxScriptError,
SandboxStepSpec,
UnconsumedSandboxSteps,
UnexpectedSandboxCall,
scripted_sandbox_session,
)
__all__ = [
"InvalidModelStep",
"ModelCall",
"ModelScriptError",
"ModelStep",
"ModelStepSpec",
"InvalidSandboxStep",
"SandboxCall",
"SandboxCallMatcherError",
"SandboxScriptError",
"SandboxStepSpec",
"ScriptedModel",
"UnconsumedModelSteps",
"UnexpectedModelCall",
"UnconsumedSandboxSteps",
"UnexpectedSandboxCall",
"assistant_message",
"function_call",
"scripted_sandbox_session",
]
File diff suppressed because it is too large Load Diff
+570
View File
@@ -0,0 +1,570 @@
from __future__ import annotations
import copy
import inspect
import io
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Any, Literal, TypeAlias, cast, get_args
from typing_extensions import TypedDict
from ..editor import ApplyPatchOperation
from ..sandbox.apply_patch import PatchFormat
from ..sandbox.files import FileEntry
from ..sandbox.manifest import Manifest
from ..sandbox.session.base_sandbox_session import BaseSandboxSession
from ..sandbox.session.pty_types import PtyExecUpdate
from ..sandbox.session.sandbox_session_state import SandboxSessionState
from ..sandbox.snapshot import NoopSnapshot
from ..sandbox.types import ExecResult, User
SandboxMethod = Literal[
"apply_patch",
"exec",
"ls",
"mkdir",
"pty_exec_start",
"pty_write_stdin",
"read",
"rm",
"write",
]
SandboxStepReason = Literal["invalid_input", "unknown_method", "invalid_matcher", "invalid_outcome"]
_SCRIPTABLE_METHODS: frozenset[str] = frozenset(get_args(SandboxMethod))
_PTY_METHODS: frozenset[SandboxMethod] = frozenset({"pty_exec_start", "pty_write_stdin"})
_UNSUPPORTED_OPTIONAL_METHODS: frozenset[str] = frozenset(
{
"extract",
"hydrate_workspace",
"persist_workspace",
"resolve_exposed_port",
}
)
_HIDDEN_LIFECYCLE_METHODS: frozenset[str] = frozenset({"pty_terminate_all"})
for _method_name in _SCRIPTABLE_METHODS | _UNSUPPORTED_OPTIONAL_METHODS | _HIDDEN_LIFECYCLE_METHODS:
if not hasattr(BaseSandboxSession, _method_name):
raise RuntimeError(f"Unknown BaseSandboxSession method: {_method_name}")
class SandboxScriptError(Exception):
"""Base exception for an invalid or incompletely consumed sandbox script."""
class InvalidSandboxStep(SandboxScriptError):
"""Raised when a sandbox step is invalid at factory construction time."""
def __init__(
self,
message: str,
*,
reason: SandboxStepReason,
input_index: int,
method: str | None,
) -> None:
super().__init__(message)
self.reason = reason
self.input_index = input_index
self.method = method
class UnexpectedSandboxCall(SandboxScriptError):
"""Raised when a call does not match the next configured sandbox step."""
def __init__(
self,
message: str,
*,
call: SandboxCall,
call_index: int,
expected_method: str | None,
remaining_steps: int,
) -> None:
super().__init__(message)
self.call = call
self.call_index = call_index
self.actual_method = call.method
self.expected_method = expected_method
self.remaining_steps = remaining_steps
class SandboxCallMatcherError(SandboxScriptError):
"""Raised when a sandbox step matcher rejects its call."""
def __init__(self, message: str, *, call: SandboxCall, call_index: int) -> None:
super().__init__(message)
self.call = call
self.call_index = call_index
self.method = call.method
class UnconsumedSandboxSteps(SandboxScriptError):
"""Raised when configured sandbox steps remain unconsumed."""
def __init__(
self,
message: str,
*,
remaining_steps: int,
pending_methods: tuple[str, ...],
) -> None:
super().__init__(message)
self.remaining_steps = remaining_steps
self.pending_methods = pending_methods
@dataclass(frozen=True)
class SandboxCall:
"""A detached invocation-time sandbox call snapshot."""
call_index: int
method: str
args: tuple[Any, ...]
kwargs: Mapping[str, Any]
SandboxMatcher: TypeAlias = Callable[[SandboxCall], bool | None]
SandboxResponder: TypeAlias = Callable[[SandboxCall], Any | Awaitable[Any]]
class SandboxStepSpec(TypedDict, total=False):
"""Dictionary form of one FIFO scripted sandbox call."""
method: SandboxMethod
match: SandboxMatcher
result: Any
responder: SandboxResponder
error: Exception
_STEP_FIELDS = frozenset(SandboxStepSpec.__annotations__)
@dataclass(frozen=True)
class _SandboxStep:
method: SandboxMethod
match: SandboxMatcher | None
outcome: Literal["result", "responder", "error"]
value: Any
def _snapshot_value(value: Any) -> Any:
if isinstance(value, io.BytesIO):
if value.closed:
raise TypeError("Cannot snapshot a closed sandbox byte stream.")
byte_snapshot = io.BytesIO(value.getvalue())
byte_snapshot.seek(value.tell())
return byte_snapshot
if isinstance(value, io.StringIO):
if value.closed:
raise TypeError("Cannot snapshot a closed sandbox text stream.")
text_snapshot = io.StringIO(value.getvalue())
text_snapshot.seek(value.tell())
return text_snapshot
if isinstance(value, io.IOBase):
raise TypeError("Sandbox stream snapshots support only io.BytesIO and io.StringIO.")
if callable(value):
return value
if isinstance(value, tuple):
return tuple(_snapshot_value(item) for item in value)
if isinstance(value, list):
return [_snapshot_value(item) for item in value]
if isinstance(value, dict):
return {_snapshot_value(key): _snapshot_value(item) for key, item in value.items()}
if isinstance(value, set):
return {_snapshot_value(item) for item in value}
return copy.deepcopy(value)
def _snapshot_call(call: SandboxCall) -> SandboxCall:
return SandboxCall(
call_index=call.call_index,
method=call.method,
args=tuple(_snapshot_value(call.args)),
kwargs=MappingProxyType(
{name: _snapshot_value(value) for name, value in call.kwargs.items()}
),
)
def _invalid_step(
*,
reason: SandboxStepReason,
input_index: int,
method: str | None,
detail: str,
) -> InvalidSandboxStep:
return InvalidSandboxStep(
f"Scripted sandbox step #{input_index + 1} {detail}.",
reason=reason,
input_index=input_index,
method=method,
)
def _normalize_step(input: object, input_index: int) -> _SandboxStep:
if not isinstance(input, Mapping):
raise _invalid_step(
reason="invalid_input",
input_index=input_index,
method=None,
detail="must be a mapping",
)
method = input.get("method")
if any(field not in _STEP_FIELDS for field in input):
raise _invalid_step(
reason="invalid_input",
input_index=input_index,
method=method if isinstance(method, str) else None,
detail="contains unsupported fields",
)
if not isinstance(method, str) or method not in _SCRIPTABLE_METHODS:
raise _invalid_step(
reason="unknown_method",
input_index=input_index,
method=method if isinstance(method, str) else None,
detail="has an unknown method",
)
matcher = input.get("match")
if isinstance(matcher, io.IOBase):
raise _invalid_step(
reason="invalid_matcher",
input_index=input_index,
method=method,
detail=f"for {method} must use a non-stream callable matcher",
)
if matcher is not None and not callable(matcher):
raise _invalid_step(
reason="invalid_matcher",
input_index=input_index,
method=method,
detail=f"for {method} must use a callable matcher",
)
outcomes = [name for name in ("result", "responder", "error") if name in input]
if len(outcomes) != 1:
raise _invalid_step(
reason="invalid_outcome",
input_index=input_index,
method=method,
detail=f"for {method} must define exactly one outcome",
)
outcome = cast(Literal["result", "responder", "error"], outcomes[0])
value = input[outcome]
if outcome == "responder" and isinstance(value, io.IOBase):
raise _invalid_step(
reason="invalid_outcome",
input_index=input_index,
method=method,
detail=f"for {method} must use a non-stream callable responder",
)
if outcome == "responder" and not callable(value):
raise _invalid_step(
reason="invalid_outcome",
input_index=input_index,
method=method,
detail=f"for {method} must use a callable responder",
)
if outcome == "error" and not isinstance(value, Exception):
raise _invalid_step(
reason="invalid_outcome",
input_index=input_index,
method=method,
detail=f"for {method} must use an Exception",
)
if outcome == "result":
try:
value = _snapshot_value(value)
except TypeError as error:
raise _invalid_step(
reason="invalid_outcome",
input_index=input_index,
method=method,
detail=f"for {method} contains a result that cannot be snapshotted",
) from error
return _SandboxStep(
method=cast(SandboxMethod, method),
match=cast(SandboxMatcher | None, matcher),
outcome=outcome,
value=value,
)
class _ScriptedSandboxSession(BaseSandboxSession):
def __init__(self, steps: Sequence[_SandboxStep], *, manifest: Manifest | None) -> None:
self.state = SandboxSessionState(
type="scripted",
snapshot=NoopSnapshot(id="scripted"),
manifest=copy.deepcopy(manifest) if manifest is not None else Manifest(),
)
self._steps = list(steps)
configured_methods = {step.method for step in steps}
if configured_methods & _PTY_METHODS:
configured_methods.update(_PTY_METHODS)
self._configured_methods = frozenset(configured_methods)
self._calls: list[SandboxCall] = []
self._running = False
def __getattribute__(self, name: str) -> Any:
if name in _SCRIPTABLE_METHODS:
configured = object.__getattribute__(self, "_configured_methods")
if name not in configured:
raise AttributeError(name)
if name in _UNSUPPORTED_OPTIONAL_METHODS | _HIDDEN_LIFECYCLE_METHODS:
raise AttributeError(name)
return super().__getattribute__(name)
def __dir__(self) -> list[str]:
configured = self._configured_methods
return sorted(
name
for name in super().__dir__()
if name not in _UNSUPPORTED_OPTIONAL_METHODS | _HIDDEN_LIFECYCLE_METHODS
and (name not in _SCRIPTABLE_METHODS or name in configured)
)
@property
def calls(self) -> tuple[SandboxCall, ...]:
"""Return detached call-history snapshots in invocation order."""
return tuple(_snapshot_call(call) for call in self._calls)
@property
def remaining_steps(self) -> int:
"""Return the number of configured calls that remain."""
return len(self._steps)
def assert_complete(self) -> None:
"""Raise when configured sandbox calls remain unconsumed."""
if self._steps:
pending_methods = tuple(step.method for step in self._steps)
raise UnconsumedSandboxSteps(
f"Scripted sandbox session has {len(self._steps)} unconsumed step(s).",
remaining_steps=len(self._steps),
pending_methods=pending_methods,
)
async def _invoke(
self, method: SandboxMethod, args: tuple[Any, ...], kwargs: dict[str, Any]
) -> Any:
call = SandboxCall(
call_index=len(self._calls),
method=method,
args=tuple(_snapshot_value(args)),
kwargs=MappingProxyType(
{name: _snapshot_value(value) for name, value in kwargs.items()}
),
)
call_index = call.call_index
self._calls.append(call)
if not self._steps:
raise UnexpectedSandboxCall(
f"Unexpected sandbox {method} call #{call_index + 1}: no scripted steps remain.",
call=_snapshot_call(call),
call_index=call_index,
expected_method=None,
remaining_steps=0,
)
step = self._steps[0]
if step.method != method:
raise UnexpectedSandboxCall(
f"Unexpected sandbox {method} call #{call_index + 1}; expected {step.method}.",
call=_snapshot_call(call),
call_index=call_index,
expected_method=step.method,
remaining_steps=len(self._steps),
)
matcher_call = _snapshot_call(call)
if step.match is not None and step.match(matcher_call) is False:
raise SandboxCallMatcherError(
f"Sandbox matcher rejected {method} call #{call_index + 1}.",
call=_snapshot_call(call),
call_index=call_index,
)
self._steps.pop(0)
if step.outcome == "error":
raise step.value
if step.outcome == "responder":
result = step.value(_snapshot_call(call))
if inspect.isawaitable(result):
result = await result
return result
return step.value
async def start(self) -> None:
self._running = True
self.state.workspace_root_ready = True
async def stop(self) -> None:
self._running = False
async def _before_shutdown(self) -> None:
return
async def running(self) -> bool:
return self._running
def supports_pty(self) -> bool:
return _PTY_METHODS.issubset(self._configured_methods)
async def exec(
self,
*command: str | Path,
timeout: float | None = None,
shell: bool | list[str] = True,
user: str | User | None = None,
) -> ExecResult:
return cast(
ExecResult,
await self._invoke(
"exec",
command,
{"timeout": timeout, "shell": shell, "user": user},
),
)
async def read(self, path: Path, *, user: str | User | None = None) -> io.IOBase:
return cast(io.IOBase, await self._invoke("read", (path,), {"user": user}))
async def write(
self,
path: Path,
data: io.IOBase,
*,
user: str | User | None = None,
) -> None:
await self._invoke("write", (path, data), {"user": user})
async def ls(
self,
path: Path | str,
*,
user: str | User | None = None,
) -> list[FileEntry]:
return cast(list[FileEntry], await self._invoke("ls", (path,), {"user": user}))
async def rm(
self,
path: Path | str,
*,
recursive: bool = False,
user: str | User | None = None,
) -> None:
await self._invoke("rm", (path,), {"recursive": recursive, "user": user})
async def mkdir(
self,
path: Path | str,
*,
parents: bool = False,
user: str | User | None = None,
) -> None:
await self._invoke("mkdir", (path,), {"parents": parents, "user": user})
async def apply_patch(
self,
operations: ApplyPatchOperation
| dict[str, object]
| list[ApplyPatchOperation | dict[str, object]],
*,
patch_format: PatchFormat | Literal["v4a"] = "v4a",
) -> str:
return cast(
str,
await self._invoke(
"apply_patch",
(operations,),
{"patch_format": patch_format},
),
)
async def _exec_internal(
self,
*command: str | Path,
timeout: float | None = None,
) -> ExecResult:
_ = (command, timeout)
raise NotImplementedError
async def persist_workspace(self) -> io.IOBase:
raise NotImplementedError
async def hydrate_workspace(self, data: io.IOBase) -> None:
_ = data
raise NotImplementedError
async def pty_exec_start(
self,
*command: str | Path,
timeout: float | None = None,
shell: bool | list[str] = True,
user: str | User | None = None,
tty: bool = False,
yield_time_s: float | None = None,
max_output_tokens: int | None = None,
) -> PtyExecUpdate:
return cast(
PtyExecUpdate,
await self._invoke(
"pty_exec_start",
command,
{
"timeout": timeout,
"shell": shell,
"user": user,
"tty": tty,
"yield_time_s": yield_time_s,
"max_output_tokens": max_output_tokens,
},
),
)
async def pty_write_stdin(
self,
*,
session_id: int,
chars: str,
yield_time_s: float | None = None,
max_output_tokens: int | None = None,
) -> PtyExecUpdate:
return cast(
PtyExecUpdate,
await self._invoke(
"pty_write_stdin",
(),
{
"session_id": session_id,
"chars": chars,
"yield_time_s": yield_time_s,
"max_output_tokens": max_output_tokens,
},
),
)
def scripted_sandbox_session(
steps: Iterable[SandboxStepSpec | Mapping[str, Any]] = (),
*,
manifest: Manifest | None = None,
) -> _ScriptedSandboxSession:
"""Create a deterministic provider-free sandbox session for agent workflow tests.
Each FIFO step defines ``method`` plus exactly one of ``result``, ``responder``, or ``error``.
An optional ``match`` callable receives a detached ``SandboxCall``. The returned object is the
session itself, so pass it directly to ``SandboxRunConfig(session=session)``. Only configured
model-facing methods are visible. The two PTY methods are exposed together when either one is
configured because they form one advertised session capability. Use a custom
``BaseSandboxSession`` or a real provider for lifecycle, persistence, mount, or broader
filesystem behavior.
"""
normalized = [_normalize_step(step, index) for index, step in enumerate(steps)]
return _ScriptedSandboxSession(normalized, manifest=manifest)
+11
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import copy
import json
from collections.abc import Mapping
from dataclasses import field
@@ -11,6 +12,7 @@ from pydantic import BeforeValidator, JsonValue, TypeAdapter, ValidationError
from pydantic.dataclasses import dataclass
_RAW_USAGE_ATTRIBUTE = "_agents_sdk_raw_usage"
_NORMALIZED_USAGE_ATTRIBUTE = "_agents_sdk_normalized_usage"
_RAW_USAGE_ADAPTER = TypeAdapter(dict[str, JsonValue])
_RAW_USAGE_MISSING = object()
@@ -333,6 +335,10 @@ def _requests_for_response_without_usage(response: Any) -> int:
def _response_usage_to_usage(response_usage: Any) -> Usage:
"""Convert Responses API usage, including adapter-supplied per-request details."""
normalized_usage = getattr(response_usage, _NORMALIZED_USAGE_ATTRIBUTE, None)
if isinstance(normalized_usage, Usage):
return copy.deepcopy(normalized_usage)
request_usages = getattr(response_usage, "_agents_sdk_request_usages", None)
request_count = getattr(response_usage, "_agents_sdk_request_count", 1)
@@ -362,6 +368,11 @@ def _response_usage_to_usage(response_usage: Any) -> Usage:
)
def _attach_normalized_usage(target: Any, usage: Usage) -> None:
"""Attach a detached normalized usage snapshot for lossless internal conversion."""
object.__setattr__(target, _NORMALIZED_USAGE_ATTRIBUTE, copy.deepcopy(usage))
def _serialize_usage_details(details: Any, default: dict[str, int]) -> dict[str, Any]:
"""Serialize token details while applying the given default when empty."""
if hasattr(details, "model_dump"):
+421
View File
@@ -0,0 +1,421 @@
"""Deterministic speech and workflow components for Voice pipeline tests."""
from __future__ import annotations
import copy
from collections.abc import AsyncIterator, Iterable, Sequence
from dataclasses import dataclass, field
from typing import Any
from .imports import np
from .input import AudioInput, StreamedAudioInput
from .model import (
StreamedTranscriptionSession,
STTModel,
STTModelSettings,
TTSModel,
TTSModelSettings,
)
from .workflow import VoiceWorkflowBase
class VoiceScriptError(Exception):
"""Base exception for an invalid or incompletely consumed Voice script."""
class UnexpectedVoiceCall(VoiceScriptError):
"""Raised when a Voice component is called after its script is exhausted."""
def __init__(self, message: str, *, operation: str) -> None:
super().__init__(message)
self.operation = operation
class UnconsumedVoiceSteps(VoiceScriptError):
"""Raised when a test finishes before consuming every configured Voice step."""
def __init__(self, message: str, *, remaining_steps: int) -> None:
super().__init__(message)
self.remaining_steps = remaining_steps
@dataclass(frozen=True)
class STTCall:
"""A recorded static transcription call."""
input: AudioInput
settings: STTModelSettings
trace_include_sensitive_data: bool
trace_include_sensitive_audio_data: bool
@dataclass(frozen=True)
class STTSessionCall:
"""A recorded streamed transcription-session creation call."""
input: StreamedAudioInput
settings: STTModelSettings
trace_include_sensitive_data: bool
trace_include_sensitive_audio_data: bool
@dataclass(frozen=True)
class TTSCall:
"""A recorded text-to-speech call."""
text: str
settings: TTSModelSettings
@dataclass(frozen=True)
class TTSResult:
"""The PCM byte chunks returned by one text-to-speech call."""
chunks: Sequence[bytes] = field(default_factory=tuple)
TranscriptionResult = str | Exception
TTSResultItem = TTSResult | Sequence[bytes] | Exception
WorkflowResult = str | Sequence[str] | Exception
_START_NOT_CONFIGURED: Any = object()
def _snapshot_audio_input(input: AudioInput) -> AudioInput:
return AudioInput(
buffer=input.buffer.copy(),
frame_rate=input.frame_rate,
sample_width=input.sample_width,
channels=input.channels,
)
def _snapshot_stt_call(call: STTCall) -> STTCall:
return STTCall(
input=_snapshot_audio_input(call.input),
settings=copy.deepcopy(call.settings),
trace_include_sensitive_data=call.trace_include_sensitive_data,
trace_include_sensitive_audio_data=call.trace_include_sensitive_audio_data,
)
def _snapshot_stt_session_call(call: STTSessionCall) -> STTSessionCall:
return STTSessionCall(
input=call.input,
settings=copy.deepcopy(call.settings),
trace_include_sensitive_data=call.trace_include_sensitive_data,
trace_include_sensitive_audio_data=call.trace_include_sensitive_audio_data,
)
def _snapshot_tts_call(call: TTSCall) -> TTSCall:
return TTSCall(text=call.text, settings=copy.deepcopy(call.settings))
class ScriptedTranscriptionSession(StreamedTranscriptionSession):
"""A closable stream of configured transcription turns."""
def __init__(
self,
turns: str | Iterable[TranscriptionResult] = (),
*,
close_error: Exception | None = None,
) -> None:
self._turns: list[TranscriptionResult] = [turns] if isinstance(turns, str) else list(turns)
self._close_error = close_error
self.closed = False
self.close_calls = 0
async def transcribe_turns(self) -> AsyncIterator[str]:
while self._turns and not self.closed:
turn = self._turns.pop(0)
if isinstance(turn, Exception):
raise turn
yield turn
async def close(self) -> None:
self.close_calls += 1
if self.closed:
return
self.closed = True
if self._close_error is not None:
raise self._close_error
def assert_complete(self) -> None:
"""Raise when configured transcript turns remain unconsumed."""
if self._turns:
raise UnconsumedVoiceSteps(
f"{len(self._turns)} scripted transcription turn(s) were not consumed.",
remaining_steps=len(self._turns),
)
class ScriptedSTTModel(STTModel):
"""A deterministic speech-to-text model for static and streamed audio tests."""
def __init__(
self,
transcriptions: str | Iterable[TranscriptionResult] = (),
*,
sessions: str
| Iterable[ScriptedTranscriptionSession | Iterable[TranscriptionResult] | Exception] = (),
model_name: str = "scripted-stt",
) -> None:
self._transcriptions: list[TranscriptionResult] = (
[transcriptions] if isinstance(transcriptions, str) else list(transcriptions)
)
configured_sessions = [sessions] if isinstance(sessions, str) else sessions
self._sessions: list[
ScriptedTranscriptionSession | tuple[TranscriptionResult, ...] | Exception
] = [
configured
if isinstance(configured, ScriptedTranscriptionSession | Exception)
else (configured,)
if isinstance(configured, str)
else tuple(configured)
for configured in configured_sessions
]
self._model_name = model_name
self._calls: list[STTCall] = []
self._session_calls: list[STTSessionCall] = []
self._created_sessions: list[ScriptedTranscriptionSession] = []
@property
def model_name(self) -> str:
return self._model_name
@property
def calls(self) -> tuple[STTCall, ...]:
"""Return detached snapshots of recorded static transcription calls."""
return tuple(_snapshot_stt_call(call) for call in self._calls)
@property
def session_calls(self) -> tuple[STTSessionCall, ...]:
"""Return detached snapshots of recorded streamed-session calls."""
return tuple(_snapshot_stt_session_call(call) for call in self._session_calls)
@property
def created_sessions(self) -> tuple[ScriptedTranscriptionSession, ...]:
"""Return created sessions while preserving their live object identity."""
return tuple(self._created_sessions)
async def transcribe(
self,
input: AudioInput,
settings: STTModelSettings,
trace_include_sensitive_data: bool,
trace_include_sensitive_audio_data: bool,
) -> str:
call = STTCall(
input=input,
settings=settings,
trace_include_sensitive_data=trace_include_sensitive_data,
trace_include_sensitive_audio_data=trace_include_sensitive_audio_data,
)
call = _snapshot_stt_call(call)
self._calls.append(call)
if not self._transcriptions:
raise UnexpectedVoiceCall(
"Unexpected static transcription call: no scripted transcriptions remain.",
operation="static_transcription",
)
result = self._transcriptions.pop(0)
if isinstance(result, Exception):
raise result
return result
async def create_session(
self,
input: StreamedAudioInput,
settings: STTModelSettings,
trace_include_sensitive_data: bool,
trace_include_sensitive_audio_data: bool,
) -> StreamedTranscriptionSession:
call = STTSessionCall(
input=input,
settings=settings,
trace_include_sensitive_data=trace_include_sensitive_data,
trace_include_sensitive_audio_data=trace_include_sensitive_audio_data,
)
call = _snapshot_stt_session_call(call)
self._session_calls.append(call)
if not self._sessions:
raise UnexpectedVoiceCall(
"Unexpected streamed transcription session: no scripted sessions remain.",
operation="streamed_session",
)
configured = self._sessions.pop(0)
if isinstance(configured, Exception):
raise configured
session = (
configured
if isinstance(configured, ScriptedTranscriptionSession)
else ScriptedTranscriptionSession(configured)
)
self._created_sessions.append(session)
return session
def assert_complete(self) -> None:
"""Raise when static transcriptions or sessions remain unconsumed."""
remaining = len(self._transcriptions) + len(self._sessions)
if remaining:
raise UnconsumedVoiceSteps(
f"{remaining} scripted STT step(s) were not consumed.",
remaining_steps=remaining,
)
for session in self._created_sessions:
session.assert_complete()
class ScriptedTTSModel(TTSModel):
"""A deterministic text-to-speech model that yields configured PCM byte chunks."""
def __init__(
self,
results: Iterable[TTSResultItem] = (),
*,
model_name: str = "scripted-tts",
) -> None:
self._results = [self._coerce_result(result) for result in results]
self._model_name = model_name
self._calls: list[TTSCall] = []
@property
def model_name(self) -> str:
return self._model_name
@property
def calls(self) -> tuple[TTSCall, ...]:
"""Return detached snapshots of recorded text-to-speech calls."""
return tuple(_snapshot_tts_call(call) for call in self._calls)
async def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]:
call = _snapshot_tts_call(TTSCall(text=text, settings=settings))
self._calls.append(call)
if not self._results:
raise UnexpectedVoiceCall(
"Unexpected TTS call: no scripted results remain.",
operation="tts",
)
result = self._results.pop(0)
if isinstance(result, Exception):
raise result
for chunk in result.chunks:
yield chunk
def assert_complete(self) -> None:
"""Raise when configured TTS results remain unconsumed."""
if self._results:
raise UnconsumedVoiceSteps(
f"{len(self._results)} scripted TTS result(s) were not consumed.",
remaining_steps=len(self._results),
)
@staticmethod
def _coerce_result(result: TTSResultItem) -> TTSResult | Exception:
if isinstance(result, Exception):
return result
if isinstance(result, TTSResult):
return TTSResult(chunks=tuple(result.chunks))
return TTSResult(chunks=tuple(result))
class ScriptedVoiceWorkflow(VoiceWorkflowBase):
"""A deterministic Voice workflow that yields configured text fragments per turn."""
def __init__(
self,
turns: str | Iterable[WorkflowResult] = (),
*,
start: str | Sequence[str] | Exception = _START_NOT_CONFIGURED,
) -> None:
configured_turns = [turns] if isinstance(turns, str) else turns
self._turns = [
turn if isinstance(turn, Exception) else _normalize_fragments(turn)
for turn in configured_turns
]
self._start = (
()
if start is _START_NOT_CONFIGURED
else start
if isinstance(start, Exception)
else _normalize_fragments(start)
)
self._start_configured = start is not _START_NOT_CONFIGURED
self._start_pending = self._start_configured
self._transcriptions: list[str] = []
@property
def transcriptions(self) -> tuple[str, ...]:
"""Return the recorded workflow transcriptions."""
return tuple(self._transcriptions)
async def on_start(self) -> AsyncIterator[str]:
if self._start_configured and not self._start_pending:
raise UnexpectedVoiceCall(
"Unexpected workflow startup call: no scripted startup step remains.",
operation="workflow_start",
)
self._start_pending = False
if isinstance(self._start, Exception):
raise self._start
for fragment in self._start:
yield fragment
async def run(self, transcription: str) -> AsyncIterator[str]:
self._transcriptions.append(transcription)
if not self._turns:
raise UnexpectedVoiceCall(
"Unexpected workflow turn: no scripted turns remain.",
operation="workflow_turn",
)
result = self._turns.pop(0)
if isinstance(result, Exception):
raise result
for fragment in result:
yield fragment
def assert_complete(self) -> None:
"""Raise when the startup step or configured workflow turns remain unconsumed."""
remaining = int(self._start_pending) + len(self._turns)
if not remaining:
return
if self._start_pending and not self._turns:
raise UnconsumedVoiceSteps(
"1 scripted workflow startup step was not consumed.",
remaining_steps=1,
)
if not self._start_pending:
raise UnconsumedVoiceSteps(
f"{len(self._turns)} scripted workflow turn(s) were not consumed.",
remaining_steps=len(self._turns),
)
raise UnconsumedVoiceSteps(
f"{remaining} scripted workflow step(s) were not consumed.",
remaining_steps=remaining,
)
def _normalize_fragments(fragments: str | Sequence[str]) -> tuple[str, ...]:
return (fragments,) if isinstance(fragments, str) else tuple(fragments)
def pcm16_samples(samples: Iterable[int]) -> bytes:
"""Encode integer samples as native little-endian PCM16 bytes."""
return np.asarray(list(samples), dtype="<i2").tobytes()
__all__ = [
"STTCall",
"STTSessionCall",
"ScriptedSTTModel",
"ScriptedTTSModel",
"ScriptedTranscriptionSession",
"ScriptedVoiceWorkflow",
"TTSCall",
"TTSResult",
"UnconsumedVoiceSteps",
"UnexpectedVoiceCall",
"VoiceScriptError",
"pcm16_samples",
]
+2
View File
@@ -4,6 +4,8 @@ Before running any tests, make sure you have `uv` installed (and ideally run `ma
## Running tests
For provider-neutral agent workflow tests, prefer `ScriptedModel` from `agents.testing` instead of adding a new mock or fake `Model`. Use `ScriptedRealtimeModel` from `agents.realtime.testing` for Realtime session tests, the scripted utilities from `agents.voice.testing` for Voice pipeline tests, and `scripted_sandbox_session()` from `agents.testing` for deterministic Sandbox session calls. Keep a specialized test double only when the test specifically requires provider-wire conversion, malformed streams, controlled suspension or concurrency, or an exact cancellation or lifecycle boundary that the scripted utilities cannot preserve; document that boundary in the test.
```
make tests
```
@@ -25,8 +25,8 @@ from agents import Agent, Runner, TResponseInputItem, function_tool
from agents.extensions.memory import AdvancedSQLiteSession
from agents.result import RunResult
from agents.run_context import RunContextWrapper
from agents.testing import ScriptedModel
from agents.usage import Usage
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
# Mark all tests in this file as asyncio
@@ -52,8 +52,8 @@ async def test_tool(query: str) -> str:
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel(), tools=[test_tool])
"""Fixture for a basic agent with a scripted model."""
return Agent(name="test", model=ScriptedModel(), tools=[test_tool])
@pytest.fixture
@@ -74,7 +74,7 @@ def usage_data() -> Usage:
def create_mock_run_result(usage: Usage | None = None, agent: Agent | None = None) -> RunResult:
"""Helper function to create a mock RunResult for testing."""
if agent is None:
agent = Agent(name="test", model=FakeModel())
agent = Agent(name="test", model=ScriptedModel())
if usage is None:
usage = Usage(
@@ -2057,10 +2057,10 @@ async def test_runner_integration_with_usage_tracking(agent: Agent):
# Ignore errors in test helper
pass
# Set up fake model responses
assert isinstance(agent.model, FakeModel)
fake_model = agent.model
fake_model.set_next_output([get_text_message("San Francisco")])
# Set up scripted model responses.
assert isinstance(agent.model, ScriptedModel)
scripted_model = agent.model
scripted_model.enqueue([get_text_message("San Francisco")])
# First turn
result1 = await Runner.run(
@@ -2072,7 +2072,7 @@ async def test_runner_integration_with_usage_tracking(agent: Agent):
await store_session_usage(result1, session)
# Second turn
fake_model.set_next_output([get_text_message("California")])
scripted_model.enqueue([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
await store_session_usage(result2, session)
@@ -2085,7 +2085,7 @@ async def test_runner_integration_with_usage_tracking(agent: Agent):
session_usage = await session.get_session_usage()
assert session_usage is not None
assert session_usage["total_turns"] == 2
# FakeModel doesn't generate realistic usage data, so we just check structure exists
# ScriptedModel doesn't generate realistic usage data, so we just check structure exists
assert "requests" in session_usage
assert "total_tokens" in session_usage
@@ -2480,9 +2480,9 @@ async def test_tool_execution_integration(agent: Agent):
session_id = "tool_integration_test"
session = AdvancedSQLiteSession(session_id=session_id, create_tables=True)
# Set up the fake model to trigger a tool call
fake_model = cast(FakeModel, agent.model)
fake_model.set_next_output(
# Set up the scripted model to trigger a tool call.
scripted_model = cast(ScriptedModel, agent.model)
scripted_model.enqueue(
[
{ # type: ignore
"type": "function_call",
@@ -2494,7 +2494,7 @@ async def test_tool_execution_integration(agent: Agent):
)
# Then set the final response
fake_model.set_next_output([get_text_message("Tool executed successfully")])
scripted_model.enqueue([get_text_message("Tool executed successfully")])
# Run the agent
result = await Runner.run(
@@ -2687,8 +2687,8 @@ async def test_runner_with_session_settings_override(agent: Agent):
await session.add_items(items)
# Use RunConfig to override limit to 2
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("Got it")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("Got it")])
await Runner.run(
agent,
@@ -2700,7 +2700,7 @@ async def test_runner_with_session_settings_override(agent: Agent):
)
# Verify the agent received only the last 2 history items + new question
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
@@ -19,7 +19,7 @@ pytest.importorskip("aiosqlite") # Skip tests if aiosqlite is not installed
from agents import Agent, Runner, TResponseInputItem
from agents.extensions.memory import AsyncSQLiteSession
from agents.memory import SessionSettings
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
pytestmark = pytest.mark.asyncio
@@ -33,8 +33,8 @@ def _assert_cancel_message(exc: asyncio.CancelledError, expected: str) -> None:
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
"""Fixture for a basic agent with a scripted model."""
return Agent(name="test", model=ScriptedModel())
def _item_ids(items: Sequence[TResponseInputItem]) -> list[str]:
@@ -267,9 +267,9 @@ async def test_async_sqlite_session_runner_integration(agent: Agent):
db_path = Path(temp_dir) / "async_runner_integration.db"
session = AsyncSQLiteSession("runner_integration_test", db_path)
assert isinstance(agent.model, FakeModel)
assert isinstance(agent.model, ScriptedModel)
agent.model.set_next_output([get_text_message("San Francisco")])
agent.model.enqueue([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
@@ -277,11 +277,11 @@ async def test_async_sqlite_session_runner_integration(agent: Agent):
)
assert result1.final_output == "San Francisco"
agent.model.set_next_output([get_text_message("California")])
agent.model.enqueue([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
assert isinstance(last_input, list)
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
@@ -296,14 +296,14 @@ async def test_async_sqlite_session_session_isolation(agent: Agent):
session1 = AsyncSQLiteSession("session_1", db_path)
session2 = AsyncSQLiteSession("session_2", db_path)
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("I like cats.")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("I like cats.")])
await Runner.run(agent, "I like cats.", session=session1)
agent.model.set_next_output([get_text_message("I like dogs.")])
agent.model.enqueue([get_text_message("I like dogs.")])
await Runner.run(agent, "I like dogs.", session=session2)
agent.model.set_next_output([get_text_message("You said you like cats.")])
agent.model.enqueue([get_text_message("You said you like cats.")])
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
assert "cats" in result.final_output.lower()
assert "dogs" not in result.final_output.lower()
@@ -45,7 +45,7 @@ from agents.extensions.memory import (
DAPR_CONSISTENCY_STRONG,
DaprSession,
)
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
# Docker-backed integration tests should stay on the exclusive serial test path.
@@ -234,8 +234,8 @@ spec:
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
"""Fixture for a basic agent with a scripted model."""
return Agent(name="test", model=ScriptedModel())
async def test_dapr_redis_integration(dapr_container, monkeypatch):
@@ -321,8 +321,8 @@ async def test_dapr_runner_integration(agent: Agent, dapr_container, monkeypatch
await session.clear_session()
# First turn
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
@@ -331,12 +331,12 @@ async def test_dapr_runner_integration(agent: Agent, dapr_container, monkeypatch
assert result1.final_output == "San Francisco"
# Second turn - should remember context
agent.model.set_next_output([get_text_message("California")])
agent.model.enqueue([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
# Verify history
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
+16 -16
View File
@@ -14,7 +14,7 @@ from agents.extensions.memory import (
DAPR_CONSISTENCY_STRONG,
DaprSession,
)
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
# Mark all tests in this file as asyncio
@@ -166,8 +166,8 @@ def conflict_dapr_client() -> ConflictFakeDaprClient:
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
"""Fixture for a basic agent with a scripted model."""
return Agent(name="test", model=ScriptedModel())
async def _create_test_session(
@@ -233,8 +233,8 @@ async def test_runner_integration(agent: Agent, fake_dapr_client: FakeDaprClient
try:
# First turn
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
@@ -243,12 +243,12 @@ async def test_runner_integration(agent: Agent, fake_dapr_client: FakeDaprClient
assert result1.final_output == "San Francisco"
# Second turn
agent.model.set_next_output([get_text_message("California")])
agent.model.enqueue([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
# Verify history was passed to the model on the second turn
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
@@ -270,23 +270,23 @@ async def test_session_isolation(fake_dapr_client: FakeDaprClient):
)
try:
agent = Agent(name="test", model=FakeModel())
agent = Agent(name="test", model=ScriptedModel())
# Clean up any existing data
await session1.clear_session()
await session2.clear_session()
# Interact with session 1
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("I like cats.")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("I like cats.")])
await Runner.run(agent, "I like cats.", session=session1)
# Interact with session 2
agent.model.set_next_output([get_text_message("I like dogs.")])
agent.model.enqueue([get_text_message("I like dogs.")])
await Runner.run(agent, "I like dogs.", session=session2)
# Go back to session 1 and check its memory
agent.model.set_next_output([get_text_message("You said you like cats.")])
agent.model.enqueue([get_text_message("You said you like cats.")])
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
assert "cats" in result.final_output.lower()
assert "dogs" not in result.final_output.lower()
@@ -1033,7 +1033,7 @@ async def test_runner_with_session_settings_override(fake_dapr_client: FakeDaprC
"""Test that RunConfig can override session's default settings."""
from agents import Agent, RunConfig, Runner
from agents.memory import SessionSettings
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
session = DaprSession(
@@ -1052,9 +1052,9 @@ async def test_runner_with_session_settings_override(fake_dapr_client: FakeDaprC
]
await session.add_items(items)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("Got it")])
model.enqueue([get_text_message("Got it")])
await Runner.run(
agent,
@@ -1066,7 +1066,7 @@ async def test_runner_with_session_settings_override(fake_dapr_client: FakeDaprC
)
# Verify the agent received only the last 2 history items + new question
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
+11 -11
View File
@@ -19,7 +19,7 @@ from agents import (
TResponseInputItem,
)
from agents.extensions.memory.encrypt_session import EncryptedSession
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
# Mark all tests in this file as asyncio
@@ -35,8 +35,8 @@ def _invalid_encrypted_envelope() -> TResponseInputItem:
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
"""Fixture for a basic agent with a scripted model."""
return Agent(name="test", model=ScriptedModel())
@pytest.fixture
@@ -106,8 +106,8 @@ async def test_encrypted_session_with_runner(
encryption_key=encryption_key,
)
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
@@ -115,11 +115,11 @@ async def test_encrypted_session_with_runner(
)
assert result1.final_output == "San Francisco"
agent.model.set_next_output([get_text_message("California")])
agent.model.enqueue([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
@@ -603,7 +603,7 @@ async def test_runner_with_session_settings_override(encryption_key: str):
"""Test that RunConfig can override session's default settings."""
from agents import Agent, RunConfig, Runner
from agents.memory import SessionSettings
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
temp_dir = tempfile.mkdtemp()
@@ -620,9 +620,9 @@ async def test_runner_with_session_settings_override(encryption_key: str):
items: list[TResponseInputItem] = [{"role": "user", "content": f"Turn {i}"} for i in range(10)]
await session.add_items(items)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("Got it")])
model.enqueue([get_text_message("Got it")])
await Runner.run(
agent,
@@ -634,7 +634,7 @@ async def test_runner_with_session_settings_override(encryption_key: str):
)
# Verify the agent received only the last 2 history items + new question
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
+13 -13
View File
@@ -22,7 +22,7 @@ import pytest
from agents import Agent, Runner, TResponseInputItem
from agents.memory.session_settings import SessionSettings
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
pytestmark = pytest.mark.asyncio
@@ -338,7 +338,7 @@ def session() -> MongoDBSession:
@pytest.fixture
def agent() -> Agent:
return Agent(name="test", model=FakeModel())
return Agent(name="test", model=ScriptedModel())
# ---------------------------------------------------------------------------
@@ -1048,16 +1048,16 @@ async def test_runner_integration(agent: Agent) -> None:
"""MongoDBSession must supply conversation history to the Runner."""
session = _make_session("runner-test")
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("San Francisco")])
result1 = await Runner.run(agent, "Where is the Golden Gate Bridge?", session=session)
assert result1.final_output == "San Francisco"
agent.model.set_next_output([get_text_message("California")])
agent.model.enqueue([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
@@ -1069,14 +1069,14 @@ async def test_runner_session_isolation(agent: Agent) -> None:
s1 = MongoDBSession("user-a", client=client, database="agents_test") # type: ignore[arg-type]
s2 = MongoDBSession("user-b", client=client, database="agents_test") # type: ignore[arg-type]
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("I like cats.")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("I like cats.")])
await Runner.run(agent, "I like cats.", session=s1)
agent.model.set_next_output([get_text_message("I like dogs.")])
agent.model.enqueue([get_text_message("I like dogs.")])
await Runner.run(agent, "I like dogs.", session=s2)
agent.model.set_next_output([get_text_message("You said you like cats.")])
agent.model.enqueue([get_text_message("You said you like cats.")])
result = await Runner.run(agent, "What animal did I mention?", session=s1)
assert "cats" in result.final_output.lower()
assert "dogs" not in result.final_output.lower()
@@ -1099,8 +1099,8 @@ async def test_runner_with_session_settings_limit(agent: Agent) -> None:
]
await session.add_items(history)
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("Got it")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("Got it")])
await Runner.run(
agent,
"New question",
@@ -1108,7 +1108,7 @@ async def test_runner_with_session_settings_limit(agent: Agent) -> None:
run_config=RunConfig(session_settings=SessionSettings(limit=2)),
)
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
history_items = [i for i in last_input if i.get("content") != "New question"]
assert len(history_items) == 2
+16 -16
View File
@@ -13,7 +13,7 @@ pytest.importorskip("redis") # Skip tests if Redis is not installed
from agents import Agent, Runner, TResponseInputItem
from agents.extensions.memory.redis_session import RedisSession
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
# Keep the fallback-to-real-Redis path isolated from xdist workers.
@@ -61,8 +61,8 @@ async def _safe_rpush(client: Redis, key: str, value: str) -> None:
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
"""Fixture for a basic agent with a scripted model."""
return Agent(name="test", model=ScriptedModel())
async def _create_redis_session(
@@ -205,8 +205,8 @@ async def test_runner_integration(agent: Agent):
try:
# First turn
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
@@ -215,12 +215,12 @@ async def test_runner_integration(agent: Agent):
assert result1.final_output == "San Francisco"
# Second turn
agent.model.set_next_output([get_text_message("California")])
agent.model.enqueue([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
# Verify history was passed to the model on the second turn
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
@@ -234,23 +234,23 @@ async def test_session_isolation():
session2 = await _create_redis_session("session_2")
try:
agent = Agent(name="test", model=FakeModel())
agent = Agent(name="test", model=ScriptedModel())
# Clean up any existing data
await session1.clear_session()
await session2.clear_session()
# Interact with session 1
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("I like cats.")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("I like cats.")])
await Runner.run(agent, "I like cats.", session=session1)
# Interact with session 2
agent.model.set_next_output([get_text_message("I like dogs.")])
agent.model.enqueue([get_text_message("I like dogs.")])
await Runner.run(agent, "I like dogs.", session=session2)
# Go back to session 1 and check its memory
agent.model.set_next_output([get_text_message("You said you like cats.")])
agent.model.enqueue([get_text_message("You said you like cats.")])
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
assert "cats" in result.final_output.lower()
assert "dogs" not in result.final_output.lower()
@@ -1094,7 +1094,7 @@ async def test_runner_with_session_settings_override():
"""Test that RunConfig can override session's default settings."""
from agents import Agent, RunConfig, Runner
from agents.memory import SessionSettings
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
if USE_FAKE_REDIS:
@@ -1118,9 +1118,9 @@ async def test_runner_with_session_settings_override():
]
await session.add_items(items)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("Got it")])
model.enqueue([get_text_message("Got it")])
await Runner.run(
agent,
@@ -1132,7 +1132,7 @@ async def test_runner_with_session_settings_override():
)
# Verify the agent received only the last 2 history items + new question
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
@@ -25,7 +25,7 @@ pytest.importorskip("sqlalchemy") # Skip tests if SQLAlchemy is not installed
from agents import Agent, Runner, TResponseInputItem
from agents.extensions.memory.sqlalchemy_session import SQLAlchemySession
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
# Mark all tests in this file as asyncio
@@ -72,8 +72,8 @@ def _item_ids(items: Sequence[TResponseInputItem]) -> list[str]:
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
"""Fixture for a basic agent with a scripted model."""
return Agent(name="test", model=ScriptedModel())
async def test_sqlalchemy_session_direct_ops(agent: Agent):
@@ -159,8 +159,8 @@ async def test_runner_integration(agent: Agent):
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
# First turn
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
@@ -169,12 +169,12 @@ async def test_runner_integration(agent: Agent):
assert result1.final_output == "San Francisco"
# Second turn
agent.model.set_next_output([get_text_message("California")])
agent.model.enqueue([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
# Verify history was passed to the model on the second turn
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
@@ -188,16 +188,16 @@ async def test_session_isolation(agent: Agent):
session2 = SQLAlchemySession.from_url(session_id_2, url=DB_URL, create_tables=True)
# Interact with session 1
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("I like cats.")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("I like cats.")])
await Runner.run(agent, "I like cats.", session=session1)
# Interact with session 2
agent.model.set_next_output([get_text_message("I like dogs.")])
agent.model.enqueue([get_text_message("I like dogs.")])
await Runner.run(agent, "I like dogs.", session=session2)
# Go back to session 1 and check its memory
agent.model.set_next_output([get_text_message("You said you like cats.")])
agent.model.enqueue([get_text_message("You said you like cats.")])
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
assert "cats" in result.final_output.lower()
assert "dogs" not in result.final_output.lower()
@@ -1259,8 +1259,8 @@ async def test_runner_with_session_settings_override(agent: Agent):
await session.add_items(items)
# Use RunConfig to override limit to 2
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("Got it")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("Got it")])
await Runner.run(
agent,
@@ -1272,7 +1272,7 @@ async def test_runner_with_session_settings_override(agent: Agent):
)
# Verify the agent received only the last 2 history items + new question
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
+18 -18
View File
@@ -1,34 +1,28 @@
from __future__ import annotations
import shlex
from types import SimpleNamespace
from typing import Any
from agents.extensions.sandbox.blaxel.mounts import (
BlaxelCloudBucketMountConfig,
_mount_gcs,
_mount_s3,
)
from agents.sandbox import ExecResult
from agents.testing import scripted_sandbox_session
_INJECTION = "x; touch /tmp/pwned"
class _RecordingSession:
"""Minimal sandbox session that records the `sh -c` commands it is asked to run."""
def __init__(self) -> None:
self.commands: list[str] = []
async def exec(self, *args: Any, **kwargs: Any) -> Any:
if len(args) >= 3 and args[0] == "sh" and args[1] == "-c":
self.commands.append(args[2])
return SimpleNamespace(exit_code=0, stdout=b"", stderr=b"")
def _successful_exec(_call: object) -> ExecResult:
return ExecResult(exit_code=0, stdout=b"", stderr=b"")
async def test_s3_mount_options_are_shell_quoted() -> None:
session = _RecordingSession()
session = scripted_sandbox_session(
[{"method": "exec", "responder": _successful_exec} for _ in range(3)]
)
await _mount_s3(
session, # type: ignore[arg-type]
session,
BlaxelCloudBucketMountConfig(
provider="s3",
bucket="bucket",
@@ -36,15 +30,19 @@ async def test_s3_mount_options_are_shell_quoted() -> None:
endpoint_url=f"http://{_INJECTION}",
),
)
cmd = next(c for c in session.commands if c.startswith("s3fs"))
commands = [call.args[2] for call in session.calls if call.args[:2] == ("sh", "-c")]
cmd = next(command for command in commands if command.startswith("s3fs"))
# The injected `; touch` must stay inside the -o option token, not become its own command.
assert "touch" not in shlex.split(cmd)
session.assert_complete()
async def test_gcs_mount_prefix_is_shell_quoted() -> None:
session = _RecordingSession()
session = scripted_sandbox_session(
[{"method": "exec", "responder": _successful_exec} for _ in range(3)]
)
await _mount_gcs(
session, # type: ignore[arg-type]
session,
BlaxelCloudBucketMountConfig(
provider="gcs",
bucket="bucket",
@@ -52,5 +50,7 @@ async def test_gcs_mount_prefix_is_shell_quoted() -> None:
prefix=_INJECTION,
),
)
cmd = next(c for c in session.commands if c.startswith("gcsfuse"))
commands = [call.args[2] for call in session.calls if call.args[:2] == ("sh", "-c")]
cmd = next(command for command in commands if command.startswith("gcsfuse"))
assert "touch" not in shlex.split(cmd)
session.assert_complete()
+19 -25
View File
@@ -12,22 +12,13 @@ from agents.extensions.sandbox._rclone import (
)
from agents.sandbox.errors import MountConfigError
from agents.sandbox.types import ExecResult
from agents.testing import scripted_sandbox_session
def _result(*, exit_code: int = 0, stdout: bytes = b"") -> ExecResult:
return ExecResult(stdout=stdout, stderr=b"", exit_code=exit_code)
class _FakeSession:
def __init__(self, results: list[ExecResult]) -> None:
self.results = results
self.calls: list[tuple[tuple[str, ...], dict[str, object]]] = []
async def exec(self, *command: str, **kwargs: object) -> ExecResult:
self.calls.append((command, kwargs))
return self.results.pop(0)
@pytest.mark.parametrize(
("machine", "expected"),
[
@@ -73,25 +64,25 @@ def test_rclone_install_command_pins_and_verifies_archive() -> None:
@pytest.mark.asyncio
async def test_ensure_rclone_preserves_preinstalled_binary() -> None:
session = _FakeSession([_result()])
session = scripted_sandbox_session([{"method": "exec", "result": _result()}])
await ensure_rclone(session) # type: ignore[arg-type]
await ensure_rclone(session)
assert len(session.calls) == 1
@pytest.mark.asyncio
async def test_ensure_rclone_rejects_unsupported_architecture_before_install() -> None:
session = _FakeSession(
session = scripted_sandbox_session(
[
_result(exit_code=1),
_result(),
_result(stdout=b"mips64\n"),
{"method": "exec", "result": _result(exit_code=1)},
{"method": "exec", "result": _result()},
{"method": "exec", "result": _result(stdout=b"mips64\n")},
]
)
with pytest.raises(MountConfigError, match="architecture is unsupported") as exc_info:
await ensure_rclone(session) # type: ignore[arg-type]
await ensure_rclone(session)
assert exc_info.value.context["architecture"] == "mips64"
assert len(session.calls) == 3
@@ -99,19 +90,22 @@ async def test_ensure_rclone_rejects_unsupported_architecture_before_install() -
@pytest.mark.asyncio
async def test_ensure_rclone_reports_checksum_mismatch() -> None:
session = _FakeSession(
session = scripted_sandbox_session(
[
_result(exit_code=1),
_result(),
_result(stdout=b"x86_64\n"),
_result(),
_result(),
_result(exit_code=_RCLONE_CHECKSUM_MISMATCH_EXIT),
{"method": "exec", "result": _result(exit_code=1)},
{"method": "exec", "result": _result()},
{"method": "exec", "result": _result(stdout=b"x86_64\n")},
{"method": "exec", "result": _result()},
{"method": "exec", "result": _result()},
{
"method": "exec",
"result": _result(exit_code=_RCLONE_CHECKSUM_MISMATCH_EXIT),
},
]
)
with pytest.raises(MountConfigError, match="checksum verification failed") as exc_info:
await ensure_rclone(session) # type: ignore[arg-type]
await ensure_rclone(session)
assert exc_info.value.context == {
"package": "rclone",
+6 -6
View File
@@ -48,8 +48,8 @@ from agents.sandbox.session.manager import Instrumentation
from agents.sandbox.session.sinks import CallbackSink
from agents.sandbox.snapshot import NoopSnapshot, SnapshotBase
from agents.sandbox.types import User
from agents.testing import ScriptedModel
from tests._fake_workspace_paths import resolve_fake_workspace_path
from tests.fake_model import FakeModel
class _FakeNetworkPolicyRule(BaseModel):
@@ -1045,7 +1045,7 @@ async def test_vercel_injected_session_accepts_unchanged_s3_manifest(
manifest=_vercel_s3_manifest(package_module),
options=vercel_module.VercelSandboxClientOptions(),
)
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=session),
@@ -1074,7 +1074,7 @@ async def test_vercel_injected_session_revalidates_preexisting_s3_topology_mutat
)
mount = cast(S3Mount, session.state.manifest.entries["remote"])
mount.bucket = "tampered-bucket"
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=session),
@@ -1105,7 +1105,7 @@ async def test_vercel_injected_session_applies_non_mount_delta_with_fixed_s3_top
options=vercel_module.VercelSandboxClientOptions(),
)
sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox)
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=session),
@@ -1147,7 +1147,7 @@ async def test_vercel_live_manifest_update_uses_one_running_snapshot(
return True
monkeypatch.setattr(session, "running", running_once)
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
update = await SandboxRuntimeSessionManager._process_live_session_manifest(
agent=agent,
@@ -1176,7 +1176,7 @@ async def test_vercel_stopped_injected_session_rejects_non_mount_delta_before_st
)
sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox)
sandbox.status = "stopped"
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
manager = SandboxRuntimeSessionManager(
starting_agent=agent,
sandbox_config=SandboxRunConfig(session=session),
-404
View File
@@ -1,404 +0,0 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any
from openai.types.responses import (
Response,
ResponseApplyPatchToolCall,
ResponseCompletedEvent,
ResponseContentPartAddedEvent,
ResponseContentPartDoneEvent,
ResponseCreatedEvent,
ResponseFunctionCallArgumentsDeltaEvent,
ResponseFunctionCallArgumentsDoneEvent,
ResponseFunctionToolCall,
ResponseInProgressEvent,
ResponseOutputItemAddedEvent,
ResponseOutputItemDoneEvent,
ResponseOutputMessage,
ResponseOutputText,
ResponseReasoningSummaryPartAddedEvent,
ResponseReasoningSummaryPartDoneEvent,
ResponseReasoningSummaryTextDeltaEvent,
ResponseReasoningSummaryTextDoneEvent,
ResponseTextDeltaEvent,
ResponseTextDoneEvent,
ResponseUsage,
)
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
from openai.types.responses.response_reasoning_summary_part_added_event import (
Part as AddedEventPart,
)
from openai.types.responses.response_reasoning_summary_part_done_event import Part as DoneEventPart
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
from agents.agent_output import AgentOutputSchemaBase
from agents.handoffs import Handoff
from agents.items import (
ModelResponse,
TResponseInputItem,
TResponseOutputItem,
TResponseStreamEvent,
)
from agents.model_settings import ModelSettings
from agents.models.interface import Model, ModelTracing
from agents.tool import Tool
from agents.tracing import SpanError, generation_span
from agents.usage import Usage
class FakeModel(Model):
def __init__(
self,
tracing_enabled: bool = False,
initial_output: list[TResponseOutputItem] | Exception | None = None,
):
if initial_output is None:
initial_output = []
self.turn_outputs: list[list[TResponseOutputItem] | Exception] = (
[initial_output] if initial_output else []
)
self.tracing_enabled = tracing_enabled
self.last_turn_args: dict[str, Any] = {}
self.first_turn_args: dict[str, Any] | None = None
self.hardcoded_usage: Usage | None = None
def set_hardcoded_usage(self, usage: Usage):
self.hardcoded_usage = usage
def set_next_output(self, output: list[TResponseOutputItem] | Exception):
self.turn_outputs.append(output)
def add_multiple_turn_outputs(self, outputs: list[list[TResponseOutputItem] | Exception]):
self.turn_outputs.extend(outputs)
def get_next_output(self) -> list[TResponseOutputItem] | Exception:
if not self.turn_outputs:
return []
return self.turn_outputs.pop(0)
def _record_turn_args(
self,
*,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
previous_response_id: str | None,
conversation_id: str | None,
prompt: Any | None,
) -> None:
turn_args = {
"system_instructions": system_instructions,
"input": input,
"model_settings": model_settings,
"tools": tools,
"output_schema": output_schema,
"handoffs": handoffs,
"tracing": tracing,
"previous_response_id": previous_response_id,
"conversation_id": conversation_id,
"prompt": prompt,
}
if self.first_turn_args is None:
self.first_turn_args = turn_args.copy()
self.last_turn_args = turn_args
async def get_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: Any | None,
) -> ModelResponse:
self._record_turn_args(
system_instructions=system_instructions,
input=input,
model_settings=model_settings,
tools=tools,
output_schema=output_schema,
handoffs=handoffs,
tracing=tracing,
previous_response_id=previous_response_id,
conversation_id=conversation_id,
prompt=prompt,
)
with generation_span(disabled=not self.tracing_enabled) as span:
output = self.get_next_output()
if isinstance(output, Exception):
span.set_error(
SpanError(
message="Error",
data={
"name": output.__class__.__name__,
"message": str(output),
},
)
)
raise output
converted_output = []
for item in output:
if isinstance(item, dict) and item.get("type") == "apply_patch_call":
call_id = str(item.get("call_id") or item.get("id") or "")
converted_output.append(
ResponseApplyPatchToolCall(
type="apply_patch_call",
id=str(item.get("id") or call_id),
call_id=call_id,
status=item.get("status") or "completed",
operation=item.get("operation"),
)
)
else:
converted_output.append(item)
return ModelResponse(
output=converted_output,
usage=self.hardcoded_usage or Usage(),
response_id="resp-789",
)
async def stream_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None = None,
conversation_id: str | None = None,
prompt: Any | None = None,
) -> AsyncIterator[TResponseStreamEvent]:
self._record_turn_args(
system_instructions=system_instructions,
input=input,
model_settings=model_settings,
tools=tools,
output_schema=output_schema,
handoffs=handoffs,
tracing=tracing,
previous_response_id=previous_response_id,
conversation_id=conversation_id,
prompt=prompt,
)
with generation_span(disabled=not self.tracing_enabled) as span:
output = self.get_next_output()
if isinstance(output, Exception):
span.set_error(
SpanError(
message="Error",
data={
"name": output.__class__.__name__,
"message": str(output),
},
)
)
raise output
response = get_response_obj(output, usage=self.hardcoded_usage)
sequence_number = 0
yield ResponseCreatedEvent(
type="response.created",
response=response,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseInProgressEvent(
type="response.in_progress",
response=response,
sequence_number=sequence_number,
)
sequence_number += 1
for output_index, output_item in enumerate(output):
yield ResponseOutputItemAddedEvent(
type="response.output_item.added",
item=output_item,
output_index=output_index,
sequence_number=sequence_number,
)
sequence_number += 1
if isinstance(output_item, ResponseReasoningItem):
if output_item.summary:
for summary_index, summary in enumerate(output_item.summary):
yield ResponseReasoningSummaryPartAddedEvent(
type="response.reasoning_summary_part.added",
item_id=output_item.id,
output_index=output_index,
summary_index=summary_index,
part=AddedEventPart(text=summary.text, type=summary.type),
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseReasoningSummaryTextDeltaEvent(
type="response.reasoning_summary_text.delta",
item_id=output_item.id,
output_index=output_index,
summary_index=summary_index,
delta=summary.text,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseReasoningSummaryTextDoneEvent(
type="response.reasoning_summary_text.done",
item_id=output_item.id,
output_index=output_index,
summary_index=summary_index,
text=summary.text,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseReasoningSummaryPartDoneEvent(
type="response.reasoning_summary_part.done",
item_id=output_item.id,
output_index=output_index,
summary_index=summary_index,
part=DoneEventPart(text=summary.text, type=summary.type),
sequence_number=sequence_number,
)
sequence_number += 1
elif isinstance(output_item, ResponseFunctionToolCall):
yield ResponseFunctionCallArgumentsDeltaEvent(
type="response.function_call_arguments.delta",
item_id=output_item.call_id,
output_index=output_index,
delta=output_item.arguments,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseFunctionCallArgumentsDoneEvent(
type="response.function_call_arguments.done",
item_id=output_item.call_id,
output_index=output_index,
arguments=output_item.arguments,
name=output_item.name,
sequence_number=sequence_number,
)
sequence_number += 1
elif isinstance(output_item, ResponseOutputMessage):
for content_index, content_part in enumerate(output_item.content or []):
if isinstance(content_part, ResponseOutputText):
yield ResponseContentPartAddedEvent(
type="response.content_part.added",
item_id=output_item.id,
output_index=output_index,
content_index=content_index,
part=content_part,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseTextDeltaEvent(
type="response.output_text.delta",
item_id=output_item.id,
output_index=output_index,
content_index=content_index,
delta=content_part.text,
logprobs=[],
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseTextDoneEvent(
type="response.output_text.done",
item_id=output_item.id,
output_index=output_index,
content_index=content_index,
text=content_part.text,
logprobs=[],
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseContentPartDoneEvent(
type="response.content_part.done",
item_id=output_item.id,
output_index=output_index,
content_index=content_index,
part=content_part,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseOutputItemDoneEvent(
type="response.output_item.done",
item=output_item,
output_index=output_index,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseCompletedEvent(
type="response.completed",
response=response,
sequence_number=sequence_number,
)
class PromptCacheFakeModel(FakeModel):
def _supports_default_prompt_cache_key(self) -> bool:
return True
def get_response_obj(
output: list[TResponseOutputItem],
response_id: str | None = None,
usage: Usage | None = None,
) -> Response:
return Response(
id=response_id or "resp-789",
created_at=123,
model="test_model",
object="response",
output=output,
tool_choice="none",
tools=[],
top_p=None,
parallel_tool_calls=False,
usage=ResponseUsage(
input_tokens=usage.input_tokens if usage else 0,
output_tokens=usage.output_tokens if usage else 0,
total_tokens=usage.total_tokens if usage else 0,
input_tokens_details=InputTokensDetails.model_validate(
{
"cache_write_tokens": (
getattr(usage.input_tokens_details, "cache_write_tokens", 0) if usage else 0
),
"cached_tokens": (
getattr(usage.input_tokens_details, "cached_tokens", 0) if usage else 0
),
}
),
output_tokens_details=OutputTokensDetails(
reasoning_tokens=(
getattr(usage.output_tokens_details, "reasoning_tokens", 0) if usage else 0
)
),
),
)
+4 -3
View File
@@ -2,7 +2,8 @@ import pytest
from httpx import ASGITransport, AsyncClient
from inline_snapshot import snapshot
from ..fake_model import FakeModel
from agents.testing import ScriptedModel
from ..test_responses import get_text_message
from .streaming_app import agent, app
@@ -14,9 +15,9 @@ async def test_streaming_context():
leading to a tracing error because the context was closed in the wrong context. This test
ensures that this actually works.
"""
model = FakeModel()
model = ScriptedModel()
agent.model = model
model.set_next_output([get_text_message("done")])
model.enqueue([get_text_message("done")])
transport = ASGITransport(app)
async with AsyncClient(transport=transport, base_url="http://test") as ac:
+3 -3
View File
@@ -446,7 +446,7 @@ PENDING_TOOL_APPROVAL = Scenario(
import asyncio
from agents import Runner, function_tool
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_function_tool_call
@function_tool(needs_approval=True)
@@ -454,8 +454,8 @@ def historical_approval(account_id: str) -> str:
return f"approved:{account_id}"
async def produce_pending_state():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[[get_function_tool_call(
"historical_approval",
'{"account_id":"account-1"}',
+7 -7
View File
@@ -5,8 +5,8 @@ from mcp.types import Tool as MCPTool
from agents import Agent, RunContextWrapper, Runner
from agents.exceptions import UserError
from agents.testing import ScriptedModel
from ..fake_model import FakeModel
from ..test_responses import get_function_tool_call, get_text_message
from ..utils.hitl import queue_function_call_and_text, resume_after_first_approval
from .helpers import FakeMCPServer
@@ -19,7 +19,7 @@ async def test_mcp_require_approval_pauses_and_resumes():
server = FakeMCPServer(require_approval="always")
server.add_tool("add", {"type": "object", "properties": {}})
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
@@ -51,7 +51,7 @@ async def test_mcp_require_approval_tool_lists():
server = FakeMCPServer(require_approval=require_approval)
server.add_tool("add", {"type": "object", "properties": {}})
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
@@ -76,7 +76,7 @@ async def test_mcp_require_approval_tool_mapping():
server = FakeMCPServer(require_approval=require_approval)
server.add_tool("add", {"type": "object", "properties": {}})
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
@@ -102,7 +102,7 @@ async def test_mcp_require_approval_mapping_allows_policy_keyword_tool_names():
server.add_tool("always", {"type": "object", "properties": {}})
server.add_tool("never", {"type": "object", "properties": {}})
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
@@ -167,7 +167,7 @@ async def test_mcp_require_approval_callable_can_allow_and_block_by_tool_name():
server.add_tool("guarded", {"type": "object", "properties": {}})
server.add_tool("safe", {"type": "object", "properties": {}})
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
@@ -212,7 +212,7 @@ async def test_mcp_require_approval_async_callable_uses_run_context():
server = FakeMCPServer(require_approval=require_approval)
server.add_tool("conditional", {"type": "object", "properties": {}})
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
+3 -3
View File
@@ -8,8 +8,8 @@ import pytest
from agents import Agent, Runner
from agents.mcp import MCPServerStdio
from agents.mcp._compat import MCP_V2, result_next_cursor
from agents.testing import ScriptedModel
from ..fake_model import FakeModel
from ..test_responses import get_function_tool_call, get_text_message
PAGINATED_SERVER_PATH = Path(__file__).parent / "servers" / "paginated.py"
@@ -50,8 +50,8 @@ async def test_stdio_server_auto_paginates_tools_and_prompts():
@pytest.mark.asyncio
async def test_agent_calls_tool_from_second_stdio_page():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("second_page_tool", "{}")],
[get_text_message("done")],
+9 -9
View File
@@ -4,8 +4,8 @@ import pytest
from inline_snapshot import snapshot
from agents import Agent, RunConfig, Runner
from agents.testing import ScriptedModel
from ..fake_model import FakeModel
from ..test_responses import get_function_tool, get_function_tool_call, get_text_message
from ..testing_processor import SPAN_PROCESSOR_TESTING, fetch_normalized_spans
from .helpers import FakeMCPServer
@@ -13,7 +13,7 @@ from .helpers import FakeMCPServer
@pytest.mark.asyncio
async def test_mcp_tracing():
model = FakeModel()
model = ScriptedModel()
server = FakeMCPServer()
server.add_tool("test_tool_1", {})
agent = Agent(
@@ -23,7 +23,7 @@ async def test_mcp_tracing():
tools=[get_function_tool("non_mcp_tool", "tool_result")],
)
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[
@@ -86,7 +86,7 @@ async def test_mcp_tracing():
SPAN_PROCESSOR_TESTING.clear()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[
@@ -161,7 +161,7 @@ async def test_mcp_tracing():
# Add more tools to the server
server.add_tool("test_tool_3", {})
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
@@ -223,12 +223,12 @@ async def test_mcp_tracing():
@pytest.mark.asyncio
async def test_mcp_tracing_redacts_output_when_sensitive_data_disabled():
model = FakeModel()
model = ScriptedModel()
server = FakeMCPServer()
server.add_tool("test_tool_1", {})
agent = Agent(name="test", model=model, mcp_servers=[server])
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("test_tool_1", "")],
[get_text_message("done")],
@@ -284,7 +284,7 @@ async def test_mcp_tracing_redacts_output_when_sensitive_data_disabled():
async def test_mcp_tracing_always_hides_url_credentials(
trace_include_sensitive_data: bool,
):
model = FakeModel()
model = ScriptedModel()
server = FakeMCPServer(
server_name=(
"streamable_http: https://user:s3cr3t_pw@mcp.example.test:8443/mcp"
@@ -293,7 +293,7 @@ async def test_mcp_tracing_always_hides_url_credentials(
)
server.add_tool("search", {})
agent = Agent(name="test", model=model, mcp_servers=[server])
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("search", "")],
[get_text_message("done")],
+5 -7
View File
@@ -5,8 +5,8 @@ from mcp.types import ListResourcesResult, ReadResourceResult
from agents import Agent, Runner
from agents.mcp import MCPServer, MCPToolMetaResolver
from agents.testing import ScriptedModel
from ..fake_model import FakeModel
from ..test_responses import get_text_message
from .model_compat import ListResourceTemplatesResult
@@ -173,13 +173,11 @@ async def test_agent_with_prompt_instructions():
instructions = prompt_result.messages[0].content.text
# Create agent with prompt-generated instructions
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="prompt_agent", instructions=instructions, model=model, mcp_servers=[server])
# Mock model response
model.add_multiple_turn_outputs(
[[get_text_message("Code analysis complete. Found security vulnerability.")]]
)
model.extend([[get_text_message("Code analysis complete. Found security vulnerability.")]])
# Run the agent
result = await Runner.run(agent, input="Review this code: def unsafe_exec(cmd): os.system(cmd)")
@@ -211,12 +209,12 @@ async def test_agent_with_prompt_instructions_streaming(streaming: bool):
instructions = prompt_result.messages[0].content.text
# Create agent
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="streaming_prompt_agent", instructions=instructions, model=model, mcp_servers=[server]
)
model.add_multiple_turn_outputs([[get_text_message("Security analysis complete.")]])
model.extend([[get_text_message("Security analysis complete.")]])
if streaming:
streaming_result = Runner.run_streamed(agent, input="Review code")
+22 -22
View File
@@ -15,8 +15,8 @@ from agents import (
handoff,
)
from agents.exceptions import AgentsException
from agents.testing import ScriptedModel
from ..fake_model import FakeModel
from ..test_responses import get_function_tool_call, get_text_message
from .helpers import FakeMCPServer
@@ -29,14 +29,14 @@ async def test_runner_calls_mcp_tool(streaming: bool):
server.add_tool("test_tool_1", {})
server.add_tool("test_tool_2", {})
server.add_tool("test_tool_3", {})
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server],
)
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
@@ -63,14 +63,14 @@ async def test_runner_asserts_when_mcp_tool_not_found(streaming: bool):
server.add_tool("test_tool_1", {})
server.add_tool("test_tool_2", {})
server.add_tool("test_tool_3", {})
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server],
)
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_doesnt_exist", "")],
@@ -99,14 +99,14 @@ async def test_runner_works_with_multiple_mcp_servers(streaming: bool):
server2.add_tool("test_tool_2", {})
server2.add_tool("test_tool_3", {})
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server1, server2],
)
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
@@ -138,14 +138,14 @@ async def test_runner_errors_when_mcp_tools_clash(streaming: bool):
server2.add_tool("test_tool_2", {})
server2.add_tool("test_tool_3", {})
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server1, server2],
)
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
@@ -172,7 +172,7 @@ async def test_runner_can_call_server_prefixed_mcp_tool_names(streaming: bool):
server2 = FakeMCPServer(server_name="calendar")
server2.add_tool("search", {})
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
@@ -180,7 +180,7 @@ async def test_runner_can_call_server_prefixed_mcp_tool_names(streaming: bool):
mcp_config={"include_server_in_tool_names": True},
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("a_message"), get_function_tool_call("mcp_calendar__search", "")],
[get_text_message("done")],
@@ -220,7 +220,7 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_agent_tools(st
on_invoke_tool=invoke_local_tool,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
@@ -238,7 +238,7 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_agent_tools(st
assert calendar_search_tool_name != "mcp_calendar__search"
assert calendar_search_tool_name.startswith("mcp_calendar__search_")
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
[get_text_message("done")],
@@ -263,11 +263,11 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(strea
server = FakeMCPServer(server_name="calendar")
server.add_tool("search", {})
target_model = FakeModel()
target_model = ScriptedModel()
target_agent = Agent(name="calendar_agent", model=target_model)
target_model.add_multiple_turn_outputs([[get_text_message("handoff target")]])
target_model.extend([[get_text_message("handoff target")]])
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
@@ -282,7 +282,7 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(strea
assert calendar_search_tool_name != "mcp_calendar__search"
assert calendar_search_tool_name.startswith("mcp_calendar__search_")
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
[get_text_message("done")],
@@ -297,7 +297,7 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(strea
await Runner.run(agent, input="user_message")
assert server.tool_calls == ["search"]
assert target_model.first_turn_args is None
assert not target_model.calls
class Foo(BaseModel):
@@ -314,7 +314,7 @@ async def test_runner_calls_mcp_tool_with_args(streaming: bool):
server.add_tool("test_tool_1", {})
server.add_tool("test_tool_2", Foo.model_json_schema())
server.add_tool("test_tool_3", {})
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
@@ -323,7 +323,7 @@ async def test_runner_calls_mcp_tool_with_args(streaming: bool):
json_args = json.dumps(Foo(bar="baz", baz=1).model_dump())
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_2", json_args)],
@@ -362,14 +362,14 @@ async def test_runner_emits_mcp_error_tool_call_output_item(streaming: bool):
server = CrashingFakeMCPServer()
server.add_tool("crashing_tool", {})
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server],
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("a_message"), get_function_tool_call("crashing_tool", "{}")],
[get_text_message("done")],
@@ -20,7 +20,7 @@ from agents.memory.openai_conversations_session import (
OpenAIConversationsSession,
start_openai_conversations_session,
)
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
@@ -46,8 +46,8 @@ def mock_openai_client():
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
"""Fixture for a basic agent with a scripted model."""
return Agent(name="test", model=ScriptedModel())
class TestStartOpenAIConversationsSession:
@@ -451,8 +451,8 @@ class TestOpenAIConversationsSessionRunnerIntegration:
with patch.object(session, "get_items", return_value=[]):
with patch.object(session, "add_items") as mock_add_items:
# Run the agent
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("San Francisco")])
result = await Runner.run(
agent, "What city is the Golden Gate Bridge in?", session=session
@@ -477,15 +477,15 @@ class TestOpenAIConversationsSessionRunnerIntegration:
with patch.object(session, "get_items", return_value=conversation_history):
with patch.object(session, "add_items"):
# Second turn - should have access to previous conversation
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("California")])
assert isinstance(agent.model, ScriptedModel)
agent.model.enqueue([get_text_message("California")])
result = await Runner.run(agent, "What state is it in?", session=session)
assert result.final_output == "California"
# Verify that the model received the conversation history
last_input = agent.model.last_turn_args["input"]
last_input = agent.model.calls[-1].input
assert len(last_input) > 1 # Should include previous messages
# Check that previous conversation is included
@@ -495,8 +495,8 @@ class TestOpenAIConversationsSessionRunnerIntegration:
@pytest.mark.asyncio
async def test_runner_persists_program_item_ids(self, mock_openai_client):
"""Program items keep the id the Conversations create-item schema requires."""
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
Program(
@@ -29,7 +29,7 @@ from agents.run_internal.items import (
TOOL_CALL_SESSION_DESCRIPTION_KEY,
TOOL_CALL_SESSION_TITLE_KEY,
)
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_function_tool, get_function_tool_call, get_text_message
from tests.utils.simple_session import SimpleListSession
@@ -1463,7 +1463,7 @@ class TestOpenAIResponsesCompactionSession:
should_trigger_compaction=lambda ctx: True,
)
model = FakeModel(initial_output=[get_text_message("ok")])
model = ScriptedModel(steps=[[get_text_message("ok")]])
agent = Agent(name="assistant", model=model)
await Runner.run(agent, "hello", session=session)
@@ -1486,7 +1486,7 @@ class TestOpenAIResponsesCompactionSession:
)
tool = get_function_tool(name="do_thing", return_value="done")
model = FakeModel(initial_output=[get_function_tool_call("do_thing")])
model = ScriptedModel(steps=[[get_function_tool_call("do_thing")]])
agent = Agent(
name="assistant",
model=model,
@@ -1518,7 +1518,7 @@ class TestOpenAIResponsesCompactionSession:
)
tool = get_function_tool(name="do_thing", return_value="done")
model = FakeModel(initial_output=[get_function_tool_call("do_thing")])
model = ScriptedModel(steps=[[get_function_tool_call("do_thing")]])
agent = Agent(
name="assistant",
model=model,
@@ -1554,8 +1554,8 @@ class TestOpenAIResponsesCompactionSession:
)
tool = get_function_tool(name="do_thing", return_value="done")
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("do_thing")],
[get_text_message("ok")],
@@ -1596,8 +1596,8 @@ class TestOpenAIResponsesCompactionSession:
)
tool = get_function_tool(name="do_thing", return_value="done")
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("do_thing")],
[get_function_tool_call("do_thing")],
+30 -30
View File
@@ -11,7 +11,7 @@ import pytest
from agents import Agent, RunConfig, Runner, SessionSettings, SQLiteSession, TResponseInputItem
from agents.memory.sqlite_session import _await_mutation
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
@@ -95,11 +95,11 @@ async def test_session_memory_basic_functionality_parametrized(runner_method):
session_id = "test_session_123"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# First turn
model.set_next_output([get_text_message("San Francisco")])
model.enqueue([get_text_message("San Francisco")])
result1 = await run_agent_async(
runner_method,
agent,
@@ -109,7 +109,7 @@ async def test_session_memory_basic_functionality_parametrized(runner_method):
assert result1.final_output == "San Francisco"
# Second turn - should have conversation history
model.set_next_output([get_text_message("California")])
model.enqueue([get_text_message("California")])
result2 = await run_agent_async(
runner_method,
agent,
@@ -120,7 +120,7 @@ async def test_session_memory_basic_functionality_parametrized(runner_method):
# Verify that the input to the second turn includes the previous conversation
# The model should have received the full conversation history
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
assert len(last_input) > 1 # Should have more than just the current message
session.close()
@@ -135,16 +135,16 @@ async def test_session_memory_with_explicit_instance_parametrized(runner_method)
session_id = "test_session_456"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# First turn
model.set_next_output([get_text_message("Hello")])
model.enqueue([get_text_message("Hello")])
result1 = await run_agent_async(runner_method, agent, "Hi there", session=session)
assert result1.final_output == "Hello"
# Second turn
model.set_next_output([get_text_message("I remember you said hi")])
model.enqueue([get_text_message("I remember you said hi")])
result2 = await run_agent_async(
runner_method,
agent,
@@ -160,21 +160,21 @@ async def test_session_memory_with_explicit_instance_parametrized(runner_method)
@pytest.mark.asyncio
async def test_session_memory_disabled_parametrized(runner_method):
"""Test that session memory is disabled when session=None across all runner methods."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# First turn (no session parameters = disabled)
model.set_next_output([get_text_message("Hello")])
model.enqueue([get_text_message("Hello")])
result1 = await run_agent_async(runner_method, agent, "Hi there")
assert result1.final_output == "Hello"
# Second turn - should NOT have conversation history
model.set_next_output([get_text_message("I don't remember")])
model.enqueue([get_text_message("I don't remember")])
result2 = await run_agent_async(runner_method, agent, "Do you remember what I said?")
assert result2.final_output == "I don't remember"
# Verify that the input to the second turn is just the current message
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
assert len(last_input) == 1 # Should only have the current message
@@ -186,14 +186,14 @@ async def test_session_memory_different_sessions_parametrized(runner_method):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_memory.db"
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# Session 1
session_id_1 = "session_1"
session_1 = SQLiteSession(session_id_1, db_path)
model.set_next_output([get_text_message("I like cats")])
model.enqueue([get_text_message("I like cats")])
result1 = await run_agent_async(runner_method, agent, "I like cats", session=session_1)
assert result1.final_output == "I like cats"
@@ -201,12 +201,12 @@ async def test_session_memory_different_sessions_parametrized(runner_method):
session_id_2 = "session_2"
session_2 = SQLiteSession(session_id_2, db_path)
model.set_next_output([get_text_message("I like dogs")])
model.enqueue([get_text_message("I like dogs")])
result2 = await run_agent_async(runner_method, agent, "I like dogs", session=session_2)
assert result2.final_output == "I like dogs"
# Back to Session 1 - should remember cats, not dogs
model.set_next_output([get_text_message("Yes, you mentioned cats")])
model.enqueue([get_text_message("Yes, you mentioned cats")])
result3 = await run_agent_async(
runner_method,
agent,
@@ -548,7 +548,7 @@ async def test_session_memory_appends_list_input_by_default(runner_method):
session_id = "test_validation_parametrized"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
initial_history: list[TResponseInputItem] = [
@@ -559,10 +559,10 @@ async def test_session_memory_appends_list_input_by_default(runner_method):
list_input = [{"role": "user", "content": "Test message"}]
model.set_next_output([get_text_message("This should run")])
model.enqueue([get_text_message("This should run")])
await run_agent_async(runner_method, agent, list_input, session=session)
assert model.last_turn_args["input"] == initial_history + list_input
assert model.calls[-1].input == initial_history + list_input
session.close()
@@ -574,7 +574,7 @@ async def test_session_callback_prepared_input(runner_method):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_memory.db"
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# Session
@@ -594,7 +594,7 @@ async def test_session_callback_prepared_input(runner_method):
return [item for item in history if item["role"] == "user"] + new_input
new_turn_input = [{"role": "user", "content": "What your name?"}]
model.set_next_output([get_text_message("I'm gpt-4o")])
model.enqueue([get_text_message("I'm gpt-4o")])
# Run the agent with the callable
await run_agent_async(
@@ -610,8 +610,8 @@ async def test_session_callback_prepared_input(runner_method):
new_turn_input[0], # New input
]
assert len(model.last_turn_args["input"]) == 2
assert model.last_turn_args["input"] == expected_model_input
assert len(model.calls[-1].input) == 2
assert model.calls[-1].input == expected_model_input
finally:
session.close()
@@ -621,7 +621,7 @@ async def test_session_callback_prepared_input(runner_method):
async def test_session_callback_repeating_history_does_not_grow_session(runner_method):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_memory.db"
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
session = SQLiteSession("session_repeat", db_path)
@@ -632,7 +632,7 @@ async def test_session_callback_repeating_history_does_not_grow_session(runner_m
try:
for turn in range(3):
model.set_next_output([get_text_message(f"assistant {turn}")])
model.enqueue([get_text_message(f"assistant {turn}")])
await run_agent_async(
runner_method,
agent,
@@ -825,9 +825,9 @@ async def test_session_add_items_exception_propagates_in_streamed():
session.add_items = _failing_add_items # type: ignore[method-assign]
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("This should not be reached")])
model.enqueue([get_text_message("This should not be reached")])
result = Runner.run_streamed(agent, "Hello", session=session)
@@ -981,9 +981,9 @@ async def test_runner_with_session_settings_override():
]
await session.add_items(items)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("Got it")])
model.enqueue([get_text_message("Got it")])
await Runner.run(
agent,
@@ -995,7 +995,7 @@ async def test_runner_with_session_settings_override():
)
# Verify the agent received only the last 2 history items + new question
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
+10 -10
View File
@@ -11,8 +11,8 @@ from agents.guardrail import GuardrailFunctionOutput, InputGuardrail
from agents.memory import OpenAIResponsesCompactionSession
from agents.memory.session import _session_accepts_wrapper, _session_method_accepts_wrapper
from agents.run_internal.session_persistence import rewind_session_items
from agents.testing import ScriptedModel
from agents.tool import function_tool
from tests.fake_model import FakeModel
from tests.test_responses import get_function_tool_call, get_text_message
@@ -136,7 +136,7 @@ class UninspectableAsyncMethod:
@pytest.mark.asyncio
async def test_runner_passes_same_wrapper_to_context_aware_session(streamed: bool) -> None:
session = ContextAwareSession()
model = FakeModel(initial_output=[get_text_message("ok")])
model = ScriptedModel(steps=[[get_text_message("ok")]])
agent = Agent(name="test", model=model)
context = TenantContext(tenant_id="tenant-a")
@@ -161,7 +161,7 @@ async def test_runner_passes_same_wrapper_to_context_aware_session(streamed: boo
@pytest.mark.asyncio
async def test_runner_preserves_legacy_session_call_shapes() -> None:
session = LegacySession()
model = FakeModel(initial_output=[get_text_message("ok")])
model = ScriptedModel(steps=[[get_text_message("ok")]])
agent = Agent(name="test", model=model)
result = await Runner.run(
@@ -179,7 +179,7 @@ async def test_runner_preserves_legacy_session_call_shapes() -> None:
@pytest.mark.asyncio
async def test_runner_does_not_treat_legacy_kwargs_as_wrapper_opt_in() -> None:
session = LegacyKwargsSession()
model = FakeModel(initial_output=[get_text_message("ok")])
model = ScriptedModel(steps=[[get_text_message("ok")]])
result = await Runner.run(
Agent(name="test", model=model),
@@ -197,7 +197,7 @@ async def test_runner_does_not_treat_legacy_kwargs_as_wrapper_opt_in() -> None:
async def test_runner_preserves_legacy_calls_when_signature_inspection_fails() -> None:
session = cast(Any, LegacySession())
session.get_items = UninspectableAsyncMethod(session.get_items)
model = FakeModel(initial_output=[get_text_message("ok")])
model = ScriptedModel(steps=[[get_text_message("ok")]])
result = await Runner.run(
Agent(name="test", model=model),
@@ -245,7 +245,7 @@ async def test_runner_does_not_partially_enable_context_aware_session() -> None:
pass
session = PartialSession()
model = FakeModel(initial_output=[get_text_message("ok")])
model = ScriptedModel(steps=[[get_text_message("ok")]])
result = await Runner.run(
Agent(name="test", model=model),
@@ -338,7 +338,7 @@ async def test_input_guardrail_persists_in_the_context_scope(streamed: bool) ->
context = TenantContext(tenant_id="tenant-a")
agent = Agent(
name="test",
model=FakeModel(initial_output=[get_text_message("not persisted")]),
model=ScriptedModel(steps=[[get_text_message("not persisted")]]),
input_guardrails=[InputGuardrail(guardrail_function=guardrail_function)],
)
@@ -362,8 +362,8 @@ async def test_resumed_run_persists_in_the_context_scope(streamed: bool) -> None
return "tool result"
tool = function_tool(test_tool, name_override="test_tool", needs_approval=True)
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("test_tool", "{}", call_id="call-resume")],
[get_text_message("done")],
@@ -415,7 +415,7 @@ async def test_compaction_session_keeps_context_aware_underlying_on_legacy_scope
)
result = await Runner.run(
Agent(name="test", model=FakeModel(initial_output=[get_text_message("done")])),
Agent(name="test", model=ScriptedModel(steps=[[get_text_message("done")]])),
"hello",
context=TenantContext(tenant_id="tenant-a"),
session=session,
+23 -23
View File
@@ -9,7 +9,7 @@ import pytest
from agents import Agent, RunConfig, SQLiteSession
from agents.items import TResponseInputItem
from agents.memory import SessionSettings
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.memory.test_session import run_agent_async
from tests.test_responses import get_text_message
@@ -24,17 +24,17 @@ async def test_session_limit_parameter(runner_method):
session_id = "limit_test"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# Build up a longer conversation history
model.set_next_output([get_text_message("Reply 1")])
model.enqueue([get_text_message("Reply 1")])
await run_agent_async(runner_method, agent, "Message 1", session=session)
model.set_next_output([get_text_message("Reply 2")])
model.enqueue([get_text_message("Reply 2")])
await run_agent_async(runner_method, agent, "Message 2", session=session)
model.set_next_output([get_text_message("Reply 3")])
model.enqueue([get_text_message("Reply 3")])
await run_agent_async(runner_method, agent, "Message 3", session=session)
# Verify we have 6 items in total (3 user + 3 assistant)
@@ -42,7 +42,7 @@ async def test_session_limit_parameter(runner_method):
assert len(all_items) == 6
# Test session_limit via RunConfig - should only get last 2 history items + new input
model.set_next_output([get_text_message("Reply 4")])
model.enqueue([get_text_message("Reply 4")])
await run_agent_async(
runner_method,
agent,
@@ -52,7 +52,7 @@ async def test_session_limit_parameter(runner_method):
)
# Verify model received limited history
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
# Should have: 2 history items + 1 new message = 3 total
assert len(last_input) == 3
# First item should be "Message 3" (not Message 1 or 2)
@@ -102,8 +102,8 @@ async def test_session_limit_drops_unmatched_history_function_call_output(runner
assert await session.get_items(limit=2) == history[-2:]
model = FakeModel()
model.set_next_output([get_text_message("Tomorrow is sunny too.")])
model = ScriptedModel()
model.enqueue([get_text_message("Tomorrow is sunny too.")])
agent = Agent(name="test", model=model)
await run_agent_async(
@@ -114,7 +114,7 @@ async def test_session_limit_drops_unmatched_history_function_call_output(runner
run_config=RunConfig(session_settings=SessionSettings(limit=2)),
)
assert model.last_turn_args["input"] == [
assert model.calls[-1].input == [
history[-1],
{"role": "user", "content": "What about tomorrow?"},
]
@@ -130,18 +130,18 @@ async def test_session_limit_zero(runner_method):
session_id = "limit_zero_test"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# Build conversation history
model.set_next_output([get_text_message("Reply 1")])
model.enqueue([get_text_message("Reply 1")])
await run_agent_async(runner_method, agent, "Message 1", session=session)
model.set_next_output([get_text_message("Reply 2")])
model.enqueue([get_text_message("Reply 2")])
await run_agent_async(runner_method, agent, "Message 2", session=session)
# Test with limit=0 - should get NO history, just new message
model.set_next_output([get_text_message("Reply 3")])
model.enqueue([get_text_message("Reply 3")])
await run_agent_async(
runner_method,
agent,
@@ -151,7 +151,7 @@ async def test_session_limit_zero(runner_method):
)
# Verify model received only the new message
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
assert len(last_input) == 1
assert last_input[0].get("content") == "Message 3"
@@ -167,12 +167,12 @@ async def test_session_limit_none_gets_all_history(runner_method):
session_id = "limit_none_test"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# Build longer conversation
for i in range(1, 6):
model.set_next_output([get_text_message(f"Reply {i}")])
model.enqueue([get_text_message(f"Reply {i}")])
await run_agent_async(runner_method, agent, f"Message {i}", session=session)
# Verify 10 items in session (5 user + 5 assistant)
@@ -180,7 +180,7 @@ async def test_session_limit_none_gets_all_history(runner_method):
assert len(all_items) == 10
# Test with session_limit=None (default) - should get all history
model.set_next_output([get_text_message("Reply 6")])
model.enqueue([get_text_message("Reply 6")])
await run_agent_async(
runner_method,
agent,
@@ -190,7 +190,7 @@ async def test_session_limit_none_gets_all_history(runner_method):
)
# Verify model received all history + new message
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
assert len(last_input) == 11 # 10 history + 1 new
assert last_input[0].get("content") == "Message 1"
assert last_input[-1].get("content") == "Message 6"
@@ -207,15 +207,15 @@ async def test_session_limit_larger_than_history(runner_method):
session_id = "limit_large_test"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# Build small conversation
model.set_next_output([get_text_message("Reply 1")])
model.enqueue([get_text_message("Reply 1")])
await run_agent_async(runner_method, agent, "Message 1", session=session)
# Test with limit=100 (much larger than actual history)
model.set_next_output([get_text_message("Reply 2")])
model.enqueue([get_text_message("Reply 2")])
await run_agent_async(
runner_method,
agent,
@@ -225,7 +225,7 @@ async def test_session_limit_larger_than_history(runner_method):
)
# Verify model received all available history + new message
last_input = model.last_turn_args["input"]
last_input = model.calls[-1].input
assert len(last_input) == 3 # 2 history + 1 new
assert last_input[0].get("content") == "Message 1"
# Assistant message has content as a list
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
import copy
from collections.abc import AsyncIterator
from openai.types.responses import (
Response,
ResponseCompletedEvent,
ResponseOutputItemDoneEvent,
ResponseUsage,
)
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
from agents.items import TResponseOutputItem, TResponseStreamEvent
from agents.testing import ModelStep
from agents.usage import Usage
def get_response_obj(
output: list[TResponseOutputItem],
response_id: str | None = None,
usage: Usage | None = None,
) -> Response:
"""Build an OpenAI response object for adapter-level tests."""
return Response(
id=response_id or "resp-789",
created_at=123,
model="test_model",
object="response",
output=output,
tool_choice="none",
tools=[],
top_p=None,
parallel_tool_calls=False,
usage=ResponseUsage(
input_tokens=usage.input_tokens if usage else 0,
output_tokens=usage.output_tokens if usage else 0,
total_tokens=usage.total_tokens if usage else 0,
input_tokens_details=InputTokensDetails.model_validate(
{
"cache_write_tokens": (
getattr(usage.input_tokens_details, "cache_write_tokens", 0) if usage else 0
),
"cached_tokens": (
getattr(usage.input_tokens_details, "cached_tokens", 0) if usage else 0
),
}
),
output_tokens_details=OutputTokensDetails(
reasoning_tokens=(
getattr(usage.output_tokens_details, "reasoning_tokens", 0) if usage else 0
)
),
),
)
def get_exact_output_stream_step(output: list[TResponseOutputItem]) -> ModelStep:
"""Build an exact normalized stream for tests whose subject is downstream processing."""
stream_output = copy.deepcopy(output)
async def events(_call: object) -> AsyncIterator[TResponseStreamEvent]:
for output_index, output_item in enumerate(stream_output):
yield ResponseOutputItemDoneEvent(
type="response.output_item.done",
item=output_item,
output_index=output_index,
sequence_number=output_index,
)
yield ResponseCompletedEvent(
type="response.completed",
response=get_response_obj(stream_output),
sequence_number=len(stream_output),
)
return ModelStep.stream(events)
+3 -3
View File
@@ -158,9 +158,9 @@ def test_unknown_prefix_can_be_preserved_for_openai_compatible_model_ids(monkeyp
def get_model(self, model_name):
captured_model["value"] = model_name
fake_model = object()
captured_result["value"] = fake_model
return fake_model
mapped_model = object()
captured_result["value"] = mapped_model
return mapped_model
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
+1 -1
View File
@@ -41,7 +41,7 @@ from agents.models.openai_responses import (
)
from agents.retry import ModelRetryAdviceRequest
from agents.usage import Usage
from tests.fake_model import get_response_obj
from tests.model_test_helpers import get_response_obj
from tests.testing_processor import fetch_ordered_spans
+17 -39
View File
@@ -4,42 +4,20 @@ import pytest
from agents.realtime.agent import RealtimeAgent
from agents.realtime.config import RealtimeRunConfig, RealtimeSessionModelSettings
from agents.realtime.model import RealtimeModel, RealtimeModelConfig
from agents.realtime.model import RealtimeModelConfig
from agents.realtime.runner import RealtimeRunner
from agents.realtime.session import RealtimeSession
from agents.realtime.testing import RealtimeConnectCall, ScriptedRealtimeModel
from agents.tool import function_tool
class MockRealtimeModel(RealtimeModel):
def __init__(self):
self.connect_args = None
class RunnerRealtimeModel(ScriptedRealtimeModel):
def __init__(self) -> None:
super().__init__(strict=False)
async def connect(self, options=None):
self.connect_args = options
def add_listener(self, listener):
pass
def remove_listener(self, listener):
pass
async def send_event(self, event):
pass
async def send_message(self, message, other_event_data=None):
pass
async def send_audio(self, audio, commit=False):
pass
async def send_tool_output(self, tool_call, output, start_response=True):
pass
async def interrupt(self):
pass
async def close(self):
pass
@property
def connect_args(self) -> RealtimeConnectCall | None:
return self.connect_calls[-1] if self.connect_calls else None
@pytest.fixture
@@ -52,12 +30,12 @@ def mock_agent():
@pytest.fixture
def mock_model():
return MockRealtimeModel()
return RunnerRealtimeModel()
@pytest.mark.asyncio
async def test_run_preserves_falsey_custom_model(mock_agent: Mock):
class FalseyRealtimeModel(MockRealtimeModel):
class FalseyRealtimeModel(RunnerRealtimeModel):
def __bool__(self) -> bool:
return False
@@ -70,7 +48,7 @@ async def test_run_preserves_falsey_custom_model(mock_agent: Mock):
@pytest.mark.asyncio
async def test_run_creates_session_with_no_settings(
mock_agent: Mock, mock_model: MockRealtimeModel
mock_agent: Mock, mock_model: RunnerRealtimeModel
):
"""Test that run() creates a session correctly if no settings are provided"""
runner = RealtimeRunner(mock_agent, model=mock_model)
@@ -98,7 +76,7 @@ async def test_run_creates_session_with_no_settings(
@pytest.mark.asyncio
async def test_run_creates_session_with_settings_only_in_init(
mock_agent: Mock, mock_model: MockRealtimeModel
mock_agent: Mock, mock_model: RunnerRealtimeModel
):
"""Test that it creates a session with the right settings if they are provided only in init"""
config = RealtimeRunConfig(
@@ -122,7 +100,7 @@ async def test_run_creates_session_with_settings_only_in_init(
@pytest.mark.asyncio
async def test_run_creates_session_with_settings_in_both_init_and_run_overrides(
mock_agent: Mock, mock_model: MockRealtimeModel
mock_agent: Mock, mock_model: RunnerRealtimeModel
):
"""Test settings provided in run() parameter are passed through"""
init_config = RealtimeRunConfig(
@@ -152,7 +130,7 @@ async def test_run_creates_session_with_settings_in_both_init_and_run_overrides(
@pytest.mark.asyncio
async def test_run_creates_session_with_settings_only_in_run(
mock_agent: Mock, mock_model: MockRealtimeModel
mock_agent: Mock, mock_model: RunnerRealtimeModel
):
"""Test settings provided only in run()"""
runner = RealtimeRunner(mock_agent, model=mock_model)
@@ -178,7 +156,7 @@ async def test_run_creates_session_with_settings_only_in_run(
@pytest.mark.asyncio
async def test_run_with_context_parameter(mock_agent: Mock, mock_model: MockRealtimeModel):
async def test_run_with_context_parameter(mock_agent: Mock, mock_model: RunnerRealtimeModel):
"""Test that context parameter is passed through to session"""
runner = RealtimeRunner(mock_agent, model=mock_model)
test_context = {"user_id": "test123"}
@@ -194,7 +172,7 @@ async def test_run_with_context_parameter(mock_agent: Mock, mock_model: MockReal
@pytest.mark.asyncio
async def test_run_with_none_values_from_agent_does_not_crash(mock_model: MockRealtimeModel):
async def test_run_with_none_values_from_agent_does_not_crash(mock_model: RunnerRealtimeModel):
"""Test that runner handles agents with None values without crashing"""
agent = Mock(spec=RealtimeAgent)
agent.get_system_prompt = AsyncMock(return_value=None)
@@ -216,7 +194,7 @@ async def test_run_with_none_values_from_agent_does_not_crash(mock_model: MockRe
@pytest.mark.asyncio
async def test_tool_and_handoffs_are_correct(mock_model: MockRealtimeModel):
async def test_tool_and_handoffs_are_correct(mock_model: RunnerRealtimeModel):
@function_tool
def tool_one():
return "result_one"
+36 -63
View File
@@ -76,6 +76,7 @@ from agents.realtime.session import (
_PendingToolOutputSendError,
_serialize_tool_output,
)
from agents.realtime.testing import RealtimeConnectCall, ScriptedRealtimeModel
from agents.run_context import RunContextWrapper
from agents.tool import FunctionTool, function_tool, tool_namespace
from agents.tool_context import ToolContext
@@ -87,39 +88,23 @@ from agents.tool_guardrails import (
from agents.usage import Usage
class _DummyModel(RealtimeModel):
class _DummyModel(ScriptedRealtimeModel):
def __init__(self) -> None:
super().__init__()
self.events: list[Any] = []
self.listeners: list[Any] = []
self.connect_options: Any | None = None
super().__init__(strict=False)
async def connect(self, options=None):
self.connect_options = options
@property
def events(self) -> tuple[Any, ...]:
return self.sent_events
async def close(self): # pragma: no cover - not used here
pass
async def send_event(self, event):
self.events.append(event)
def add_listener(self, listener):
self.listeners.append(listener)
def remove_listener(self, listener):
if listener in self.listeners:
self.listeners.remove(listener)
@property
def connect_options(self) -> RealtimeConnectCall | None:
return self.connect_calls[-1] if self.connect_calls else None
class _FailingConnectModel(_DummyModel):
def __init__(self, exc: BaseException) -> None:
super().__init__()
self.exc = exc
self.connect_options: Any | None = None
async def connect(self, options=None):
self.connect_options = options
raise self.exc
self._connect_error = exc
def _agent_with_ambiguous_realtime_tools(name: str = "invalid_agent") -> RealtimeAgent:
@@ -208,8 +193,11 @@ async def test_property_and_send_helpers_and_enter_alias():
# property
assert session.model is model
# enter alias calls __aenter__
async with await session.enter():
# The enter alias calls __aenter__, so callers close it manually.
entered = await session.enter()
try:
assert entered is session
# send helpers
await session.send_message("hi")
await session.send_audio(b"abc", commit=True)
@@ -219,6 +207,8 @@ async def test_property_and_send_helpers_and_enter_alias():
assert any(isinstance(e, RealtimeModelSendUserInput) for e in model.events)
assert any(isinstance(e, RealtimeModelSendAudio) and e.commit for e in model.events)
assert any(isinstance(e, RealtimeModelSendInterrupt) for e in model.events)
finally:
await session.close()
@pytest.mark.asyncio
@@ -1278,7 +1268,7 @@ async def test_aenter_validates_initial_model_settings_before_listener_registrat
with pytest.raises(UserError, match="Duplicate Realtime tool"):
await session.__aenter__()
assert model.listeners == []
assert model.listeners == ()
@pytest.mark.parametrize(
@@ -1296,16 +1286,12 @@ async def test_aenter_removes_listener_when_connect_fails(exc: BaseException):
await session.__aenter__()
assert model.connect_options is not None
assert model.listeners == []
assert model.listeners == ()
class MockRealtimeModel(RealtimeModel):
class RecordingRealtimeModel(ScriptedRealtimeModel):
def __init__(self):
super().__init__()
self.listeners = []
self.connect_called = False
self.close_called = False
self.sent_events = []
super().__init__(strict=False)
# Legacy tracking for tests that haven't been updated yet
self.sent_messages = []
self.sent_audio = []
@@ -1313,16 +1299,6 @@ class MockRealtimeModel(RealtimeModel):
self.interrupts_called = 0
self.retired_audio_response_ids = []
async def connect(self, options=None):
self.connect_called = True
def add_listener(self, listener):
self.listeners.append(listener)
def remove_listener(self, listener):
if listener in self.listeners:
self.listeners.remove(listener)
async def send_event(self, event):
from agents.realtime.model_inputs import (
RealtimeModelSendAudio,
@@ -1331,7 +1307,7 @@ class MockRealtimeModel(RealtimeModel):
RealtimeModelSendUserInput,
)
self.sent_events.append(event)
self._sent_events.append(self._snapshot_send_event(event))
# Update legacy tracking for compatibility
if isinstance(event, RealtimeModelSendUserInput):
@@ -1349,9 +1325,6 @@ class MockRealtimeModel(RealtimeModel):
await self.send_event(event)
return True
async def close(self):
self.close_called = True
def _retire_response_audio(self, response_id: str) -> None:
self.retired_audio_response_ids.append(response_id)
@@ -1368,7 +1341,7 @@ def mock_agent():
@pytest.fixture
def mock_model():
return MockRealtimeModel()
return RecordingRealtimeModel()
def _set_default_timeout_fields(tool: Mock) -> Mock:
@@ -1392,7 +1365,7 @@ def _named_function_tool(
return tool
def _sent_tool_output_strings(model: MockRealtimeModel) -> list[str]:
def _sent_tool_output_strings(model: RecordingRealtimeModel) -> list[str]:
return [output for _call, output, _start_response in model.sent_tool_outputs]
@@ -2598,7 +2571,7 @@ class TestToolCallExecution:
):
"""An approved call should retry cached output only for the same invocation."""
class FailingToolOutputModel(MockRealtimeModel):
class FailingToolOutputModel(RecordingRealtimeModel):
def __init__(self):
super().__init__()
self.fail_next_tool_output = True
@@ -2696,7 +2669,7 @@ class TestToolCallExecution:
):
"""The async approval path should bind retries to the original invocation."""
class FailingToolOutputModel(MockRealtimeModel):
class FailingToolOutputModel(RecordingRealtimeModel):
def __init__(self):
super().__init__()
self.fail_next_tool_output = True
@@ -3012,7 +2985,7 @@ class TestToolCallExecution:
)
assert session._current_agent is first_agent
assert mock_model.sent_events == []
assert mock_model.sent_events == ()
assert mock_model.sent_tool_outputs == []
assert "call_invalid" not in session._active_tool_invocations
assert not session._context_wrapper._tool_invocations["call_invalid"].completed
@@ -3807,7 +3780,7 @@ class TestToolCallExecution:
):
"""A duplicate event during rejection output sending should not emit a second output."""
class BlockingToolOutputModel(MockRealtimeModel):
class BlockingToolOutputModel(RecordingRealtimeModel):
def __init__(self):
super().__init__()
self.started = asyncio.Event()
@@ -4171,7 +4144,7 @@ class TestToolCallExecution:
@pytest.mark.asyncio
async def test_pending_function_output_rejects_handoff_role_reuse(self):
class FailingToolOutputModel(MockRealtimeModel):
class FailingToolOutputModel(RecordingRealtimeModel):
async def send_event(self, event):
if isinstance(event, RealtimeModelSendToolOutput):
raise RuntimeError("send failed")
@@ -4627,14 +4600,14 @@ class TestToolCallExecution:
assert sent_output == json.dumps({"name": "demo", "score": 7})
def test_serialize_tool_output_ignores_non_pydantic_model_dump_objects(self) -> None:
class FakeModelDump:
class ModelDumpObject:
def model_dump(self, *_args: Any, **_kwargs: Any) -> dict[str, Any]:
raise AssertionError("non-pydantic objects should not use model_dump")
def __str__(self) -> str:
return "fake-model-dump-object"
assert _serialize_tool_output(FakeModelDump()) == "fake-model-dump-object"
assert _serialize_tool_output(ModelDumpObject()) == "fake-model-dump-object"
def test_serialize_tool_output_falls_back_when_pydantic_json_dump_fails(self) -> None:
class FallbackModel(BaseModel):
@@ -5044,7 +5017,7 @@ class TestGuardrailFunctionality:
release_guardrail = asyncio.Event()
operations: list[str] = []
class TrackingModel(MockRealtimeModel):
class TrackingModel(RecordingRealtimeModel):
async def send_event(self, event):
await super().send_event(event)
if isinstance(event, RealtimeModelSendInterrupt):
@@ -5158,7 +5131,7 @@ class TestGuardrailFunctionality:
@pytest.mark.asyncio
async def test_response_audio_cleanup_error_releases_session_suppression(self, mock_agent):
class FailingRetirementModel(MockRealtimeModel):
class FailingRetirementModel(RecordingRealtimeModel):
def _retire_response_audio(self, response_id: str) -> None:
raise RuntimeError(f"failed to retire {response_id}")
@@ -5251,7 +5224,7 @@ class TestGuardrailFunctionality:
feedback_send_started = asyncio.Event()
release_feedback_send = asyncio.Event()
class BoundaryCheckingModel(MockRealtimeModel):
class BoundaryCheckingModel(RecordingRealtimeModel):
async def send_event_if(self, event, send_if):
feedback_send_started.set()
await release_feedback_send.wait()
@@ -5288,7 +5261,7 @@ class TestGuardrailFunctionality:
async def test_output_text_guardrail_skips_feedback_without_atomic_model_send(
self, mock_agent, triggered_guardrail
):
class CustomModelWithoutAtomicSend(MockRealtimeModel):
class CustomModelWithoutAtomicSend(RecordingRealtimeModel):
def __init__(self):
super().__init__()
self.feedback_send_started = False
@@ -6128,7 +6101,7 @@ class TestUpdateAgentFunctionality:
await session.update_agent(invalid_agent)
assert session._current_agent is first_agent
assert mock_model.sent_events == []
assert mock_model.sent_events == ()
class TestTranscriptPreservation:
+37 -118
View File
@@ -2,87 +2,23 @@ from __future__ import annotations
import asyncio
import json
from typing import Any
from unittest.mock import AsyncMock, Mock
import pytest
import websockets.exceptions
from agents.realtime.events import RealtimeError
from agents.realtime.model import RealtimeModel, RealtimeModelConfig, RealtimeModelListener
from agents.realtime.model_events import (
RealtimeModelErrorEvent,
RealtimeModelEvent,
RealtimeModelExceptionEvent,
)
from agents.realtime.session import RealtimeSession
from agents.realtime.testing import ScriptedRealtimeModel
class FakeRealtimeModel(RealtimeModel):
"""Fake model for testing that forwards events to listeners."""
def __init__(self):
self._listeners: list[RealtimeModelListener] = []
self._events_to_send: list[RealtimeModelEvent] = []
self._is_connected = False
self._send_task: asyncio.Task[None] | None = None
def set_next_events(self, events: list[RealtimeModelEvent]) -> None:
"""Set events to be sent to listeners."""
self._events_to_send = events.copy()
async def connect(self, options: RealtimeModelConfig) -> None:
"""Fake connection that starts sending events."""
self._is_connected = True
self._send_task = asyncio.create_task(self._send_events())
async def _send_events(self) -> None:
"""Send queued events to all listeners."""
for event in self._events_to_send:
await asyncio.sleep(0.001) # Small delay to simulate async behavior
for listener in self._listeners:
await listener.on_event(event)
def add_listener(self, listener: RealtimeModelListener) -> None:
"""Add a listener."""
self._listeners.append(listener)
def remove_listener(self, listener: RealtimeModelListener) -> None:
"""Remove a listener."""
if listener in self._listeners:
self._listeners.remove(listener)
async def close(self) -> None:
"""Close the fake model."""
self._is_connected = False
if self._send_task and not self._send_task.done():
self._send_task.cancel()
try:
await self._send_task
except asyncio.CancelledError:
pass
async def send_message(
self, message: Any, other_event_data: dict[str, Any] | None = None
) -> None:
"""Fake send message."""
pass
async def send_audio(self, audio: bytes, *, commit: bool = False) -> None:
"""Fake send audio."""
pass
async def send_event(self, event: Any) -> None:
"""Fake send event."""
pass
async def send_tool_output(self, tool_call: Any, output: str, start_response: bool) -> None:
"""Fake send tool output."""
pass
async def interrupt(self) -> None:
"""Fake interrupt."""
pass
def model_with_events(*events: RealtimeModelEvent) -> ScriptedRealtimeModel:
return ScriptedRealtimeModel(connect_events=events, strict=False)
@pytest.fixture
@@ -95,19 +31,11 @@ def fake_agent():
return agent
@pytest.fixture
def fake_model():
"""Create a fake model for testing."""
return FakeRealtimeModel()
class TestSessionExceptions:
"""Test exception handling in RealtimeSession."""
@pytest.mark.asyncio
async def test_end_to_end_exception_propagation_and_cleanup(
self, fake_model: FakeRealtimeModel, fake_agent
):
async def test_end_to_end_exception_propagation_and_cleanup(self, fake_agent):
"""Test that exceptions are stored, trigger cleanup, and are raised in __aiter__."""
# Create test exception
test_exception = ValueError("Test error")
@@ -116,10 +44,8 @@ class TestSessionExceptions:
)
# Set up session
session = RealtimeSession(fake_model, fake_agent, None)
# Set events to send
fake_model.set_next_events([exception_event])
model = model_with_events(exception_event)
session = RealtimeSession(model, fake_agent, None)
# Start session
async with session:
@@ -131,13 +57,11 @@ class TestSessionExceptions:
# Verify cleanup occurred
assert session._closed is True
assert session._stored_exception == test_exception
assert fake_model._is_connected is False
assert len(fake_model._listeners) == 0
assert model.connected is False
assert model.listeners == ()
@pytest.mark.asyncio
async def test_websocket_connection_closure_type_distinction(
self, fake_model: FakeRealtimeModel, fake_agent
):
async def test_websocket_connection_closure_type_distinction(self, fake_agent):
"""Test different WebSocket closure types generate appropriate events."""
# Test ConnectionClosed (should create exception event)
error_closure = websockets.exceptions.ConnectionClosed(None, None)
@@ -145,8 +69,7 @@ class TestSessionExceptions:
exception=error_closure, context="WebSocket connection closed unexpectedly"
)
session = RealtimeSession(fake_model, fake_agent, None)
fake_model.set_next_events([error_event])
session = RealtimeSession(model_with_events(error_event), fake_agent, None)
with pytest.raises(websockets.exceptions.ConnectionClosed):
async with session:
@@ -158,7 +81,7 @@ class TestSessionExceptions:
assert isinstance(session._stored_exception, websockets.exceptions.ConnectionClosed)
@pytest.mark.asyncio
async def test_json_parsing_error_handling(self, fake_model: FakeRealtimeModel, fake_agent):
async def test_json_parsing_error_handling(self, fake_agent):
"""Test JSON parsing errors are properly handled and contextualized."""
# Create JSON decode error
json_error = json.JSONDecodeError("Invalid JSON", "bad json", 0)
@@ -166,8 +89,7 @@ class TestSessionExceptions:
exception=json_error, context="Failed to parse WebSocket message as JSON"
)
session = RealtimeSession(fake_model, fake_agent, None)
fake_model.set_next_events([json_exception_event])
session = RealtimeSession(model_with_events(json_exception_event), fake_agent, None)
with pytest.raises(json.JSONDecodeError):
async with session:
@@ -179,7 +101,7 @@ class TestSessionExceptions:
assert session._closed is True
@pytest.mark.asyncio
async def test_exception_context_preservation(self, fake_model: FakeRealtimeModel, fake_agent):
async def test_exception_context_preservation(self, fake_agent):
"""Test that exception context information is preserved through the handling process."""
test_contexts = [
("Failed to send audio", RuntimeError("Audio encoding failed")),
@@ -190,8 +112,8 @@ class TestSessionExceptions:
for context, exception in test_contexts:
exception_event = RealtimeModelExceptionEvent(exception=exception, context=context)
session = RealtimeSession(fake_model, fake_agent, None)
fake_model.set_next_events([exception_event])
model = model_with_events(exception_event)
session = RealtimeSession(model, fake_agent, None)
with pytest.raises(type(exception)):
async with session:
@@ -202,14 +124,8 @@ class TestSessionExceptions:
assert session._stored_exception == exception
assert session._closed is True
# Reset for next iteration
fake_model._is_connected = False
fake_model._listeners.clear()
@pytest.mark.asyncio
async def test_multiple_exception_handling_behavior(
self, fake_model: FakeRealtimeModel, fake_agent
):
async def test_multiple_exception_handling_behavior(self, fake_agent):
"""Test behavior when multiple exceptions occur before consumption."""
# Create multiple exceptions
first_exception = ValueError("First error")
@@ -222,13 +138,11 @@ class TestSessionExceptions:
exception=second_exception, context="Second context"
)
session = RealtimeSession(fake_model, fake_agent, None)
fake_model.set_next_events([first_event, second_event])
session = RealtimeSession(model_with_events(first_event, second_event), fake_agent, None)
# Start session and let events process
# Start the session after both events are configured for connection.
async with session:
# Give time for events to be processed
await asyncio.sleep(0.05)
pass
# The first exception should be stored (second should overwrite, but that's
# the current behavior). In practice, once an exception occurs, cleanup
@@ -237,9 +151,7 @@ class TestSessionExceptions:
assert session._closed is True
@pytest.mark.asyncio
async def test_exception_during_guardrail_processing(
self, fake_model: FakeRealtimeModel, fake_agent
):
async def test_exception_during_guardrail_processing(self, fake_agent):
"""Test that exceptions don't interfere with guardrail task cleanup."""
# Create exception event
test_exception = RuntimeError("Processing error")
@@ -247,7 +159,8 @@ class TestSessionExceptions:
exception=test_exception, context="Processing failed"
)
session = RealtimeSession(fake_model, fake_agent, None)
model = model_with_events(exception_event)
session = RealtimeSession(model, fake_agent, None)
async def running_task() -> None:
await asyncio.Event().wait()
@@ -261,8 +174,6 @@ class TestSessionExceptions:
await completed
session._guardrail_tasks = {pending, completed}
fake_model.set_next_events([exception_event])
with pytest.raises(RuntimeError, match="Processing error"):
async with session:
async for _event in session:
@@ -275,9 +186,7 @@ class TestSessionExceptions:
assert len(session._guardrail_tasks) == 0
@pytest.mark.asyncio
async def test_normal_events_still_work_before_exception(
self, fake_model: FakeRealtimeModel, fake_agent
):
async def test_normal_events_still_work_before_exception(self, fake_agent):
"""Test that normal events are processed before an exception occurs."""
# Create normal event followed by exception
normal_event = RealtimeModelErrorEvent(error={"message": "Normal error"})
@@ -285,15 +194,25 @@ class TestSessionExceptions:
exception=ValueError("Fatal error"), context="Fatal context"
)
session = RealtimeSession(fake_model, fake_agent, None)
fake_model.set_next_events([normal_event, exception_event])
model = ScriptedRealtimeModel(strict=False)
session = RealtimeSession(model, fake_agent, None)
events_received = []
with pytest.raises(ValueError, match="Fatal error"):
async with session:
async for event in session:
events_received.append(event)
async def emit_events() -> None:
await model.emit(normal_event)
await asyncio.sleep(0)
await model.emit(exception_event)
emitter = asyncio.create_task(emit_events())
try:
async for event in session:
events_received.append(event)
finally:
await emitter
# Should have received events before exception
assert len(events_received) >= 1
File diff suppressed because it is too large Load Diff
@@ -17,6 +17,7 @@ from agents.run_internal.run_steps import ToolRunCustom
from agents.run_internal.tool_actions import CustomToolAction
from agents.sandbox.capabilities.tools import SandboxApplyPatchTool
from agents.sandbox.types import User
from agents.testing import scripted_sandbox_session
from tests.sandbox._apply_patch_test_session import (
ApplyPatchSession,
UserRecordingApplyPatchSession,
@@ -26,7 +27,7 @@ from tests.utils.hitl import make_context_wrapper
class TestSandboxApplyPatchTool:
def test_exposes_custom_apply_patch_tool(self) -> None:
tool = SandboxApplyPatchTool(session=ApplyPatchSession())
tool = SandboxApplyPatchTool(session=scripted_sandbox_session())
assert isinstance(tool, CustomTool)
assert tool.name == "apply_patch"
@@ -36,7 +37,7 @@ class TestSandboxApplyPatchTool:
assert tool.tool_config["format"]["syntax"] == "lark"
def test_converter_uses_sandbox_custom_apply_patch_tool_config(self) -> None:
tool = SandboxApplyPatchTool(session=ApplyPatchSession())
tool = SandboxApplyPatchTool(session=scripted_sandbox_session())
converted = Converter.convert_tools([tool], handoffs=[])
@@ -55,7 +56,9 @@ class TestSandboxApplyPatchTool:
) -> bool:
return operation.type != "create_file"
tool = SandboxApplyPatchTool(session=ApplyPatchSession(), needs_approval=needs_approval)
tool = SandboxApplyPatchTool(
session=scripted_sandbox_session(), needs_approval=needs_approval
)
assert cast(object, tool.needs_approval) is needs_approval
assert cast(object, tool.operation_needs_approval) is needs_approval
@@ -67,7 +70,7 @@ class TestSandboxApplyPatchTool:
) -> bool:
return operation.type == "delete_file"
tool = SandboxApplyPatchTool(session=ApplyPatchSession())
tool = SandboxApplyPatchTool(session=scripted_sandbox_session())
tool.needs_approval = needs_approval
result = await _execute_custom_tool_call(
@@ -147,7 +150,7 @@ class TestSandboxApplyPatchTool:
@pytest.mark.asyncio
async def test_invalid_patch_input_surfaces_tool_error_after_approval_precheck(self) -> None:
tool = SandboxApplyPatchTool(session=ApplyPatchSession(), needs_approval=True)
tool = SandboxApplyPatchTool(session=scripted_sandbox_session(), needs_approval=True)
result = await _execute_custom_tool_call(
tool,
@@ -1,8 +1,6 @@
from __future__ import annotations
import io
import uuid
from pathlib import Path
from typing import Any, cast
import pytest
@@ -17,243 +15,60 @@ from agents.sandbox.capabilities.tools import (
)
from agents.sandbox.capabilities.tools.shell_tool import _resolve_shell
from agents.sandbox.errors import ExecTimeoutError, ExecTransportError, PtySessionNotFoundError
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
from agents.sandbox.session.pty_types import PtyExecUpdate
from agents.sandbox.snapshot import NoopSnapshot
from agents.sandbox.types import ExecResult, User
from agents.testing import scripted_sandbox_session
from agents.tool import FunctionTool
from agents.tool_context import ToolContext
from tests.utils.factories import TestSessionState
class _ShellSession(BaseSandboxSession):
def __init__(self, manifest: Manifest) -> None:
self.state = TestSessionState(
manifest=manifest,
snapshot=NoopSnapshot(id=str(uuid.uuid4())),
)
self.exec_calls: list[tuple[str, float | None, bool | list[str]]] = []
self.exec_users: list[str | None] = []
async def start(self) -> None:
return None
async def stop(self) -> None:
return None
async def shutdown(self) -> None:
return None
async def running(self) -> bool:
return True
async def read(self, path: Path, *, user: object = None) -> io.BytesIO:
_ = (path, user)
raise AssertionError("read() should not be called")
async def write(self, path: Path, data: io.IOBase, *, user: object = None) -> None:
_ = (path, data, user)
raise AssertionError("write() should not be called")
async def _exec_internal(
self,
*command: str | Path,
timeout: float | None = None,
) -> ExecResult:
_ = command
_ = timeout
raise AssertionError("_exec_internal() should not be called directly")
async def exec(
self,
*command: str | Path,
timeout: float | None = None,
user: str | User | None = None,
shell: bool | list[str] = False,
) -> ExecResult:
self.exec_users.append(user.name if isinstance(user, User) else user)
rendered_command = " ".join(str(part) for part in command)
self.exec_calls.append((rendered_command, timeout, shell))
return ExecResult(
stdout=f"stdout: {rendered_command}".encode(),
stderr=f"stderr: {rendered_command}".encode(),
exit_code=7,
)
async def persist_workspace(self) -> io.IOBase:
return io.BytesIO()
async def hydrate_workspace(self, data: io.IOBase) -> None:
_ = data
def _default_exec_result(call: Any) -> ExecResult:
rendered_command = " ".join(str(part) for part in call.args)
return ExecResult(
stdout=f"stdout: {rendered_command}".encode(),
stderr=f"stderr: {rendered_command}".encode(),
exit_code=7,
)
class _TimeoutShellSession(_ShellSession):
async def exec(
self,
*command: str | Path,
timeout: float | None = None,
user: str | User | None = None,
shell: bool | list[str] = False,
) -> ExecResult:
_ = (command, user, shell)
raise ExecTimeoutError(command=("sleep 30",), timeout_s=timeout)
def _shell_session(
*,
manifest: Manifest | None = None,
result: ExecResult | None = None,
error: Exception | None = None,
) -> Any:
outcome: dict[str, object]
if error is not None:
outcome = {"error": error}
elif result is not None:
outcome = {"result": result}
else:
outcome = {"responder": _default_exec_result}
step: dict[str, object] = {"method": "exec"}
step.update(outcome)
return scripted_sandbox_session(
cast(Any, [step]),
manifest=manifest or Manifest(root="/workspace"),
)
class _OutputShellSession(_ShellSession):
def __init__(
self,
manifest: Manifest,
*,
stdout: bytes,
stderr: bytes,
exit_code: int = 7,
) -> None:
super().__init__(manifest)
self.stdout = stdout
self.stderr = stderr
self.exit_code = exit_code
async def exec(
self,
*command: str | Path,
timeout: float | None = None,
user: str | User | None = None,
shell: bool | list[str] = False,
) -> ExecResult:
self.exec_users.append(user.name if isinstance(user, User) else user)
rendered_command = " ".join(str(part) for part in command)
self.exec_calls.append((rendered_command, timeout, shell))
return ExecResult(stdout=self.stdout, stderr=self.stderr, exit_code=self.exit_code)
def _pty_session(
steps: list[dict[str, object]],
*,
manifest: Manifest | None = None,
) -> Any:
return scripted_sandbox_session(
cast(Any, steps),
manifest=manifest or Manifest(root="/workspace"),
)
class _PtyShellSession(_ShellSession):
def __init__(self, manifest: Manifest) -> None:
super().__init__(manifest)
self._next_session_id = 1337
self._live_sessions: set[int] = set()
self.last_exec_yield_time_s: float | None = None
self.last_exec_user: str | None = None
self.last_write_yield_time_s: float | None = None
def supports_pty(self) -> bool:
return True
async def pty_exec_start(
self,
*command: str | Path,
timeout: float | None = None,
shell: bool | list[str] = True,
user: str | User | None = None,
tty: bool = False,
yield_time_s: float | None = None,
max_output_tokens: int | None = None,
) -> PtyExecUpdate:
_ = (command, timeout, shell, tty, max_output_tokens)
self.last_exec_user = user.name if isinstance(user, User) else user
self.last_exec_yield_time_s = yield_time_s
session_id = self._next_session_id
self._next_session_id += 1
self._live_sessions.add(session_id)
return PtyExecUpdate(
process_id=session_id,
output=b"",
exit_code=None,
original_token_count=None,
)
async def pty_write_stdin(
self,
*,
session_id: int,
chars: str,
yield_time_s: float | None = None,
max_output_tokens: int | None = None,
) -> PtyExecUpdate:
_ = max_output_tokens
self.last_write_yield_time_s = yield_time_s
if session_id not in self._live_sessions:
raise PtySessionNotFoundError(session_id=session_id)
self._live_sessions.discard(session_id)
return PtyExecUpdate(
process_id=None,
output=chars.encode("utf-8", errors="replace"),
exit_code=0,
original_token_count=None,
)
class _PtyNoStdinShellSession(_PtyShellSession):
async def pty_write_stdin(
self,
*,
session_id: int,
chars: str,
yield_time_s: float | None = None,
max_output_tokens: int | None = None,
) -> PtyExecUpdate:
_ = (chars, yield_time_s, max_output_tokens)
if session_id not in self._live_sessions:
raise PtySessionNotFoundError(session_id=session_id)
raise RuntimeError("stdin is not available for this process")
class _PtyUnexpectedStdinErrorShellSession(_PtyShellSession):
async def pty_write_stdin(
self,
*,
session_id: int,
chars: str,
yield_time_s: float | None = None,
max_output_tokens: int | None = None,
) -> PtyExecUpdate:
_ = (session_id, chars, yield_time_s, max_output_tokens)
raise RuntimeError("unexpected stdin failure")
class _PtyTransportFailingShellSession(_OutputShellSession):
def __init__(
self,
manifest: Manifest,
*,
stdout: bytes = b"",
stderr: bytes = b"",
exit_code: int = 0,
transport_context: dict[str, object] | None = None,
) -> None:
super().__init__(manifest, stdout=stdout, stderr=stderr, exit_code=exit_code)
self.transport_context = transport_context or {}
self.exec_call_count = 0
def supports_pty(self) -> bool:
return True
async def exec(
self,
*command: str | Path,
timeout: float | None = None,
user: str | User | None = None,
shell: bool | list[str] = False,
) -> ExecResult:
self.exec_call_count += 1
return await super().exec(*command, timeout=timeout, user=user, shell=shell)
async def pty_exec_start(
self,
*command: str | Path,
timeout: float | None = None,
shell: bool | list[str] = True,
user: str | User | None = None,
tty: bool = False,
yield_time_s: float | None = None,
max_output_tokens: int | None = None,
) -> PtyExecUpdate:
_ = (timeout, shell, user, tty, yield_time_s, max_output_tokens)
raise ExecTransportError(
command=command,
context=self.transport_context,
cause=RuntimeError("connection closed while reading HTTP status line"),
)
def _transport_error(context: dict[str, object]) -> ExecTransportError:
return ExecTransportError(
command=("pwd",),
context=context,
cause=RuntimeError("connection closed while reading HTTP status line"),
)
def _patch_shell_tool_clock(
@@ -286,7 +101,7 @@ class TestShellCapability:
def test_tools_exposes_exec_command_function_tool_after_bind(self) -> None:
capability = Shell()
capability.bind(_ShellSession(Manifest(root="/workspace")))
capability.bind(_shell_session())
tools = capability.tools()
@@ -297,7 +112,7 @@ class TestShellCapability:
def test_tools_exposes_write_stdin_for_pty_sessions(self) -> None:
capability = Shell()
capability.bind(_PtyShellSession(Manifest(root="/workspace")))
capability.bind(_pty_session([{"method": "pty_write_stdin", "result": None}]))
tools = capability.tools()
@@ -307,6 +122,17 @@ class TestShellCapability:
assert tools[0].name == "exec_command"
assert tools[1].name == "write_stdin"
def test_tools_keep_both_pty_session_methods_callable(self) -> None:
capability = Shell()
session = _pty_session([{"method": "pty_exec_start", "result": None}])
capability.bind(session)
tools = capability.tools()
assert len(tools) == 2
assert hasattr(session, "pty_exec_start")
assert hasattr(session, "pty_write_stdin")
def test_configure_tools_can_customize_shell_approvals_after_clone(self) -> None:
async def exec_command_needs_approval(
_ctx: Any, params: dict[str, Any], _call_id: str
@@ -324,7 +150,7 @@ class TestShellCapability:
toolset.write_stdin.needs_approval = write_stdin_needs_approval
capability = Shell(configure_tools=configure_tools).clone()
capability.bind(_PtyShellSession(Manifest(root="/workspace")))
capability.bind(_pty_session([{"method": "pty_write_stdin", "result": None}]))
tools = capability.tools()
exec_command_tool = cast(ExecCommandTool, tools[0])
@@ -341,7 +167,7 @@ class TestShellCapability:
saw_missing_write_stdin = toolset.write_stdin is None
capability = Shell(configure_tools=configure_tools)
capability.bind(_ShellSession(Manifest(root="/workspace")))
capability.bind(_shell_session())
tools = capability.tools()
@@ -361,7 +187,7 @@ class TestShellCapability:
toolset.exec_command = replacement_exec_command
capability = Shell(configure_tools=configure_tools)
capability.bind(_ShellSession(Manifest(root="/workspace")))
capability.bind(_shell_session())
tools = capability.tools()
exec_command_tool = cast(ExecCommandTool, tools[0])
@@ -392,7 +218,7 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
capability = Shell()
session = _ShellSession(Manifest(root="/workspace"))
session = _shell_session()
capability.bind(session)
tool = cast(FunctionTool, capability.tools()[0])
@@ -412,7 +238,9 @@ class TestShellCapability:
ExecCommandArgs(cmd="pwd", yield_time_ms=1500).model_dump_json(),
)
assert session.exec_calls == [("pwd", 1.5, True)]
assert session.calls[0].args == ("pwd",)
assert session.calls[0].kwargs["timeout"] == 1.5
assert session.calls[0].kwargs["shell"] is True
assert (
output == "Chunk ID: 123456\n"
"Wall time: 0.2500 seconds\n"
@@ -425,7 +253,14 @@ class TestShellCapability:
@pytest.mark.asyncio
async def test_exec_command_tool_runs_as_bound_user(self) -> None:
capability = Shell()
session = _ShellSession(Manifest(root="/workspace"))
session = scripted_sandbox_session(
[
{
"method": "exec",
"result": ExecResult(stdout=b"", stderr=b"", exit_code=0),
}
]
)
capability.bind(session)
capability.bind_run_as(User(name="sandbox-user"))
tool = cast(FunctionTool, capability.tools()[0])
@@ -435,7 +270,8 @@ class TestShellCapability:
ExecCommandArgs(cmd="pwd").model_dump_json(),
)
assert session.exec_users == ["sandbox-user"]
assert session.calls[0].kwargs["user"] == User(name="sandbox-user")
session.assert_complete()
@pytest.mark.asyncio
async def test_exec_command_tool_includes_original_token_count_when_truncating(
@@ -443,7 +279,7 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
capability = Shell()
session = _ShellSession(Manifest(root="/workspace"))
session = _shell_session()
capability.bind(session)
tool = cast(FunctionTool, capability.tools()[0])
@@ -478,7 +314,7 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
capability = Shell()
session = _ShellSession(Manifest(root="/workspace"))
session = _shell_session()
capability.bind(session)
tool = cast(FunctionTool, capability.tools()[0])
_patch_shell_tool_clock(
@@ -498,9 +334,9 @@ class TestShellCapability:
).model_dump_json(),
)
assert session.exec_calls == [
("cd /workspace/src/project && pwd", 10.0, ["/bin/bash", "-c"])
]
assert session.calls[0].args == ("cd /workspace/src/project && pwd",)
assert session.calls[0].kwargs["timeout"] == 10.0
assert session.calls[0].kwargs["shell"] == ["/bin/bash", "-c"]
assert (
output == "Chunk ID: 876543\n"
"Wall time: 0.1250 seconds\n"
@@ -516,8 +352,8 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
capability = Shell()
session = _ShellSession(
Manifest(
session = _shell_session(
manifest=Manifest(
root="/workspace",
extra_path_grants=(
SandboxPathGrant(
@@ -547,7 +383,9 @@ class TestShellCapability:
).model_dump_json(),
)
assert session.exec_calls == [("cd /mnt/shared-data && pwd", 10.0, ["/bin/bash", "-c"])]
assert session.calls[0].args == ("cd /mnt/shared-data && pwd",)
assert session.calls[0].kwargs["timeout"] == 10.0
assert session.calls[0].kwargs["shell"] == ["/bin/bash", "-c"]
assert (
output == "Chunk ID: 111111\n"
"Wall time: 0.2500 seconds\n"
@@ -563,7 +401,19 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
capability = Shell()
session = _PtyShellSession(Manifest(root="/workspace"))
session = _pty_session(
[
{
"method": "pty_exec_start",
"result": PtyExecUpdate(
process_id=1337,
output=b"",
exit_code=None,
original_token_count=None,
),
}
]
)
capability.bind(session)
tool = cast(FunctionTool, capability.tools()[0])
_patch_shell_tool_clock(
@@ -578,7 +428,7 @@ class TestShellCapability:
ExecCommandArgs(cmd="pwd", yield_time_ms=0, tty=True).model_dump_json(),
)
assert session.last_exec_yield_time_s == 0.0
assert session.calls[0].kwargs["yield_time_s"] == 0.0
assert (
output == "Chunk ID: abcdef\n"
"Wall time: 0.0500 seconds\n"
@@ -590,7 +440,19 @@ class TestShellCapability:
@pytest.mark.asyncio
async def test_exec_command_tool_starts_pty_as_bound_user(self) -> None:
capability = Shell()
session = _PtyShellSession(Manifest(root="/workspace"))
session = _pty_session(
[
{
"method": "pty_exec_start",
"result": PtyExecUpdate(
process_id=1337,
output=b"",
exit_code=None,
original_token_count=None,
),
}
]
)
capability.bind(session)
capability.bind_run_as(User(name="sandbox-user"))
tool = cast(FunctionTool, capability.tools()[0])
@@ -600,7 +462,7 @@ class TestShellCapability:
ExecCommandArgs(cmd="pwd", yield_time_ms=0, tty=True).model_dump_json(),
)
assert session.last_exec_user == "sandbox-user"
assert session.calls[0].kwargs["user"] == User(name="sandbox-user")
@pytest.mark.asyncio
async def test_exec_command_tool_formats_timeout_without_exit_code(
@@ -608,7 +470,7 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
capability = Shell()
session = _TimeoutShellSession(Manifest(root="/workspace"))
session = _shell_session(error=ExecTimeoutError(command=("sleep 30",), timeout_s=0.005))
capability.bind(session)
tool = cast(FunctionTool, capability.tools()[0])
_patch_shell_tool_clock(
@@ -635,13 +497,19 @@ class TestShellCapability:
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
tool = ExecCommandTool(
session=_PtyTransportFailingShellSession(
Manifest(root="/workspace"),
stdout=b"fallback ok",
transport_context={"stage": "open_pipe", "retry_safe": True},
)
session = _pty_session(
[
{
"method": "pty_exec_start",
"error": _transport_error({"stage": "open_pipe", "retry_safe": True}),
},
{
"method": "exec",
"result": ExecResult(stdout=b"fallback ok", stderr=b"", exit_code=0),
},
]
)
tool = ExecCommandTool(session=session)
_patch_shell_tool_clock(
monkeypatch,
chunk_id="44444444444444444444444444444444",
@@ -661,12 +529,17 @@ class TestShellCapability:
@pytest.mark.asyncio
async def test_exec_command_tool_does_not_fall_back_for_tty_sessions(self) -> None:
tool = ExecCommandTool(
session=_PtyTransportFailingShellSession(
Manifest(root="/workspace"),
transport_context={"stage": "open_pipe", "retry_safe": True, "tty": True},
)
session = _pty_session(
[
{
"method": "pty_exec_start",
"error": _transport_error(
{"stage": "open_pipe", "retry_safe": True, "tty": True}
),
}
]
)
tool = ExecCommandTool(session=session)
with pytest.raises(ExecTransportError):
await tool.on_invoke_tool(
@@ -678,12 +551,15 @@ class TestShellCapability:
async def test_exec_command_tool_does_not_fall_back_for_non_retry_safe_transport_errors(
self,
) -> None:
tool = ExecCommandTool(
session=_PtyTransportFailingShellSession(
Manifest(root="/workspace"),
transport_context={"stage": "open_pipe"},
)
session = _pty_session(
[
{
"method": "pty_exec_start",
"error": _transport_error({"stage": "open_pipe"}),
}
]
)
tool = ExecCommandTool(session=session)
with pytest.raises(ExecTransportError):
await tool.on_invoke_tool(
@@ -697,10 +573,8 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
tool = ExecCommandTool(
session=_OutputShellSession(
Manifest(root="/workspace"),
stdout=b"stdout only\n",
stderr=b"",
session=_shell_session(
result=ExecResult(stdout=b"stdout only\n", stderr=b"", exit_code=7)
)
)
_patch_shell_tool_clock(
@@ -729,10 +603,8 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
tool = ExecCommandTool(
session=_OutputShellSession(
Manifest(root="/workspace"),
stdout=b"",
stderr=b"stderr only\n",
session=_shell_session(
result=ExecResult(stdout=b"", stderr=b"stderr only\n", exit_code=7)
)
)
_patch_shell_tool_clock(
@@ -761,10 +633,12 @@ class TestShellCapability:
monkeypatch: pytest.MonkeyPatch,
) -> None:
tool = ExecCommandTool(
session=_OutputShellSession(
Manifest(root="/workspace"),
stdout=b"stdout line\n",
stderr=b"stderr line\n",
session=_shell_session(
result=ExecResult(
stdout=b"stdout line\n",
stderr=b"stderr line\n",
exit_code=7,
)
)
)
_patch_shell_tool_clock(
@@ -793,8 +667,19 @@ class TestShellCapability:
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
session = _PtyShellSession(Manifest(root="/workspace"))
session._live_sessions.add(1337)
session = _pty_session(
[
{
"method": "pty_write_stdin",
"result": PtyExecUpdate(
process_id=None,
output=b"hello",
exit_code=0,
original_token_count=None,
),
}
]
)
tool = WriteStdinTool(session=session)
_patch_shell_tool_clock(
monkeypatch,
@@ -818,7 +703,7 @@ class TestShellCapability:
@pytest.mark.asyncio
async def test_write_stdin_tool_rejects_non_pty_sessions(self) -> None:
tool = WriteStdinTool(session=_ShellSession(Manifest(root="/workspace")))
tool = WriteStdinTool(session=_shell_session())
with pytest.raises(
RuntimeError, match="write_stdin is not available for non-PTY sandboxes"
@@ -833,7 +718,15 @@ class TestShellCapability:
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
tool = WriteStdinTool(session=_PtyShellSession(Manifest(root="/workspace")))
session = _pty_session(
[
{
"method": "pty_write_stdin",
"error": PtySessionNotFoundError(session_id=9999),
}
]
)
tool = WriteStdinTool(session=session)
_patch_shell_tool_clock(
monkeypatch,
chunk_id="66666666666666666666666666666666",
@@ -859,8 +752,14 @@ class TestShellCapability:
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
session = _PtyNoStdinShellSession(Manifest(root="/workspace"))
session._live_sessions.add(1337)
session = _pty_session(
[
{
"method": "pty_write_stdin",
"error": RuntimeError("stdin is not available for this process"),
}
]
)
tool = WriteStdinTool(session=session)
_patch_shell_tool_clock(
monkeypatch,
@@ -885,9 +784,15 @@ class TestShellCapability:
@pytest.mark.asyncio
async def test_write_stdin_tool_reraises_unexpected_runtime_error(self) -> None:
tool = WriteStdinTool(
session=_PtyUnexpectedStdinErrorShellSession(Manifest(root="/workspace"))
session = _pty_session(
[
{
"method": "pty_write_stdin",
"error": RuntimeError("unexpected stdin failure"),
}
]
)
tool = WriteStdinTool(session=session)
with pytest.raises(RuntimeError, match="unexpected stdin failure"):
await tool.on_invoke_tool(
@@ -21,6 +21,7 @@ from agents.sandbox.session.sandbox_session import SandboxSession
from agents.sandbox.snapshot import NoopSnapshot
from agents.sandbox.types import ExecResult, FileMode, Group, Permissions, User
from agents.sandbox.workspace_paths import coerce_posix_path, sandbox_path_str
from agents.testing import scripted_sandbox_session
from agents.tool import FunctionTool
from agents.tool_context import ToolContext
from agents.tracing import trace
@@ -644,7 +645,11 @@ class TestSkillsLazyLoading:
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
capability = Skills(lazy_from=LocalDirLazySkillSource(source=LocalDir(src=src_root)))
capability.bind(_SkillsSession(_source_granted_manifest(workspace_root, source=src_root)))
capability.bind(
scripted_sandbox_session(
manifest=_source_granted_manifest(workspace_root, source=src_root)
)
)
tools = capability.tools()
@@ -785,8 +790,10 @@ class TestSkillsLazyLoading:
lazy_from=LocalDirLazySkillSource(source=LocalDir(src=tmp_path / "missing-skills"))
)
capability.bind(
_SkillsSession(
_source_granted_manifest(workspace_root, source=tmp_path / "missing-skills")
scripted_sandbox_session(
manifest=_source_granted_manifest(
workspace_root, source=tmp_path / "missing-skills"
)
)
)
@@ -811,7 +818,11 @@ class TestSkillsLazyLoading:
encoding="utf-8",
)
capability = Skills(lazy_from=LocalDirLazySkillSource(source=LocalDir(src=src_root)))
capability.bind(_SkillsSession(_source_granted_manifest(workspace_root, source=src_root)))
capability.bind(
scripted_sandbox_session(
manifest=_source_granted_manifest(workspace_root, source=src_root)
)
)
with pytest.raises(SkillsConfigError):
await capability.load_skill("shared-skill")
@@ -840,7 +851,11 @@ class TestSkillsLazyLoading:
second_instructions = await capability.instructions(
_source_granted_manifest(workspace_root, source=src_root)
)
capability.bind(_SkillsSession(_source_granted_manifest(workspace_root, source=src_root)))
capability.bind(
scripted_sandbox_session(
manifest=_source_granted_manifest(workspace_root, source=src_root)
)
)
third_instructions = await capability.instructions(
_source_granted_manifest(workspace_root, source=src_root)
)
@@ -2,7 +2,6 @@ from __future__ import annotations
import base64
import io
import uuid
from pathlib import Path
from typing import cast
@@ -11,12 +10,10 @@ import pytest
from agents.sandbox import Manifest
from agents.sandbox.capabilities.tools import ViewImageTool
from agents.sandbox.errors import WorkspaceReadNotFoundError
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
from agents.sandbox.snapshot import NoopSnapshot
from agents.sandbox.types import ExecResult, User
from agents.sandbox.types import User
from agents.testing import scripted_sandbox_session
from agents.tool import ToolOutputImage
from agents.tool_context import ToolContext
from tests.utils.factories import TestSessionState
_MAX_IMAGE_BYTES = 10 * 1024 * 1024
_PNG_BASE64 = (
@@ -25,76 +22,9 @@ _PNG_BASE64 = (
_PNG_BYTES = base64.b64decode(_PNG_BASE64)
class _ImageSession(BaseSandboxSession):
def __init__(self, manifest: Manifest) -> None:
self.state = TestSessionState(
manifest=manifest,
snapshot=NoopSnapshot(id=str(uuid.uuid4())),
)
self.files: dict[Path, bytes] = {}
self.read_users: list[str | None] = []
async def start(self) -> None:
return None
async def stop(self) -> None:
return None
async def shutdown(self) -> None:
return None
async def running(self) -> bool:
return True
async def read(self, path: Path, *, user: str | User | None = None) -> io.BytesIO:
self.read_users.append(user.name if isinstance(user, User) else user)
normalized = self.normalize_path(path)
if normalized not in self.files:
raise FileNotFoundError(normalized)
return io.BytesIO(self.files[normalized])
async def write(
self,
path: Path,
data: io.IOBase,
*,
user: str | User | None = None,
) -> None:
_ = user
normalized = self.normalize_path(path)
payload = data.read()
if isinstance(payload, str):
self.files[normalized] = payload.encode("utf-8")
else:
self.files[normalized] = bytes(payload)
async def _exec_internal(
self,
*command: str | Path,
timeout: float | None = None,
) -> ExecResult:
_ = (command, timeout)
raise AssertionError("_exec_internal() should not be called")
async def persist_workspace(self) -> io.IOBase:
return io.BytesIO()
async def hydrate_workspace(self, data: io.IOBase) -> None:
_ = data
class _ProviderNotFoundImageSession(_ImageSession):
async def read(self, path: Path, *, user: str | User | None = None) -> io.BytesIO:
self.read_users.append(user.name if isinstance(user, User) else user)
normalized = self.normalize_path(path)
if normalized in self.files:
return io.BytesIO(self.files[normalized])
raise WorkspaceReadNotFoundError(path=normalized)
class TestViewImageTool:
def test_view_image_accepts_needs_approval_setting(self) -> None:
session = _ImageSession(Manifest(root="/workspace"))
session = scripted_sandbox_session()
async def needs_approval(_ctx: object, params: dict[str, object], _call_id: str) -> bool:
return str(params["path"]).startswith("sensitive/")
@@ -105,8 +35,7 @@ class TestViewImageTool:
@pytest.mark.asyncio
async def test_view_image_returns_tool_output_image_for_png(self) -> None:
session = _ImageSession(Manifest(root="/workspace"))
session.files[Path("/workspace/images/dot.png")] = _PNG_BYTES
session = scripted_sandbox_session([{"method": "read", "result": io.BytesIO(_PNG_BYTES)}])
tool = ViewImageTool(session=session)
output = await tool.on_invoke_tool(
@@ -117,11 +46,11 @@ class TestViewImageTool:
assert isinstance(output, ToolOutputImage)
assert output.image_url == f"data:image/png;base64,{_PNG_BASE64}"
assert output.detail is None
session.assert_complete()
@pytest.mark.asyncio
async def test_view_image_reads_as_bound_user(self) -> None:
session = _ImageSession(Manifest(root="/workspace"))
session.files[Path("/workspace/images/dot.png")] = _PNG_BYTES
session = scripted_sandbox_session([{"method": "read", "result": io.BytesIO(_PNG_BYTES)}])
tool = ViewImageTool(session=session, user=User(name="sandbox-user"))
output = await tool.on_invoke_tool(
@@ -130,12 +59,12 @@ class TestViewImageTool:
)
assert isinstance(output, ToolOutputImage)
assert session.read_users == ["sandbox-user"]
assert session.calls[0].kwargs["user"] == User(name="sandbox-user")
session.assert_complete()
@pytest.mark.asyncio
async def test_view_image_rejects_non_image_files(self) -> None:
session = _ImageSession(Manifest(root="/workspace"))
session.files[Path("/workspace/notes.txt")] = b"hello\n"
session = scripted_sandbox_session([{"method": "read", "result": io.BytesIO(b"hello\n")}])
tool = ViewImageTool(session=session)
output = await tool.on_invoke_tool(
@@ -144,12 +73,17 @@ class TestViewImageTool:
)
assert output == "image path `notes.txt` is not a supported image file"
session.assert_complete()
@pytest.mark.asyncio
async def test_view_image_rejects_images_larger_than_10mb(self) -> None:
session = _ImageSession(Manifest(root="/workspace"))
session.files[Path("/workspace/images/huge.png")] = b"\x89PNG\r\n\x1a\n" + (
b"0" * (_MAX_IMAGE_BYTES + 1)
session = scripted_sandbox_session(
[
{
"method": "read",
"result": io.BytesIO(b"\x89PNG\r\n\x1a\n" + (b"0" * (_MAX_IMAGE_BYTES + 1))),
}
]
)
tool = ViewImageTool(session=session)
@@ -162,14 +96,24 @@ class TestViewImageTool:
"image path `images/huge.png` exceeded the allowed size of 10MB; "
"resize or compress the image and try again"
)
session.assert_complete()
@pytest.mark.asyncio
async def test_view_image_rejection_text_does_not_expose_provider_path(self) -> None:
provider_root = Path("/provider/private/root")
session = _ProviderNotFoundImageSession(Manifest(root=str(provider_root)))
session.files[provider_root / "notes.txt"] = b"hello\n"
session.files[provider_root / "images/huge.png"] = b"\x89PNG\r\n\x1a\n" + (
b"0" * (_MAX_IMAGE_BYTES + 1)
session = scripted_sandbox_session(
[
{
"method": "read",
"error": WorkspaceReadNotFoundError(path=provider_root / "images/missing.png"),
},
{"method": "read", "result": io.BytesIO(b"hello\n")},
{
"method": "read",
"result": io.BytesIO(b"\x89PNG\r\n\x1a\n" + (b"0" * (_MAX_IMAGE_BYTES + 1))),
},
],
manifest=Manifest(root=str(provider_root)),
)
tool = ViewImageTool(session=session)
@@ -5,19 +5,19 @@ from collections.abc import Mapping, Sequence
from typing import Any
from agents.items import TResponseOutputItem
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_final_output_message, get_function_tool_call
__test__ = False
class TestModel(FakeModel):
class TestModel(ScriptedModel):
"""Reusable queued model for sandbox integration tests."""
__test__ = False
def queue_turn(self, *items: TResponseOutputItem) -> None:
self.set_next_output(list(items))
self.enqueue(list(items))
def queue_function_call(
self,
+144 -123
View File
@@ -75,7 +75,7 @@ from agents.sandbox.memory.storage import (
)
from agents.sandbox.runtime import _stream_memory_input_override
from agents.sandbox.sandboxes.unix_local import UnixLocalSandboxClient
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_final_output_message, get_text_message
from tests.utils.hitl import make_shell_call
@@ -148,8 +148,8 @@ def _memory_config(
extra_prompt: str | None = None,
layout: MemoryLayoutConfig | None = None,
read: MemoryReadConfig | None = None,
phase_one_model: FakeModel | None = None,
phase_two_model: FakeModel | None = None,
phase_one_model: ScriptedModel | None = None,
phase_two_model: ScriptedModel | None = None,
) -> Memory:
return Memory(
layout=layout or MemoryLayoutConfig(),
@@ -157,14 +157,16 @@ def _memory_config(
generate=MemoryGenerateConfig(
max_raw_memories_for_consolidation=max_raw_memories_for_consolidation,
extra_prompt=extra_prompt,
phase_one_model=phase_one_model or FakeModel(initial_output=[_phase_one_message()]),
phase_one_model=phase_one_model or ScriptedModel(steps=[[_phase_one_message()]]),
phase_two_model=phase_two_model
or FakeModel(
initial_output=[
_patch_update_call("memory-md", "memories/MEMORY.md", "memory entry"),
_patch_update_call(
"memory-summary", "memories/memory_summary.md", "summary entry"
),
or ScriptedModel(
steps=[
[
_patch_update_call("memory-md", "memories/MEMORY.md", "memory entry"),
_patch_update_call(
"memory-summary", "memories/memory_summary.md", "summary entry"
),
]
]
),
),
@@ -175,13 +177,12 @@ def _run_config_for_session(session: Any) -> RunConfig:
return RunConfig(sandbox=SandboxRunConfig(session=session))
def _extract_user_text(fake_model: FakeModel) -> str:
assert fake_model.first_turn_args is not None
return _extract_user_text_from_turn_args(fake_model.first_turn_args)
def _extract_user_text(scripted_model: ScriptedModel) -> str:
assert bool(scripted_model.calls)
return _extract_user_text_from_model_input(scripted_model.calls[0].input)
def _extract_user_text_from_turn_args(turn_args: dict[str, Any]) -> str:
input_items = turn_args["input"]
def _extract_user_text_from_model_input(input_items: str | list[Any]) -> str:
assert isinstance(input_items, list)
first_item = cast(dict[str, Any], input_items[0])
content = first_item["content"]
@@ -649,23 +650,25 @@ async def test_runner_memory_generation_sanitizes_and_truncates_phase_one_prompt
monkeypatch.setattr(phase_one_module, "_PHASE_ONE_ROLLOUT_TOKEN_LIMIT", 1000)
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
phase_one_model = FakeModel(initial_output=[_phase_one_message()])
phase_one_model = ScriptedModel(steps=[[_phase_one_message()]])
memory = _memory_config(phase_one_model=phase_one_model)
agent = SandboxAgent(
name="worker",
model=FakeModel(
initial_output=[
ResponseReasoningItem(id="rs_1", summary=[], type="reasoning"),
cast(
TResponseOutputItem,
{
"id": "compaction_1",
"type": "compaction",
"summary": "compacted-so-far",
"encrypted_content": "encrypted",
},
),
get_text_message("done"),
model=ScriptedModel(
steps=[
[
ResponseReasoningItem(id="rs_1", summary=[], type="reasoning"),
cast(
TResponseOutputItem,
{
"id": "compaction_1",
"type": "compaction",
"summary": "compacted-so-far",
"encrypted_content": "encrypted",
},
),
get_text_message("done"),
]
]
),
instructions="Worker.",
@@ -694,7 +697,7 @@ async def test_runner_memory_generation_sanitizes_and_truncates_phase_one_prompt
)
assert result.final_output == "done"
assert phase_one_model.first_turn_args is None
assert not phase_one_model.calls
await session.aclose()
closed = True
@@ -721,7 +724,7 @@ async def test_sandbox_agent_without_memory_capability_skips_memory_generation()
session = await client.create(manifest=Manifest())
agent = SandboxAgent(
name="worker",
model=FakeModel(initial_output=[get_final_output_message("done")]),
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Worker.",
)
@@ -990,14 +993,16 @@ async def test_memory_capability_live_update_instructions() -> None:
async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
phase_one_model = FakeModel(initial_output=[_phase_one_message()])
phase_two_model = FakeModel(
initial_output=[
_patch_update_call("memory-md", "memories/MEMORY.md", "memory entry"),
_patch_update_call("memory-summary", "memories/memory_summary.md", "summary entry"),
phase_one_model = ScriptedModel(steps=[[_phase_one_message()]])
phase_two_model = ScriptedModel(
steps=[
[
_patch_update_call("memory-md", "memories/MEMORY.md", "memory entry"),
_patch_update_call("memory-summary", "memories/memory_summary.md", "summary entry"),
]
]
)
phase_two_model.set_next_output([get_final_output_message("consolidated")])
phase_two_model.enqueue([get_final_output_message("consolidated")])
memory = _memory_config(
extra_prompt="Track durable user preferences.",
phase_one_model=phase_one_model,
@@ -1005,7 +1010,7 @@ async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
)
agent = SandboxAgent(
name="worker",
model=FakeModel(initial_output=[get_final_output_message("done")]),
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Worker.",
capabilities=[memory],
)
@@ -1023,7 +1028,7 @@ async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
assert result.final_output == "done"
assert len(rollouts) == 1
assert phase_one_model.first_turn_args is None
assert not phase_one_model.calls
await session.aclose()
closed = True
@@ -1048,16 +1053,12 @@ async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
assert "rollout_path: sessions/" in rollout_summaries[0].read_text()
assert "terminal_state: completed" in rollout_summaries[0].read_text()
assert '"terminal_state":"completed"' in _extract_user_text(phase_one_model)
assert phase_one_model.first_turn_args is not None
assert (
"DEVELOPER-SPECIFIC EXTRA GUIDANCE"
in phase_one_model.first_turn_args["system_instructions"]
)
assert (
"Track durable user preferences."
in phase_one_model.first_turn_args["system_instructions"]
)
assert phase_two_model.first_turn_args is not None
assert bool(phase_one_model.calls)
system_instructions = phase_one_model.calls[0].system_instructions
assert system_instructions is not None
assert "DEVELOPER-SPECIFIC EXTRA GUIDANCE" in system_instructions
assert "Track durable user preferences." in system_instructions
assert bool(phase_two_model.calls)
assert "DEVELOPER-SPECIFIC EXTRA GUIDANCE" in _extract_user_text(phase_two_model)
assert "Track durable user preferences." in _extract_user_text(phase_two_model)
finally:
@@ -1068,24 +1069,28 @@ async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
async def test_sandbox_memory_uses_custom_layout() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
phase_two_model = FakeModel(
initial_output=[
_patch_update_call("memory-md", "agent_memory/MEMORY.md", "memory entry"),
_patch_update_call("memory-summary", "agent_memory/memory_summary.md", "summary entry"),
phase_two_model = ScriptedModel(
steps=[
[
_patch_update_call("memory-md", "agent_memory/MEMORY.md", "memory entry"),
_patch_update_call(
"memory-summary", "agent_memory/memory_summary.md", "summary entry"
),
]
]
)
phase_two_model.set_next_output([get_final_output_message("consolidated")])
phase_two_model.enqueue([get_final_output_message("consolidated")])
memory = Memory(
layout=MemoryLayoutConfig(memories_dir="agent_memory", sessions_dir="agent_sessions"),
read=None,
generate=MemoryGenerateConfig(
phase_one_model=FakeModel(initial_output=[_phase_one_message()]),
phase_one_model=ScriptedModel(steps=[[_phase_one_message()]]),
phase_two_model=phase_two_model,
),
)
agent = SandboxAgent(
name="worker",
model=FakeModel(initial_output=[get_final_output_message("done")]),
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Worker.",
capabilities=[memory],
)
@@ -1114,47 +1119,51 @@ async def test_sandbox_memory_uses_custom_layout() -> None:
async def test_sandbox_memory_supports_multiple_generating_layouts_in_one_session() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
phase_two_model_a = FakeModel(
initial_output=[
_patch_update_call("a-memory", "agent_a_memory/MEMORY.md", "agent a entry"),
_patch_update_call(
"a-summary",
"agent_a_memory/memory_summary.md",
"agent a summary",
),
phase_two_model_a = ScriptedModel(
steps=[
[
_patch_update_call("a-memory", "agent_a_memory/MEMORY.md", "agent a entry"),
_patch_update_call(
"a-summary",
"agent_a_memory/memory_summary.md",
"agent a summary",
),
]
]
)
phase_two_model_a.set_next_output([get_final_output_message("agent a consolidated")])
phase_two_model_b = FakeModel(
initial_output=[
_patch_update_call("b-memory", "agent_b_memory/MEMORY.md", "agent b entry"),
_patch_update_call(
"b-summary",
"agent_b_memory/memory_summary.md",
"agent b summary",
),
phase_two_model_a.enqueue([get_final_output_message("agent a consolidated")])
phase_two_model_b = ScriptedModel(
steps=[
[
_patch_update_call("b-memory", "agent_b_memory/MEMORY.md", "agent b entry"),
_patch_update_call(
"b-summary",
"agent_b_memory/memory_summary.md",
"agent b summary",
),
]
]
)
phase_two_model_b.set_next_output([get_final_output_message("agent b consolidated")])
phase_two_model_b.enqueue([get_final_output_message("agent b consolidated")])
memory_a = _memory_config(
layout=MemoryLayoutConfig(memories_dir="agent_a_memory", sessions_dir="agent_a_sessions"),
phase_one_model=FakeModel(initial_output=[_phase_one_message(raw_memory="agent a raw\n")]),
phase_one_model=ScriptedModel(steps=[[_phase_one_message(raw_memory="agent a raw\n")]]),
phase_two_model=phase_two_model_a,
)
memory_b = _memory_config(
layout=MemoryLayoutConfig(memories_dir="agent_b_memory", sessions_dir="agent_b_sessions"),
phase_one_model=FakeModel(initial_output=[_phase_one_message(raw_memory="agent b raw\n")]),
phase_one_model=ScriptedModel(steps=[[_phase_one_message(raw_memory="agent b raw\n")]]),
phase_two_model=phase_two_model_b,
)
agent_a = SandboxAgent(
name="agent-a",
model=FakeModel(initial_output=[get_final_output_message("a done")]),
model=ScriptedModel(steps=[[get_final_output_message("a done")]]),
instructions="Agent A.",
capabilities=[memory_a],
)
agent_b = SandboxAgent(
name="agent-b",
model=FakeModel(initial_output=[get_final_output_message("b done")]),
model=ScriptedModel(steps=[[get_final_output_message("b done")]]),
instructions="Agent B.",
capabilities=[memory_b],
)
@@ -1183,7 +1192,7 @@ async def test_sandbox_memory_rejects_different_generate_configs_for_same_layout
session = await client.create(manifest=Manifest())
memory = _memory_config()
different_memory = _memory_config(
phase_one_model=FakeModel(initial_output=[_phase_one_message(raw_memory="different\n")])
phase_one_model=ScriptedModel(steps=[[_phase_one_message(raw_memory="different\n")]])
)
try:
@@ -1266,27 +1275,31 @@ async def test_sandbox_memory_rejects_shared_sessions_dir_for_different_memories
async def test_sandbox_memory_groups_segments_by_sdk_session_until_close() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
phase_one_model = FakeModel(initial_output=[_phase_one_message(raw_memory="joined raw\n")])
phase_two_model = FakeModel(
initial_output=[
_patch_update_call("memory-md", "memories/MEMORY.md", "joined entry"),
_patch_update_call("memory-summary", "memories/memory_summary.md", "joined summary"),
phase_one_model = ScriptedModel(steps=[[_phase_one_message(raw_memory="joined raw\n")]])
phase_two_model = ScriptedModel(
steps=[
[
_patch_update_call("memory-md", "memories/MEMORY.md", "joined entry"),
_patch_update_call(
"memory-summary", "memories/memory_summary.md", "joined summary"
),
]
]
)
phase_two_model.set_next_output([get_final_output_message("joined")])
phase_two_model.enqueue([get_final_output_message("joined")])
memory = _memory_config(
phase_one_model=phase_one_model,
phase_two_model=phase_two_model,
)
first_agent = SandboxAgent(
name="first-worker",
model=FakeModel(initial_output=[get_final_output_message("first done")]),
model=ScriptedModel(steps=[[get_final_output_message("first done")]]),
instructions="Worker.",
capabilities=[memory],
)
second_agent = SandboxAgent(
name="second-worker",
model=FakeModel(initial_output=[get_final_output_message("second done")]),
model=ScriptedModel(steps=[[get_final_output_message("second done")]]),
instructions="Worker.",
capabilities=[memory],
)
@@ -1324,7 +1337,7 @@ async def test_sandbox_memory_groups_segments_by_sdk_session_until_close() -> No
]
assert segments[0]["input"] == [{"content": "first", "role": "user"}]
assert segments[1]["input"] == [{"content": "second", "role": "user"}]
assert phase_one_model.first_turn_args is None
assert not phase_one_model.calls
await session.aclose()
closed = True
@@ -1345,8 +1358,8 @@ async def test_sandbox_memory_groups_segments_by_sdk_session_until_close() -> No
async def test_sandbox_memory_fallback_does_not_mutate_run_config() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
agent_model = FakeModel()
agent_model.add_multiple_turn_outputs(
agent_model = ScriptedModel()
agent_model.extend(
[
[get_final_output_message("first done")],
[get_final_output_message("second done")],
@@ -1387,7 +1400,7 @@ async def test_sandbox_memory_uses_conversation_id_when_sdk_session_is_absent()
session = await client.create(manifest=Manifest())
agent = SandboxAgent(
name="worker",
model=FakeModel(initial_output=[get_final_output_message("done")]),
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Worker.",
capabilities=[_memory_config()],
)
@@ -1413,8 +1426,8 @@ async def test_sandbox_memory_uses_conversation_id_when_sdk_session_is_absent()
async def test_sandbox_memory_uses_group_id_when_sdk_session_is_absent() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
agent_model = FakeModel()
agent_model.add_multiple_turn_outputs(
agent_model = ScriptedModel()
agent_model.extend(
[
[get_final_output_message("first done")],
[get_final_output_message("second done")],
@@ -1450,8 +1463,8 @@ async def test_sandbox_memory_uses_group_id_when_sdk_session_is_absent() -> None
async def test_sandbox_memory_uses_per_run_conversation_when_no_conversation_id() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
agent_model = FakeModel()
agent_model.add_multiple_turn_outputs(
agent_model = ScriptedModel()
agent_model.extend(
[
[get_final_output_message("first done")],
[get_final_output_message("second done")],
@@ -1483,27 +1496,29 @@ async def test_sandbox_memory_uses_per_run_conversation_when_no_conversation_id(
async def test_sandbox_memory_caps_phase_two_selection_and_surfaces_removed_rollouts() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
phase_one_model = FakeModel()
phase_one_model.add_multiple_turn_outputs(
phase_one_model = ScriptedModel()
phase_one_model.extend(
[
[_phase_one_message(slug="first", raw_memory="first raw\n")],
[_phase_one_message(slug="second", raw_memory="second raw\n")],
]
)
phase_two_model = FakeModel(
initial_output=[
_patch_update_call("memory-md", "memories/MEMORY.md", "first entry"),
_patch_update_call("memory-summary", "memories/memory_summary.md", "first summary"),
phase_two_model = ScriptedModel(
steps=[
[
_patch_update_call("memory-md", "memories/MEMORY.md", "first entry"),
_patch_update_call("memory-summary", "memories/memory_summary.md", "first summary"),
]
]
)
phase_two_model.set_next_output([get_final_output_message("consolidated")])
phase_two_model.enqueue([get_final_output_message("consolidated")])
memory = _memory_config(
max_raw_memories_for_consolidation=1,
phase_one_model=phase_one_model,
phase_two_model=phase_two_model,
)
agent_model = FakeModel()
agent_model.add_multiple_turn_outputs(
agent_model = ScriptedModel()
agent_model.extend(
[
[get_final_output_message("first done")],
[get_final_output_message("second done")],
@@ -1551,8 +1566,8 @@ async def test_sandbox_memory_caps_phase_two_selection_and_surfaces_removed_roll
assert "second raw" in merged_raw_memories
assert "first raw" not in merged_raw_memories
assert phase_two_model.first_turn_args is not None
prompt = _extract_user_text_from_turn_args(phase_two_model.first_turn_args)
assert bool(phase_two_model.calls)
prompt = _extract_user_text_from_model_input(phase_two_model.calls[0].input)
assert "newly added since the last successful Phase 2 run: 1" in prompt
assert f"rollout_id={selected_rollout_ids[0]}" in prompt
finally:
@@ -1563,21 +1578,25 @@ async def test_sandbox_memory_caps_phase_two_selection_and_surfaces_removed_roll
async def test_sandbox_memory_runs_phase_one_and_phase_two_on_session_close() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
phase_one_model = FakeModel(initial_output=[_phase_one_message()])
phase_two_model = FakeModel(
initial_output=[
_patch_update_call("memory-md", "memories/MEMORY.md", "shutdown entry"),
_patch_update_call("memory-summary", "memories/memory_summary.md", "shutdown summary"),
phase_one_model = ScriptedModel(steps=[[_phase_one_message()]])
phase_two_model = ScriptedModel(
steps=[
[
_patch_update_call("memory-md", "memories/MEMORY.md", "shutdown entry"),
_patch_update_call(
"memory-summary", "memories/memory_summary.md", "shutdown summary"
),
]
]
)
phase_two_model.set_next_output([get_final_output_message("shutdown")])
phase_two_model.enqueue([get_final_output_message("shutdown")])
memory = _memory_config(
phase_one_model=phase_one_model,
phase_two_model=phase_two_model,
)
agent = SandboxAgent(
name="worker",
model=FakeModel(initial_output=[get_final_output_message("done")]),
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Worker.",
capabilities=[memory],
)
@@ -1752,7 +1771,7 @@ async def test_sandbox_memory_enqueue_failure_follows_both_data_policies(
client = _DeleteTrackingUnixLocalSandboxClient()
agent = SandboxAgent(
name="worker",
model=FakeModel(initial_output=[get_final_output_message("done")]),
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
instructions="Worker.",
capabilities=[_memory_config()],
)
@@ -1796,23 +1815,25 @@ async def test_sandbox_memory_enqueue_failure_follows_both_data_policies(
async def test_sandbox_memory_marks_interrupted_runs_in_phase_one_prompt() -> None:
client = UnixLocalSandboxClient()
session = await client.create(manifest=Manifest())
phase_one_model = FakeModel(initial_output=[_phase_one_message()])
phase_two_model = FakeModel(
initial_output=[
_patch_update_call("memory-md", "memories/MEMORY.md", "interrupted entry"),
_patch_update_call(
"memory-summary", "memories/memory_summary.md", "interrupted summary"
),
phase_one_model = ScriptedModel(steps=[[_phase_one_message()]])
phase_two_model = ScriptedModel(
steps=[
[
_patch_update_call("memory-md", "memories/MEMORY.md", "interrupted entry"),
_patch_update_call(
"memory-summary", "memories/memory_summary.md", "interrupted summary"
),
]
]
)
phase_two_model.set_next_output([get_final_output_message("done")])
phase_two_model.enqueue([get_final_output_message("done")])
memory = _memory_config(
phase_one_model=phase_one_model,
phase_two_model=phase_two_model,
)
agent = SandboxAgent(
name="worker",
model=FakeModel(initial_output=[make_shell_call("approval-call")]),
model=ScriptedModel(steps=[[make_shell_call("approval-call")]]),
instructions="Worker.",
tools=[ShellTool(executor=lambda _request: "ok", needs_approval=True)],
capabilities=[memory],
File diff suppressed because it is too large Load Diff
+11 -12
View File
@@ -3,7 +3,6 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable, Coroutine
from pathlib import Path
from types import SimpleNamespace
from typing import Any, cast
import pytest
@@ -16,8 +15,8 @@ from agents.sandbox.capabilities import Capability, Compaction, Memory
from agents.sandbox.entries import BaseEntry, File
from agents.sandbox.manifest import Manifest
from agents.sandbox.sandbox_agent import SandboxAgent
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
from agents.sandbox.types import User
from agents.testing import scripted_sandbox_session
def test_sandbox_agent_normalizes_first_party_dictionary_configuration() -> None:
@@ -84,8 +83,8 @@ class _Capability:
return self.fragment
def _session_with_manifest(manifest: Manifest | None) -> object:
return SimpleNamespace(state=SimpleNamespace(manifest=manifest))
def _session_with_manifest(manifest: Manifest | None):
return scripted_sandbox_session(manifest=manifest)
def test_prepare_sandbox_agent_passes_session_manifest_to_capability_instructions():
@@ -97,7 +96,7 @@ def test_prepare_sandbox_agent_passes_session_manifest_to_capability_instruction
base_instructions="base instructions",
instructions="additional instructions",
),
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
session=_session_with_manifest(manifest),
capabilities=cast(list[Capability], [capability]),
)
instructions = cast(
@@ -134,7 +133,7 @@ def test_prepare_sandbox_agent_wraps_capabilities_without_agent_instructions():
name="sandbox",
base_instructions="base instructions",
),
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
session=_session_with_manifest(manifest),
capabilities=cast(list[Capability], [capability]),
)
instructions = cast(
@@ -170,7 +169,7 @@ def test_prepare_sandbox_agent_passes_default_model_to_capability_sampling_param
name="sandbox",
instructions="base instructions",
),
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
session=_session_with_manifest(manifest),
capabilities=cast(list[Capability], [capability]),
)
@@ -185,7 +184,7 @@ def test_prepare_sandbox_agent_prepares_default_compaction_policy() -> None:
name="sandbox",
instructions="base instructions",
),
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
session=_session_with_manifest(manifest),
capabilities=[Compaction()],
)
@@ -203,7 +202,7 @@ def test_prepare_sandbox_agent_uses_default_sandbox_instructions_when_base_missi
name="sandbox",
instructions="additional instructions",
),
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
session=_session_with_manifest(manifest),
capabilities=cast(list[Capability], [capability]),
)
instructions = cast(
@@ -259,7 +258,7 @@ def test_prepare_sandbox_agent_validates_required_capabilities() -> None:
instructions="base instructions",
capabilities=[Memory()],
),
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
session=_session_with_manifest(manifest),
capabilities=[Memory()],
)
@@ -270,7 +269,7 @@ def test_prepare_sandbox_agent_validates_required_capabilities() -> None:
instructions="base instructions",
capabilities=[Memory(read=MemoryReadConfig(live_update=False), generate=None)],
),
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
session=_session_with_manifest(manifest),
capabilities=[Memory(read=MemoryReadConfig(live_update=False), generate=None)],
)
@@ -280,7 +279,7 @@ def test_prepare_sandbox_agent_validates_required_capabilities() -> None:
instructions="base instructions",
capabilities=[Memory()],
),
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
session=_session_with_manifest(manifest),
capabilities=cast(
list[Capability],
[
+69 -63
View File
@@ -47,8 +47,8 @@ from agents.agent_tool_state import (
from agents.run_context import _ApprovalRecord
from agents.run_state import _build_agent_map
from agents.stream_events import AgentUpdatedStreamEvent, RawResponsesStreamEvent
from agents.testing import ScriptedModel
from agents.tool_context import ToolContext
from tests.fake_model import FakeModel
from tests.mcp.helpers import FakeMCPServer
from tests.mcp.model_compat import create_mcp_error
from tests.test_responses import get_function_tool_call, get_text_message
@@ -1790,14 +1790,14 @@ async def test_agent_as_tool_resume_survives_cancellation_after_nested_output_co
nested_model_waiting = asyncio.Event()
keep_nested_model_waiting = asyncio.Event()
class BlockingSecondModel(FakeModel):
class BlockingSecondModel(ScriptedModel):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.calls = 0
self.response_calls = 0
async def get_response(self, *args: Any, **kwargs: Any) -> ModelResponse:
self.calls += 1
if self.calls == 2:
self.response_calls += 1
if self.response_calls == 2:
nested_model_waiting.set()
await keep_nested_model_waiting.wait()
return await super().get_response(*args, **kwargs)
@@ -1808,24 +1808,26 @@ async def test_agent_as_tool_resume_survives_cancellation_after_nested_output_co
return "inner value"
inner_model = BlockingSecondModel(
initial_output=[get_function_tool_call("sensitive", "{}", call_id="inner_call")]
steps=[[get_function_tool_call("sensitive", "{}", call_id="inner_call")]]
)
inner_model.set_next_output([get_text_message("inner done")])
inner_model.enqueue([get_text_message("inner done")])
inner_agent = Agent(name="inner", model=inner_model, tools=[sensitive])
nested_tool = inner_agent.as_tool(
tool_name="delegate",
tool_description="Delegate",
)
outer_model = FakeModel(
initial_output=[
get_function_tool_call(
"delegate",
'{"input":"hi"}',
call_id="outer_call",
)
outer_model = ScriptedModel(
steps=[
[
get_function_tool_call(
"delegate",
'{"input":"hi"}',
call_id="outer_call",
)
]
]
)
outer_model.set_next_output([get_text_message("outer done")])
outer_model.enqueue([get_text_message("outer done")])
outer_agent = Agent(name="outer", model=outer_model, tools=[nested_tool])
async def run_outer(input_value: Any) -> RunResult | RunResultStreaming:
@@ -1843,7 +1845,7 @@ async def test_agent_as_tool_resume_survives_cancellation_after_nested_output_co
resume_task = asyncio.create_task(run_outer(state))
await nested_model_waiting.wait()
assert tool_attempts == ["ran"]
assert inner_model.calls == 2
assert inner_model.response_calls == 2
resume_task.cancel()
with pytest.raises(asyncio.CancelledError):
@@ -1853,7 +1855,7 @@ async def test_agent_as_tool_resume_survives_cancellation_after_nested_output_co
assert result.final_output == "outer done"
assert tool_attempts == ["ran"]
assert inner_model.calls == 3
assert inner_model.response_calls == 3
@pytest.mark.parametrize(
@@ -2515,28 +2517,30 @@ async def test_agent_as_tool_streaming_works_with_custom_extractor(
async def test_agent_as_tool_streaming_settles_multi_segment_text_output() -> None:
agent = Agent(
name="streamer",
model=FakeModel(
initial_output=[
ResponseOutputMessage(
id="msg_multi_segment",
role="assistant",
status="completed",
type="message",
content=[
ResponseOutputText(
annotations=[],
text="first ",
type="output_text",
logprobs=[],
),
ResponseOutputText(
annotations=[],
text="second",
type="output_text",
logprobs=[],
),
],
)
model=ScriptedModel(
steps=[
[
ResponseOutputMessage(
id="msg_multi_segment",
role="assistant",
status="completed",
type="message",
content=[
ResponseOutputText(
annotations=[],
text="first ",
type="output_text",
logprobs=[],
),
ResponseOutputText(
annotations=[],
text="second",
type="output_text",
logprobs=[],
),
],
)
]
]
),
)
@@ -2578,28 +2582,30 @@ async def test_agent_as_tool_streaming_settles_multi_segment_structured_output()
agent = Agent(
name="streamer",
model=FakeModel(
initial_output=[
ResponseOutputMessage(
id="msg_multi_segment_structured",
role="assistant",
status="completed",
type="message",
content=[
ResponseOutputText(
annotations=[],
text='{"answer":"str',
type="output_text",
logprobs=[],
),
ResponseOutputText(
annotations=[],
text='uctured"}',
type="output_text",
logprobs=[],
),
],
)
model=ScriptedModel(
steps=[
[
ResponseOutputMessage(
id="msg_multi_segment_structured",
role="assistant",
status="completed",
type="message",
content=[
ResponseOutputText(
annotations=[],
text='{"answer":"str',
type="output_text",
logprobs=[],
),
ResponseOutputText(
annotations=[],
text='uctured"}',
type="output_text",
logprobs=[],
),
],
)
]
]
),
output_type=StructuredOutput,
@@ -2686,10 +2692,10 @@ async def test_agent_as_tool_streaming_settles_final_text_after_nested_mcp_failu
agent = Agent(
name="streamer",
model=FakeModel(),
model=ScriptedModel(),
mcp_servers=[nested_server],
)
cast(FakeModel, agent.model).add_multiple_turn_outputs(
cast(ScriptedModel, agent.model).extend(
[
[get_function_tool_call(tool_name, "{}")],
[
+30 -30
View File
@@ -11,10 +11,10 @@ from agents.agent import Agent
from agents.lifecycle import AgentHooks
from agents.run import Runner
from agents.run_context import AgentHookContext, RunContextWrapper, TContext
from agents.testing import ScriptedModel
from agents.tool import Tool
from agents.tool_context import ToolContext
from .fake_model import FakeModel
from .test_responses import (
get_final_output_message,
get_function_tool,
@@ -82,14 +82,14 @@ class FalsyAgentHooks(AgentHooksForTests):
@pytest.mark.asyncio
async def test_falsy_agent_hooks_are_invoked() -> None:
hooks = FalsyAgentHooks()
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("some_function", "result")],
hooks=hooks,
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
[get_text_message("done")],
@@ -109,7 +109,7 @@ async def test_falsy_agent_hooks_are_invoked() -> None:
@pytest.mark.asyncio
async def test_non_streamed_agent_hooks():
hooks = AgentHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(
name="test_1",
model=model,
@@ -128,12 +128,12 @@ async def test_non_streamed_agent_hooks():
agent_1.handoffs.append(agent_3)
model.set_next_output([get_text_message("user_message")])
model.enqueue([get_text_message("user_message")])
output = await Runner.run(agent_3, input="user_message")
assert hooks.events == {"on_start": 1, "on_end": 1}, f"{output}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
[get_text_message("done")],
@@ -144,7 +144,7 @@ async def test_non_streamed_agent_hooks():
assert len(set(hooks.tool_context_ids)) == 1
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -165,7 +165,7 @@ async def test_non_streamed_agent_hooks():
}, f"got unexpected event count: {hooks.events}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
@@ -196,7 +196,7 @@ async def test_non_streamed_agent_hooks():
@pytest.mark.asyncio
async def test_streamed_agent_hooks():
hooks = AgentHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(name="test_2", model=model)
agent_3 = Agent(
@@ -209,14 +209,14 @@ async def test_streamed_agent_hooks():
agent_1.handoffs.append(agent_3)
model.set_next_output([get_text_message("user_message")])
model.enqueue([get_text_message("user_message")])
output = Runner.run_streamed(agent_3, input="user_message")
async for _ in output.stream_events():
pass
assert hooks.events == {"on_start": 1, "on_end": 1}, f"{output}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -239,7 +239,7 @@ async def test_streamed_agent_hooks():
}, f"got unexpected event count: {hooks.events}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
@@ -276,7 +276,7 @@ class Foo(TypedDict):
@pytest.mark.asyncio
async def test_structured_output_non_streamed_agent_hooks():
hooks = AgentHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(name="test_2", model=model)
agent_3 = Agent(
@@ -290,12 +290,12 @@ async def test_structured_output_non_streamed_agent_hooks():
agent_1.handoffs.append(agent_3)
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
output = await Runner.run(agent_3, input="user_message")
assert hooks.events == {"on_start": 1, "on_end": 1}, f"{output}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -316,7 +316,7 @@ async def test_structured_output_non_streamed_agent_hooks():
}, f"got unexpected event count: {hooks.events}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
@@ -347,7 +347,7 @@ async def test_structured_output_non_streamed_agent_hooks():
@pytest.mark.asyncio
async def test_structured_output_streamed_agent_hooks():
hooks = AgentHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(name="test_2", model=model)
agent_3 = Agent(
@@ -361,14 +361,14 @@ async def test_structured_output_streamed_agent_hooks():
agent_1.handoffs.append(agent_3)
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
output = Runner.run_streamed(agent_3, input="user_message")
async for _ in output.stream_events():
pass
assert hooks.events == {"on_start": 1, "on_end": 1}, f"{output}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -388,7 +388,7 @@ async def test_structured_output_streamed_agent_hooks():
}, f"got unexpected event count: {hooks.events}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
@@ -425,7 +425,7 @@ class EmptyAgentHooks(AgentHooks):
@pytest.mark.asyncio
async def test_base_agent_hooks_dont_crash():
hooks = EmptyAgentHooks()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(name="test_2", model=model)
agent_3 = Agent(
@@ -438,12 +438,12 @@ async def test_base_agent_hooks_dont_crash():
)
agent_1.handoffs.append(agent_3)
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
output = Runner.run_streamed(agent_3, input="user_message")
async for _ in output.stream_events():
pass
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -455,7 +455,7 @@ async def test_base_agent_hooks_dont_crash():
)
await Runner.run(agent_3, input="user_message")
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -490,10 +490,10 @@ class AgentHooksWithTurnInput(AgentHooks):
async def test_agent_hooks_receives_turn_input_string():
"""Test that on_start receives turn_input when input is a string."""
hooks = AgentHooksWithTurnInput()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model, hooks=hooks)
model.set_next_output([get_text_message("response")])
model.enqueue([get_text_message("response")])
await Runner.run(agent, input="hello world")
assert len(hooks.captured_turn_inputs) == 1
@@ -507,7 +507,7 @@ async def test_agent_hooks_receives_turn_input_string():
async def test_agent_hooks_receives_turn_input_list():
"""Test that on_start receives turn_input when input is a list."""
hooks = AgentHooksWithTurnInput()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model, hooks=hooks)
input_items: list[Any] = [
@@ -515,7 +515,7 @@ async def test_agent_hooks_receives_turn_input_list():
{"role": "user", "content": "second message"},
]
model.set_next_output([get_text_message("response")])
model.enqueue([get_text_message("response")])
await Runner.run(agent, input=input_items)
assert len(hooks.captured_turn_inputs) == 1
@@ -529,10 +529,10 @@ async def test_agent_hooks_receives_turn_input_list():
async def test_agent_hooks_receives_turn_input_streamed():
"""Test that on_start receives turn_input in streamed mode."""
hooks = AgentHooksWithTurnInput()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model, hooks=hooks)
model.set_next_output([get_text_message("response")])
model.enqueue([get_text_message("response")])
result = Runner.run_streamed(agent, input="streamed input")
async for _ in result.stream_events():
pass
+7 -7
View File
@@ -8,9 +8,9 @@ from agents.items import ItemHelpers, ModelResponse, TResponseInputItem
from agents.lifecycle import AgentHooks
from agents.run import Runner
from agents.run_context import AgentHookContext, RunContextWrapper, TContext
from agents.testing import ScriptedModel
from agents.tool import Tool
from .fake_model import FakeModel
from .test_responses import (
get_function_tool,
get_text_message,
@@ -74,12 +74,12 @@ class AgentHooksForTests(AgentHooks):
@pytest.mark.asyncio
async def test_async_agent_hooks_with_llm():
hooks = AgentHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[], hooks=hooks
)
# Simulate a single LLM call producing an output:
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
await Runner.run(agent, input="hello")
# Expect one on_start, one on_llm_start, one on_llm_end, and one on_end
assert hooks.events == {"on_start": 1, "on_llm_start": 1, "on_llm_end": 1, "on_end": 1}
@@ -88,12 +88,12 @@ async def test_async_agent_hooks_with_llm():
# test_sync_agent_hook_with_llm()
def test_sync_agent_hook_with_llm():
hooks = AgentHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[], hooks=hooks
)
# Simulate a single LLM call producing an output:
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
Runner.run_sync(agent, input="hello")
# Expect one on_start, one on_llm_start, one on_llm_end, and one on_end
assert hooks.events == {"on_start": 1, "on_llm_start": 1, "on_llm_end": 1, "on_end": 1}
@@ -103,12 +103,12 @@ def test_sync_agent_hook_with_llm():
@pytest.mark.asyncio
async def test_streamed_agent_hooks_with_llm():
hooks = AgentHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[], hooks=hooks
)
# Simulate a single LLM call producing an output:
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
stream = Runner.run_streamed(agent, input="hello")
async for event in stream.stream_events():
+3 -3
View File
@@ -7,7 +7,7 @@ import pytest
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
from agents import Agent, Runner
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
def _make_message(text: str) -> ResponseOutputMessage:
@@ -22,8 +22,8 @@ def _make_message(text: str) -> ResponseOutputMessage:
@pytest.mark.asyncio
async def test_agent_is_released_after_run() -> None:
fake_model = FakeModel(initial_output=[_make_message("Paris")])
agent = Agent(name="leak-test-agent", instructions="Answer questions.", model=fake_model)
scripted_model = ScriptedModel(steps=[[_make_message("Paris")]])
agent = Agent(name="leak-test-agent", instructions="Answer questions.", model=scripted_model)
agent_ref = weakref.ref(agent)
# Running the agent should not leave behind strong references once the result goes out of scope.
+8 -7
View File
@@ -10,13 +10,14 @@ from agents import Agent, Prompt, RunConfig, RunContextWrapper, Runner
from agents.models.interface import Model, ModelProvider
from agents.models.openai_responses import OpenAIResponsesModel
from agents.prompts import GenerateDynamicPromptData
from agents.testing import ScriptedModel
from tests.model_test_helpers import get_response_obj
from .fake_model import FakeModel, get_response_obj
from .test_responses import get_text_message
class PromptCaptureFakeModel(FakeModel):
"""Subclass of FakeModel that records the prompt passed to the model."""
class PromptCaptureScriptedModel(ScriptedModel):
"""Subclass of ScriptedModel that records the prompt passed to the model."""
def __init__(self):
super().__init__()
@@ -91,11 +92,11 @@ async def test_dynamic_prompt_is_resolved_correctly():
async def test_prompt_is_passed_to_model():
static_prompt: Prompt = {"id": "model_prompt"}
model = PromptCaptureFakeModel()
model = PromptCaptureScriptedModel()
agent = Agent(name="test", model=model, prompt=static_prompt)
# Ensure the model returns a simple message so the run completes in one turn.
model.set_next_output([get_text_message("done")])
model.enqueue([get_text_message("done")])
await Runner.run(agent, input="hello")
@@ -171,7 +172,7 @@ async def test_run_cancels_sibling_instructions_when_prompt_resolution_fails() -
agent = Agent(
name="prompt-agent",
model=FakeModel(),
model=ScriptedModel(),
instructions=slow_instructions,
prompt=failing_prompt,
)
@@ -206,7 +207,7 @@ async def test_run_streamed_cancels_sibling_instructions_when_prompt_resolution_
agent = Agent(
name="prompt-agent",
model=FakeModel(),
model=ScriptedModel(),
instructions=slow_instructions,
prompt=failing_prompt,
)
+323 -336
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+63 -63
View File
@@ -9,9 +9,9 @@ from openai.types.responses.response_usage import InputTokensDetails
from agents import Agent, RunConfig, Runner, RunState, custom_span, function_tool, trace
from agents.sandbox.runtime import SandboxRuntime
from agents.testing import ScriptedModel
from agents.usage import Usage
from .fake_model import FakeModel
from .test_responses import get_function_tool_call, get_text_message
from .testing_processor import (
assert_no_traces,
@@ -22,7 +22,7 @@ from .testing_processor import (
)
def _make_approval_agent(model: FakeModel) -> Agent[None]:
def _make_approval_agent(model: ScriptedModel) -> Agent[None]:
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "ok"
@@ -43,8 +43,8 @@ def _usage_metadata(requests: int, input_tokens: int, output_tokens: int) -> dic
async def test_single_run_is_single_trace():
agent = Agent(
name="test_agent",
model=FakeModel(
initial_output=[get_text_message("first_test")],
model=ScriptedModel(
steps=[[get_text_message("first_test")]],
),
)
@@ -77,7 +77,7 @@ async def test_agent_span_uses_resolved_tool_name_collision_view(
surface: str,
streamed: bool,
) -> None:
model = FakeModel(initial_output=[get_text_message("done")])
model = ScriptedModel(steps=[[get_text_message("done")]])
expected_tools: list[str]
expected_handoffs: list[str]
@@ -136,14 +136,14 @@ async def test_task_and_turn_spans_export_aggregate_usage():
def foo_tool() -> str:
return "foo result"
model = FakeModel(tracing_enabled=True)
model.add_multiple_turn_outputs(
model = ScriptedModel(emit_traces=True)
model.extend(
[
[get_function_tool_call("foo_tool", "{}", call_id="call-1")],
[get_text_message("done")],
]
)
model.set_hardcoded_usage(
model.set_default_usage(
Usage(
requests=1,
input_tokens=10,
@@ -259,8 +259,8 @@ async def test_task_and_turn_spans_can_be_disabled():
def foo_tool() -> str:
return "foo result"
model = FakeModel(tracing_enabled=True)
model.add_multiple_turn_outputs(
model = ScriptedModel(emit_traces=True)
model.extend(
[
[get_function_tool_call("foo_tool", "{}", call_id="call-1")],
[get_text_message("done")],
@@ -296,9 +296,9 @@ async def test_task_and_turn_spans_can_be_disabled():
async def test_task_and_turn_spans_can_be_explicitly_enabled():
agent = Agent(
name="test_agent",
model=FakeModel(
tracing_enabled=True,
initial_output=[get_text_message("done")],
model=ScriptedModel(
emit_traces=True,
steps=[[get_text_message("done")]],
),
)
@@ -317,9 +317,9 @@ async def test_task_and_turn_spans_can_be_explicitly_enabled():
async def test_task_span_resets_current_span_if_run_setup_fails(monkeypatch: pytest.MonkeyPatch):
agent = Agent(
name="test_agent",
model=FakeModel(
tracing_enabled=True,
initial_output=[get_text_message("first_test")],
model=ScriptedModel(
emit_traces=True,
steps=[[get_text_message("first_test")]],
),
)
@@ -347,8 +347,8 @@ async def test_task_span_resets_current_span_if_run_setup_fails(monkeypatch: pyt
@pytest.mark.asyncio
async def test_multiple_runs_are_multiple_traces():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_text_message("first_test")],
[get_text_message("second_test")],
@@ -398,8 +398,8 @@ async def test_multiple_runs_are_multiple_traces():
@pytest.mark.asyncio
async def test_resumed_run_reuses_original_trace_without_duplicate_trace_start():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
[get_text_message("done")],
@@ -425,14 +425,14 @@ async def test_resumed_run_reuses_original_trace_without_duplicate_trace_start()
@pytest.mark.asyncio
async def test_resumed_run_task_span_usage_is_run_local_delta():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
[get_text_message("done")],
]
)
model.set_hardcoded_usage(Usage(requests=1, input_tokens=10, output_tokens=3, total_tokens=13))
model.set_default_usage(Usage(requests=1, input_tokens=10, output_tokens=3, total_tokens=13))
agent = _make_approval_agent(model)
first = await Runner.run(agent, input="first_test")
@@ -461,8 +461,8 @@ async def test_resumed_run_task_span_usage_is_run_local_delta():
@pytest.mark.asyncio
async def test_resumed_run_from_serialized_state_reuses_original_trace():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
[get_text_message("done")],
@@ -490,8 +490,8 @@ async def test_resumed_run_from_serialized_state_reuses_original_trace():
@pytest.mark.asyncio
async def test_resumed_run_from_serialized_state_preserves_explicit_trace_key():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
[get_text_message("done")],
@@ -530,8 +530,8 @@ async def test_resumed_run_from_serialized_state_preserves_explicit_trace_key():
@pytest.mark.asyncio
async def test_resumed_run_with_workflow_override_starts_new_trace() -> None:
trace_id = f"trace_{uuid4().hex}"
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
[get_text_message("done")],
@@ -570,8 +570,8 @@ async def test_resumed_run_with_workflow_override_starts_new_trace() -> None:
@pytest.mark.asyncio
async def test_wrapped_trace_is_single_trace():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_text_message("first_test")],
[get_text_message("second_test")],
@@ -631,8 +631,8 @@ async def test_parent_disabled_trace_disabled_agent_trace():
with trace(workflow_name="test_workflow", disabled=True):
agent = Agent(
name="test_agent",
model=FakeModel(
initial_output=[get_text_message("first_test")],
model=ScriptedModel(
steps=[[get_text_message("first_test")]],
),
)
@@ -645,8 +645,8 @@ async def test_parent_disabled_trace_disabled_agent_trace():
async def test_manual_disabling_works():
agent = Agent(
name="test_agent",
model=FakeModel(
initial_output=[get_text_message("first_test")],
model=ScriptedModel(
steps=[[get_text_message("first_test")]],
),
)
@@ -659,8 +659,8 @@ async def test_manual_disabling_works():
async def test_trace_config_works():
agent = Agent(
name="test_agent",
model=FakeModel(
initial_output=[get_text_message("first_test")],
model=ScriptedModel(
steps=[[get_text_message("first_test")]],
),
)
@@ -696,8 +696,8 @@ async def test_trace_config_works():
async def test_not_starting_streaming_creates_trace():
agent = Agent(
name="test_agent",
model=FakeModel(
initial_output=[get_text_message("first_test")],
model=ScriptedModel(
steps=[[get_text_message("first_test")]],
),
)
@@ -737,8 +737,8 @@ async def test_not_starting_streaming_creates_trace():
async def test_streaming_single_run_is_single_trace():
agent = Agent(
name="test_agent",
model=FakeModel(
initial_output=[get_text_message("first_test")],
model=ScriptedModel(
steps=[[get_text_message("first_test")]],
),
)
@@ -768,8 +768,8 @@ async def test_streaming_single_run_is_single_trace():
@pytest.mark.asyncio
async def test_multiple_streamed_runs_are_multiple_traces():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_text_message("first_test")],
[get_text_message("second_test")],
@@ -824,8 +824,8 @@ async def test_multiple_streamed_runs_are_multiple_traces():
@pytest.mark.asyncio
async def test_resumed_streaming_run_reuses_original_trace_without_duplicate_trace_start():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
[get_text_message("done")],
@@ -855,14 +855,14 @@ async def test_resumed_streaming_run_reuses_original_trace_without_duplicate_tra
@pytest.mark.asyncio
async def test_resumed_streaming_run_task_span_usage_is_run_local_delta():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
[get_text_message("done")],
]
)
model.set_hardcoded_usage(Usage(requests=1, input_tokens=11, output_tokens=4, total_tokens=15))
model.set_default_usage(Usage(requests=1, input_tokens=11, output_tokens=4, total_tokens=15))
agent = _make_approval_agent(model)
first = Runner.run_streamed(agent, input="first_test")
@@ -895,8 +895,8 @@ async def test_resumed_streaming_run_task_span_usage_is_run_local_delta():
@pytest.mark.asyncio
async def test_wrapped_streaming_trace_is_single_trace():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_text_message("first_test")],
[get_text_message("second_test")],
@@ -963,9 +963,9 @@ async def test_wrapped_streaming_trace_is_single_trace():
async def test_wrapped_streaming_run_creates_root_task_span():
agent = Agent(
name="test_agent",
model=FakeModel(
tracing_enabled=True,
initial_output=[get_text_message("first_test")],
model=ScriptedModel(
emit_traces=True,
steps=[[get_text_message("first_test")]],
),
)
@@ -997,7 +997,7 @@ async def test_wrapped_run_task_span_uses_run_workflow_name():
def _make_agent() -> Agent[None]:
return Agent(
name="test_agent",
model=FakeModel(initial_output=[get_text_message("first_test")]),
model=ScriptedModel(steps=[[get_text_message("first_test")]]),
)
run_config = RunConfig(workflow_name="inner_workflow")
@@ -1021,9 +1021,9 @@ async def test_wrapped_run_task_span_uses_run_workflow_name():
async def test_wrapped_streaming_run_can_disable_task_and_turn_spans():
agent = Agent(
name="test_agent",
model=FakeModel(
tracing_enabled=True,
initial_output=[get_text_message("done")],
model=ScriptedModel(
emit_traces=True,
steps=[[get_text_message("done")]],
),
)
@@ -1050,8 +1050,8 @@ async def test_wrapped_streaming_run_can_disable_task_and_turn_spans():
@pytest.mark.asyncio
async def test_wrapped_mixed_trace_is_single_trace():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_text_message("first_test")],
[get_text_message("second_test")],
@@ -1114,8 +1114,8 @@ async def test_wrapped_mixed_trace_is_single_trace():
@pytest.mark.asyncio
async def test_parent_disabled_trace_disables_streaming_agent_trace():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_text_message("first_test")],
[get_text_message("second_test")],
@@ -1136,8 +1136,8 @@ async def test_parent_disabled_trace_disables_streaming_agent_trace():
@pytest.mark.asyncio
async def test_manual_streaming_disabling_works():
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_text_message("first_test")],
[get_text_message("second_test")],
+39 -41
View File
@@ -6,8 +6,8 @@ import pytest
from agents import Agent, RunConfig, Runner, TResponseInputItem, UserError
from agents.run import CallModelData, ModelInputData
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import get_text_input_item, get_text_message
from .testing_processor import fetch_span_errors
@@ -16,11 +16,11 @@ SENSITIVE_ERROR_MESSAGE = "sensitive-filter-error"
@pytest.mark.asyncio
async def test_call_model_input_filter_sync_non_streamed() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# Prepare model output
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
def filter_fn(data: CallModelData[Any]) -> ModelInputData:
mi = data.model_data
@@ -33,19 +33,19 @@ async def test_call_model_input_filter_sync_non_streamed() -> None:
run_config=RunConfig(call_model_input_filter=filter_fn),
)
assert model.last_turn_args["system_instructions"] == "filtered-sync"
assert isinstance(model.last_turn_args["input"], list)
assert len(model.last_turn_args["input"]) == 2
assert model.last_turn_args["input"][-1]["content"] == "added-sync"
assert model.calls[-1].system_instructions == "filtered-sync"
assert isinstance(model.calls[-1].input, list)
assert len(model.calls[-1].input) == 2
assert model.calls[-1].input[-1]["content"] == "added-sync"
@pytest.mark.asyncio
async def test_call_model_input_filter_async_streamed() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
# Prepare model output
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
async def filter_fn(data: CallModelData[Any]) -> ModelInputData:
mi = data.model_data
@@ -60,15 +60,15 @@ async def test_call_model_input_filter_async_streamed() -> None:
async for _ in result.stream_events():
pass
assert model.last_turn_args["system_instructions"] == "filtered-async"
assert isinstance(model.last_turn_args["input"], list)
assert len(model.last_turn_args["input"]) == 2
assert model.last_turn_args["input"][-1]["content"] == "added-async"
assert model.calls[-1].system_instructions == "filtered-async"
assert isinstance(model.calls[-1].input, list)
assert len(model.calls[-1].input) == 2
assert model.calls[-1].input[-1]["content"] == "added-async"
@pytest.mark.asyncio
async def test_call_model_input_filter_invalid_return_type_raises() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
def invalid_filter(_data: CallModelData[Any]):
@@ -99,7 +99,7 @@ async def test_call_model_input_filter_error_respects_sensitive_data_setting(
with pytest.raises(ValueError, match=SENSITIVE_ERROR_MESSAGE):
await Runner.run(
Agent(name="test", model=FakeModel(tracing_enabled=False)),
Agent(name="test", model=ScriptedModel(emit_traces=False)),
input="start",
run_config=RunConfig(
call_model_input_filter=filter_fn,
@@ -117,9 +117,9 @@ async def test_call_model_input_filter_error_respects_sensitive_data_setting(
@pytest.mark.asyncio
async def test_call_model_input_filter_prefers_latest_duplicate_outputs_non_streamed() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
duplicate_old = cast(
TResponseInputItem,
@@ -152,7 +152,7 @@ async def test_call_model_input_filter_prefers_latest_duplicate_outputs_non_stre
outputs = [
item
for item in model.last_turn_args["input"]
for item in model.calls[-1].input
if item.get("type") == "function_call_output" and item.get("call_id") == "dup-call"
]
assert len(outputs) == 1
@@ -161,9 +161,9 @@ async def test_call_model_input_filter_prefers_latest_duplicate_outputs_non_stre
@pytest.mark.asyncio
async def test_call_model_input_filter_prefers_latest_duplicate_outputs_streamed() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
duplicate_old = cast(
TResponseInputItem,
@@ -198,7 +198,7 @@ async def test_call_model_input_filter_prefers_latest_duplicate_outputs_streamed
outputs = [
item
for item in model.last_turn_args["input"]
for item in model.calls[-1].input
if item.get("type") == "function_call_output" and item.get("call_id") == "dup-call-stream"
]
assert len(outputs) == 1
@@ -294,9 +294,9 @@ def _sent_item_types(sent_input: Any) -> list[str | None]:
@pytest.mark.asyncio
async def test_call_model_input_filter_keeps_duplicate_item_order_non_streamed() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
def filter_fn(data: CallModelData[Any]) -> ModelInputData:
return ModelInputData(
@@ -312,7 +312,7 @@ async def test_call_model_input_filter_keeps_duplicate_item_order_non_streamed()
# Collapsing the repeated call must not move it behind its output; the Responses API
# rejects a function_call_output whose function_call has not been sent yet.
assert _sent_item_types(model.last_turn_args["input"]) == [
assert _sent_item_types(model.calls[-1].input) == [
"function_call",
"function_call_output",
]
@@ -320,9 +320,9 @@ async def test_call_model_input_filter_keeps_duplicate_item_order_non_streamed()
@pytest.mark.asyncio
async def test_call_model_input_filter_keeps_duplicate_item_order_streamed() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
async def filter_fn(data: CallModelData[Any]) -> ModelInputData:
return ModelInputData(
@@ -338,7 +338,7 @@ async def test_call_model_input_filter_keeps_duplicate_item_order_streamed() ->
async for _ in result.stream_events():
pass
assert _sent_item_types(model.last_turn_args["input"]) == [
assert _sent_item_types(model.calls[-1].input) == [
"function_call",
"function_call_output",
]
@@ -346,9 +346,9 @@ async def test_call_model_input_filter_keeps_duplicate_item_order_streamed() ->
@pytest.mark.asyncio
async def test_call_model_input_filter_keeps_duplicate_output_order_non_streamed() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
def filter_fn(data: CallModelData[Any]) -> ModelInputData:
return ModelInputData(
@@ -362,18 +362,18 @@ async def test_call_model_input_filter_keeps_duplicate_output_order_non_streamed
run_config=RunConfig(call_model_input_filter=filter_fn),
)
assert _sent_item_types(model.last_turn_args["input"]) == [
assert _sent_item_types(model.calls[-1].input) == [
"function_call",
"function_call_output",
]
assert model.last_turn_args["input"][-1]["output"] == "new"
assert model.calls[-1].input[-1]["output"] == "new"
@pytest.mark.asyncio
async def test_call_model_input_filter_keeps_duplicate_output_order_streamed() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
async def filter_fn(data: CallModelData[Any]) -> ModelInputData:
return ModelInputData(
@@ -389,18 +389,18 @@ async def test_call_model_input_filter_keeps_duplicate_output_order_streamed() -
async for _ in result.stream_events():
pass
assert _sent_item_types(model.last_turn_args["input"]) == [
assert _sent_item_types(model.calls[-1].input) == [
"function_call",
"function_call_output",
]
assert model.last_turn_args["input"][-1]["output"] == "new"
assert model.calls[-1].input[-1]["output"] == "new"
@pytest.mark.asyncio
async def test_call_model_input_filter_keeps_reasoning_before_required_follower() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("ok")])
model.enqueue([get_text_message("ok")])
def filter_fn(data: CallModelData[Any]) -> ModelInputData:
return ModelInputData(
@@ -414,12 +414,10 @@ async def test_call_model_input_filter_keeps_reasoning_before_required_follower(
run_config=RunConfig(call_model_input_filter=filter_fn),
)
assert _sent_item_types(model.last_turn_args["input"]) == [
assert _sent_item_types(model.calls[-1].input) == [
"reasoning",
"function_call",
]
reasoning_items = [
item for item in model.last_turn_args["input"] if item.get("type") == "reasoning"
]
reasoning_items = [item for item in model.calls[-1].input if item.get("type") == "reasoning"]
assert len(reasoning_items) == 1
assert reasoning_items[0]["summary"] == [{"type": "summary_text", "text": "new"}]
+14 -14
View File
@@ -9,15 +9,15 @@ from openai.types.responses import ResponseOutputMessage, ResponseOutputText
from agents.agent import Agent
from agents.exceptions import UserError
from agents.run import CallModelData, ModelInputData, RunConfig, Runner
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
@pytest.mark.asyncio
async def test_call_model_input_filter_sync_non_streamed_unit() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output(
model.enqueue(
[
ResponseOutputMessage(
id="1",
@@ -44,18 +44,18 @@ async def test_call_model_input_filter_sync_non_streamed_unit() -> None:
run_config=RunConfig(call_model_input_filter=filter_fn),
)
assert model.last_turn_args["system_instructions"] == "filtered-sync"
assert isinstance(model.last_turn_args["input"], list)
assert len(model.last_turn_args["input"]) == 2
assert model.last_turn_args["input"][-1]["content"] == "added-sync"
assert model.calls[-1].system_instructions == "filtered-sync"
assert isinstance(model.calls[-1].input, list)
assert len(model.calls[-1].input) == 2
assert model.calls[-1].input[-1]["content"] == "added-sync"
@pytest.mark.asyncio
async def test_call_model_input_filter_async_streamed_unit() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output(
model.enqueue(
[
ResponseOutputMessage(
id="1",
@@ -84,15 +84,15 @@ async def test_call_model_input_filter_async_streamed_unit() -> None:
async for _ in result.stream_events():
pass
assert model.last_turn_args["system_instructions"] == "filtered-async"
assert isinstance(model.last_turn_args["input"], list)
assert len(model.last_turn_args["input"]) == 2
assert model.last_turn_args["input"][-1]["content"] == "added-async"
assert model.calls[-1].system_instructions == "filtered-async"
assert isinstance(model.calls[-1].input, list)
assert len(model.calls[-1].input) == 2
assert model.calls[-1].input[-1]["content"] == "added-async"
@pytest.mark.asyncio
async def test_call_model_input_filter_invalid_return_type_raises_unit() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
def invalid_filter(_data: CallModelData[Any]):
+21 -21
View File
@@ -8,13 +8,13 @@ from openai.types.responses import ResponseCompletedEvent
from agents import Agent, Runner
from agents.guardrail import input_guardrail
from agents.stream_events import RawResponsesStreamEvent
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import get_function_tool, get_function_tool_call, get_text_message
class SlowCompleteFakeModel(FakeModel):
"""A FakeModel that delays before emitting the completed event in streaming."""
class SlowCompleteScriptedModel(ScriptedModel):
"""A ScriptedModel that delays before emitting the completed event in streaming."""
def __init__(self, delay_seconds: float):
super().__init__()
@@ -29,7 +29,7 @@ class SlowCompleteFakeModel(FakeModel):
@pytest.mark.asyncio
async def test_simple_streaming_with_cancel():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Joker", model=model)
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
@@ -46,14 +46,14 @@ async def test_simple_streaming_with_cancel():
@pytest.mark.asyncio
async def test_multiple_events_streaming_with_cancel():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="Joker",
model=model,
tools=[get_function_tool("foo", "tool_result")],
)
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[
@@ -79,7 +79,7 @@ async def test_multiple_events_streaming_with_cancel():
@pytest.mark.asyncio
async def test_cancel_prevents_further_events():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Joker", model=model)
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
events = []
@@ -95,7 +95,7 @@ async def test_cancel_prevents_further_events():
@pytest.mark.asyncio
async def test_cancel_is_idempotent():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Joker", model=model)
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
events = []
@@ -110,7 +110,7 @@ async def test_cancel_is_idempotent():
@pytest.mark.asyncio
async def test_cancel_before_streaming():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Joker", model=model)
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
result.cancel() # Cancel before streaming
@@ -120,7 +120,7 @@ async def test_cancel_before_streaming():
@pytest.mark.asyncio
async def test_cancel_cleans_up_resources():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Joker", model=model)
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
# Start streaming, then cancel
@@ -138,7 +138,7 @@ async def test_cancel_cleans_up_resources():
@pytest.mark.asyncio
async def test_cancel_immediate_mode_explicit():
"""Test explicit immediate mode behaves same as default."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Joker", model=model)
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
@@ -154,8 +154,8 @@ async def test_cancel_immediate_mode_explicit():
@pytest.mark.asyncio
async def test_stream_events_respects_asyncio_timeout_cancellation():
model = SlowCompleteFakeModel(delay_seconds=0.5)
model.set_next_output([get_text_message("Final response")])
model = SlowCompleteScriptedModel(delay_seconds=0.5)
model.enqueue([get_text_message("Final response")])
agent = Agent(name="TimeoutTester", model=model)
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
@@ -183,7 +183,7 @@ async def test_stream_events_respects_asyncio_timeout_cancellation():
async def test_cancel_immediate_unblocks_waiting_stream_consumer():
block_event = asyncio.Event()
class BlockingFakeModel(FakeModel):
class BlockingScriptedModel(ScriptedModel):
async def stream_response(
self,
system_instructions,
@@ -213,7 +213,7 @@ async def test_cancel_immediate_unblocks_waiting_stream_consumer():
):
yield event
model = BlockingFakeModel()
model = BlockingScriptedModel()
agent = Agent(name="Joker", model=model)
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
@@ -236,8 +236,8 @@ async def test_cancel_immediate_unblocks_waiting_stream_consumer():
@pytest.mark.asyncio
async def test_run_loop_exception_property_is_none_on_success():
"""run_loop_exception is None when the stream completes without error."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
agent = Agent(name="A", model=model)
result = Runner.run_streamed(agent, input="hi")
@@ -251,7 +251,7 @@ async def test_run_loop_exception_property_is_none_on_success():
async def test_run_loop_exception_surfaced_after_stream():
"""run_loop_exception is set when the run loop raises before yielding events."""
class BoomModel(FakeModel):
class BoomModel(ScriptedModel):
async def get_response(self, *args, **kwargs):
raise RuntimeError("run loop boom")
@@ -278,7 +278,7 @@ async def test_falsy_run_loop_exception_is_surfaced_after_stream() -> None:
def __bool__(self) -> bool:
return False
class BoomModel(FakeModel):
class BoomModel(ScriptedModel):
async def stream_response(self, *args, **kwargs):
raise FalsyRuntimeError("falsy run loop boom")
yield
@@ -300,8 +300,8 @@ async def test_falsy_input_guardrail_exception_is_surfaced_after_stream() -> Non
async def raising_guardrail(context, agent, input):
raise FalsyRuntimeError("falsy guardrail boom")
model = FakeModel()
model.set_next_output([get_text_message("done")])
model = ScriptedModel()
model.enqueue([get_text_message("done")])
result = Runner.run_streamed(
Agent(name="A", model=model, input_guardrails=[raising_guardrail]),
input="hi",
+3 -3
View File
@@ -46,9 +46,9 @@ from agents import (
from agents.items import ToolCallOutputItem
from agents.run_internal import run_loop
from agents.run_internal.run_loop import ComputerAction, ToolRunComputerAction
from agents.testing import ScriptedModel
from agents.tool import ComputerToolSafetyCheckData
from .fake_model import FakeModel
from .test_responses import get_text_message
from .testing_processor import SPAN_PROCESSOR_TESTING
@@ -639,8 +639,8 @@ async def test_runner_trace_lists_ga_computer_tool_name() -> None:
pending_safety_checks=[],
status="completed",
)
model = FakeModel(tracing_enabled=True)
model.add_multiple_turn_outputs(
model = ScriptedModel(emit_traces=True)
model.extend(
[
[tool_call],
[get_text_message("done")],
+23 -21
View File
@@ -26,7 +26,7 @@ from agents import (
)
from agents.computer import Button, Computer, Environment
from agents.models.openai_responses import Converter
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
class FakeComputer(Computer):
@@ -158,7 +158,7 @@ async def test_runner_disposes_computer_after_run() -> None:
dispose = AsyncMock()
tool = ComputerTool(computer=ComputerProvider[FakeComputer](create=create, dispose=dispose))
model = FakeModel(initial_output=[_make_message("done")])
model = ScriptedModel(steps=[[_make_message("done")]])
agent = Agent(name="ComputerAgent", model=model, tools=[tool])
result = await Runner.run(agent, "hello")
@@ -167,7 +167,7 @@ async def test_runner_disposes_computer_after_run() -> None:
create.assert_awaited_once()
dispose.assert_awaited_once()
dispose.assert_awaited_with(run_context=result.context_wrapper, computer=created)
resolved_tool = cast(ComputerTool[Any], model.last_turn_args["tools"][0])
resolved_tool = cast(ComputerTool[Any], model.calls[-1].tools[0])
assert resolved_tool is not tool
assert resolved_tool.computer is created
@@ -194,27 +194,29 @@ async def test_runner_preserves_concrete_computer_tool_identity_for_hooks() -> N
self.ended.append(tool)
tool = ComputerTool(computer=FakeComputer("concrete"))
model = FakeModel(
initial_output=[
ResponseComputerToolCall(
id="computer-call",
type="computer_call",
action=ActionScreenshot(type="screenshot"),
call_id="computer-call",
pending_safety_checks=[],
status="completed",
)
model = ScriptedModel(
steps=[
[
ResponseComputerToolCall(
id="computer-call",
type="computer_call",
action=ActionScreenshot(type="screenshot"),
call_id="computer-call",
pending_safety_checks=[],
status="completed",
)
]
]
)
model.set_next_output([_make_message("done")])
model.enqueue([_make_message("done")])
agent = Agent(name="ComputerAgent", model=model, tools=[tool])
hooks = IdentityHooks()
result = await Runner.run(agent, "hello", hooks=hooks)
assert result.final_output == "done"
assert model.first_turn_args is not None
assert model.first_turn_args["tools"][0] is tool
assert bool(model.calls)
assert model.calls[0].tools[0] is tool
assert hooks.started == [tool]
assert hooks.ended == [tool]
@@ -239,10 +241,10 @@ async def test_concurrent_runs_keep_computer_provider_instances_isolated() -> No
release_model = [asyncio.Event(), asyncio.Event()]
serialized_widths: list[int] = []
class GatedSerializationModel(FakeModel):
class GatedSerializationModel(ScriptedModel):
def __init__(self) -> None:
super().__init__(initial_output=[_make_message("done")])
self.set_next_output([_make_message("done")])
super().__init__(steps=[[_make_message("done")]])
self.enqueue([_make_message("done")])
self.call_count = 0
async def get_response(
@@ -331,7 +333,7 @@ async def test_streamed_run_disposes_computer_after_completion() -> None:
dispose = AsyncMock()
tool = ComputerTool(computer=ComputerProvider[FakeComputer](create=create, dispose=dispose))
model = FakeModel(initial_output=[_make_message("done")])
model = ScriptedModel(steps=[[_make_message("done")]])
agent = Agent(name="ComputerAgent", model=model, tools=[tool])
streamed_result = Runner.run_streamed(agent, "hello")
@@ -342,6 +344,6 @@ async def test_streamed_run_disposes_computer_after_completion() -> None:
create.assert_awaited_once()
dispose.assert_awaited_once()
dispose.assert_awaited_with(run_context=streamed_result.context_wrapper, computer=created)
resolved_tool = cast(ComputerTool[Any], model.last_turn_args["tools"][0])
resolved_tool = cast(ComputerTool[Any], model.calls[-1].tools[0])
assert resolved_tool is not tool
assert resolved_tool.computer is created
+24 -26
View File
@@ -65,13 +65,13 @@ from agents.run_internal.tool_execution import (
resolve_approval_rejection_message,
)
from agents.run_state import _deserialize_items
from agents.testing import ScriptedModel
from agents.tool_context import ToolContext
from agents.tracing.processor_interface import TracingProcessor
from agents.tracing.provider import SynchronousMultiTracingProcessor
from agents.tracing.spans import Span
from agents.tracing.traces import Trace
from .fake_model import FakeModel
from .test_responses import get_function_tool_call, get_text_message
from .utils.simple_session import SimpleListSession
@@ -1215,9 +1215,9 @@ async def test_run_surfaces_redacted_output_validation_error(
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model.enqueue([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
session = SimpleListSession(
session_id="redacted-run",
history=[{"role": "user", "content": _MODEL_OUTPUT_SECRET}],
@@ -1243,9 +1243,9 @@ def test_run_sync_surfaces_redacted_output_validation_error_without_runner_data(
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model.enqueue([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
session = SimpleListSession(
session_id="redacted-run-sync",
history=[{"role": "user", "content": _MODEL_OUTPUT_SECRET}],
@@ -1273,7 +1273,7 @@ async def test_run_preserves_diagnostic_wrapper_traceback_locals(
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False)
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
diagnostic_input = "DIAGNOSTIC_RUNNER_INPUT_SECRET"
model = FakeModel(initial_output=[get_text_message('{"answer": "missing count"}')])
model = ScriptedModel(steps=[[get_text_message('{"answer": "missing count"}')]])
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
session = SimpleListSession(session_id="diagnostic-runner")
@@ -1292,7 +1292,7 @@ def test_run_sync_preserves_diagnostic_wrapper_traceback_locals(
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False)
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
diagnostic_input = "DIAGNOSTIC_RUNNER_SYNC_INPUT_SECRET"
model = FakeModel(initial_output=[get_text_message('{"answer": "missing count"}')])
model = ScriptedModel(steps=[[get_text_message('{"answer": "missing count"}')]])
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
session = SimpleListSession(session_id="diagnostic-runner-sync")
@@ -1311,9 +1311,9 @@ async def test_streamed_run_surfaces_redacted_output_validation_error(
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model.enqueue([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
result = Runner.run_streamed(agent, "go")
with pytest.raises(ModelBehaviorError) as exc_info:
@@ -1334,7 +1334,7 @@ async def test_streamed_run_loop_exception_follows_model_data_policy(
redacted: bool,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
result = Runner.run_streamed(agent, "go")
@@ -1378,7 +1378,7 @@ async def test_streamed_output_guardrail_omits_run_data_from_redacted_error(
AgentOutputSchema(_RequiredOutput).validate_json(payload)
raise AssertionError("validation should fail") # pragma: no cover
model = FakeModel(initial_output=[get_text_message(_MODEL_OUTPUT_SECRET)])
model = ScriptedModel(steps=[[get_text_message(_MODEL_OUTPUT_SECRET)]])
agent = Agent(
name="A",
model=model,
@@ -1434,7 +1434,7 @@ async def test_streamed_session_error_after_output_guardrail_respects_redaction(
raise AssertionError("validation should fail") # pragma: no cover
caplog.set_level(logging.ERROR, logger="openai.agents")
model = FakeModel(initial_output=[get_text_message(_MODEL_OUTPUT_SECRET)])
model = ScriptedModel(steps=[[get_text_message(_MODEL_OUTPUT_SECRET)]])
agent = Agent(
name="A",
model=model,
@@ -1536,7 +1536,7 @@ async def test_streamed_session_hostile_error_after_redacted_output_guardrail_is
agent = Agent(
name="A",
model=FakeModel(initial_output=[get_text_message(_MODEL_OUTPUT_SECRET)]),
model=ScriptedModel(steps=[[get_text_message(_MODEL_OUTPUT_SECRET)]]),
output_guardrails=[OutputGuardrail(guardrail_function=output_guardrail)],
)
result = Runner.run_streamed(agent, "go", session=FailingFinalTurnSession())
@@ -1569,7 +1569,7 @@ async def test_streamed_input_guardrail_omits_run_data_from_redacted_error(
AgentOutputSchema(_RequiredOutput).validate_json(payload)
raise AssertionError("validation should fail") # pragma: no cover
model = FakeModel(initial_output=[get_text_message("unused")])
model = ScriptedModel(steps=[[get_text_message("unused")]])
agent = Agent(
name="A",
model=model,
@@ -1596,7 +1596,7 @@ async def test_invalid_final_output_handler_receives_detached_redacted_error(
streamed: bool,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
retained_errors: list[ModelBehaviorError] = []
@@ -1636,7 +1636,7 @@ async def test_invalid_final_output_handler_invalid_fallback_preserves_redaction
) -> None:
fallback_secret = "INVALID_HANDLER_FALLBACK_SECRET"
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
def invalid_fallback(_data: RunErrorHandlerInput[None]) -> dict[str, str]:
@@ -1681,10 +1681,8 @@ async def test_invalid_final_output_handler_fallback_serialization_follows_redac
) -> None:
fallback_secret = "PERMISSIVE_HANDLER_FALLBACK_SECRET"
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
model = FakeModel(
initial_output=[
get_text_message(f'{{"payload": "{_MODEL_OUTPUT_SECRET}", "count": "invalid"}}')
]
model = ScriptedModel(
steps=[[get_text_message(f'{{"payload": "{_MODEL_OUTPUT_SECRET}", "count": "invalid"}}')]]
)
agent = Agent(name="A", model=model, output_type=_PermissiveFallbackOutput)
@@ -1743,7 +1741,7 @@ async def test_empty_final_output_handler_fallback_serialization_follows_redacti
) -> None:
fallback_secret = "EMPTY_HANDLER_FALLBACK_SECRET"
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
model = FakeModel(initial_output=[])
model = ScriptedModel(steps=[[]])
agent = Agent(name="A", model=model, output_type=_PermissiveFallbackOutput)
def permissive_fallback(_data: RunErrorHandlerInput[None]) -> RunErrorHandlerResult:
@@ -1792,7 +1790,7 @@ async def test_invalid_final_output_handler_failure_preserves_redaction(
streamed: bool,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
def fail(data: RunErrorHandlerInput[None]) -> None:
@@ -1831,7 +1829,7 @@ async def test_invalid_final_output_handler_hostile_failure_preserves_redaction(
) -> None:
handler_secret = "HOSTILE_HANDLER_FAILURE_SECRET"
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
def fail(_data: RunErrorHandlerInput[None]) -> None:
@@ -1870,7 +1868,7 @@ async def test_invalid_final_output_handler_failure_preserves_diagnostic_context
streamed: bool,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False)
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
def fail(_data: RunErrorHandlerInput[None]) -> None:
@@ -1914,8 +1912,8 @@ async def test_multiturn_output_validation_error_run_data_follows_redaction_poli
def record_value(value: str) -> str:
return "recorded"
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
+111 -110
View File
@@ -30,6 +30,7 @@ from agents import (
)
from agents.agent import ToolsToFinalOutputResult
from agents.items import TResponseInputItem
from agents.testing import ScriptedModel
from agents.tool import FunctionToolResult, function_tool
from examples.financial_research_agent.agents.verifier_agent import (
VerificationIssue,
@@ -55,7 +56,6 @@ from examples.sandbox.sandbox_agents_as_tools import (
from examples.tools.web_search_filters import _normalized_source_urls
from examples.web_search_utils import extract_url_citations, extract_web_search_source_urls
from .fake_model import FakeModel
from .test_responses import (
get_final_output_message,
get_function_tool_call,
@@ -341,16 +341,16 @@ class OutlineCheckerOutput:
@pytest.mark.asyncio
async def test_llm_as_judge_loop_handles_dataclass_feedback() -> None:
"""Mimics the llm_as_a_judge example: loop until the evaluator passes the outline."""
outline_model = FakeModel()
outline_model.add_multiple_turn_outputs(
outline_model = ScriptedModel()
outline_model.extend(
[
[get_text_message("Outline v1")],
[get_text_message("Outline v2")],
]
)
judge_model = FakeModel()
judge_model.add_multiple_turn_outputs(
judge_model = ScriptedModel()
judge_model.extend(
[
[
get_final_output_message(
@@ -400,14 +400,14 @@ async def test_llm_as_judge_loop_handles_dataclass_feedback() -> None:
assert latest_outline == "Outline v2"
assert len(conversation) == 4
assert judge_model.last_turn_args["input"] == conversation
assert judge_model.calls[-1].input == conversation
@pytest.mark.asyncio
async def test_parallel_translation_flow_reuses_runner_outputs() -> None:
"""Covers the parallelization example by feeding multiple translations into a picker agent."""
translation_model = FakeModel()
translation_model.add_multiple_turn_outputs(
translation_model = ScriptedModel()
translation_model.extend(
[
[get_text_message("Uno")],
[get_text_message("Dos")],
@@ -416,8 +416,8 @@ async def test_parallel_translation_flow_reuses_runner_outputs() -> None:
)
spanish_agent = Agent(name="spanish_agent", model=translation_model)
picker_model = FakeModel()
picker_model.set_next_output([get_text_message("Pick: Dos")])
picker_model = ScriptedModel()
picker_model.enqueue([get_text_message("Pick: Dos")])
picker_agent = Agent(name="picker", model=picker_model)
translations: list[str] = []
@@ -433,7 +433,7 @@ async def test_parallel_translation_flow_reuses_runner_outputs() -> None:
assert translations == ["Uno", "Dos", "Tres"]
assert picker_result.final_output == "Pick: Dos"
assert picker_model.last_turn_args["input"] == [
assert picker_model.calls[-1].input == [
{"content": f"Input: Hello\n\nTranslations:\n{combined}", "role": "user"}
]
@@ -441,18 +441,18 @@ async def test_parallel_translation_flow_reuses_runner_outputs() -> None:
@pytest.mark.asyncio
async def test_deterministic_story_flow_stops_when_checker_blocks() -> None:
"""Mimics deterministic flow: stop early when quality gate fails."""
outline_model = FakeModel()
outline_model.set_next_output([get_text_message("Outline v1")])
checker_model = FakeModel()
checker_model.set_next_output(
outline_model = ScriptedModel()
outline_model.enqueue([get_text_message("Outline v1")])
checker_model = ScriptedModel()
checker_model.enqueue(
[
get_final_output_message(
json.dumps({"response": {"good_quality": False, "is_scifi": True}})
)
]
)
story_model = FakeModel()
story_model.set_next_output(RuntimeError("story should not run"))
story_model = ScriptedModel()
story_model.enqueue(RuntimeError("story should not run"))
outline_agent = Agent(name="outline", model=outline_model)
checker_agent = Agent(
@@ -474,24 +474,24 @@ async def test_deterministic_story_flow_stops_when_checker_blocks() -> None:
assert decision.is_scifi is True
if decision.good_quality and decision.is_scifi:
await Runner.run(story_agent, outline_result.final_output)
assert story_model.first_turn_args is None, "story agent should never be invoked when gated"
assert not story_model.calls, "story agent should never be invoked when gated"
@pytest.mark.asyncio
async def test_deterministic_story_flow_runs_story_on_pass() -> None:
"""Mimics deterministic flow: run full path when checker approves."""
outline_model = FakeModel()
outline_model.set_next_output([get_text_message("Outline ready")])
checker_model = FakeModel()
checker_model.set_next_output(
outline_model = ScriptedModel()
outline_model.enqueue([get_text_message("Outline ready")])
checker_model = ScriptedModel()
checker_model.enqueue(
[
get_final_output_message(
json.dumps({"response": {"good_quality": True, "is_scifi": True}})
)
]
)
story_model = FakeModel()
story_model.set_next_output([get_text_message("Final story")])
story_model = ScriptedModel()
story_model.enqueue([get_text_message("Final story")])
outline_agent = Agent(name="outline", model=outline_model)
checker_agent = Agent(
@@ -513,14 +513,14 @@ async def test_deterministic_story_flow_runs_story_on_pass() -> None:
story_result = await Runner.run(story_agent, outline_result.final_output)
assert story_result.final_output == "Final story"
assert story_model.last_turn_args["input"] == [{"content": "Outline ready", "role": "user"}]
assert story_model.calls[-1].input == [{"content": "Outline ready", "role": "user"}]
@pytest.mark.asyncio
async def test_routing_stream_emits_text_and_updates_inputs() -> None:
"""Mimics routing example stream: text deltas flow through and input history updates."""
model = FakeModel()
model.set_next_output([get_text_message("Bonjour")])
model = ScriptedModel()
model.enqueue([get_text_message("Bonjour")])
triage_agent = Agent(name="triage_agent", model=model)
streamed = Runner.run_streamed(triage_agent, input="Salut")
@@ -557,8 +557,8 @@ class MathHomeworkOutput(BaseModel):
@pytest.mark.asyncio
async def test_input_guardrail_agent_trips_and_returns_info() -> None:
"""Mimics math guardrail example: guardrail agent runs and trips before main agent completes."""
guardrail_model = FakeModel()
guardrail_model.set_next_output(
guardrail_model = ScriptedModel()
guardrail_model.enqueue(
[
get_final_output_message(
json.dumps({"reasoning": "math detected", "is_math_homework": True})
@@ -577,8 +577,8 @@ async def test_input_guardrail_agent_trips_and_returns_info() -> None:
output_info=output, tripwire_triggered=output.is_math_homework
)
main_model = FakeModel()
main_model.set_next_output([get_text_message("Should not run")])
main_model = ScriptedModel()
main_model.enqueue([get_text_message("Should not run")])
main_agent = Agent(name="main", model=main_model, input_guardrails=[math_guardrail])
with pytest.raises(InputGuardrailTripwireTriggered) as excinfo:
@@ -610,8 +610,8 @@ async def test_output_guardrail_blocks_sensitive_data() -> None:
tripwire_triggered=contains_phone,
)
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[
get_final_output_message(
json.dumps(
@@ -642,8 +642,8 @@ async def test_output_guardrail_blocks_sensitive_data() -> None:
@pytest.mark.asyncio
async def test_streaming_guardrail_style_cancel_after_threshold() -> None:
"""Mimics streaming guardrail example: stop streaming once threshold is reached."""
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[
get_text_message("Chunk1 "),
get_text_message("Chunk2 "),
@@ -673,8 +673,8 @@ async def test_streaming_guardrail_style_cancel_after_threshold() -> None:
@pytest.mark.asyncio
async def test_streaming_cancel_after_turn_allows_turn_completion() -> None:
"""Ensure cancel(after_turn) lets the current turn finish and final_output is populated."""
model = FakeModel()
model.set_next_output([get_text_message("Hello"), get_text_message("World")])
model = ScriptedModel()
model.enqueue([get_text_message("Hello"), get_text_message("World")])
agent = Agent(name="talkative", model=model)
streamed = Runner.run_streamed(agent, input="Hi")
@@ -696,12 +696,12 @@ async def test_streaming_cancel_after_turn_allows_turn_completion() -> None:
@pytest.mark.asyncio
async def test_streaming_handoff_emits_agent_updated_event() -> None:
"""Mimics routing handoff stream: emits AgentUpdatedStreamEvent and switches agent."""
delegate_model = FakeModel()
delegate_model.set_next_output([get_text_message("delegate reply")])
delegate_model = ScriptedModel()
delegate_model.enqueue([get_text_message("delegate reply")])
delegate_agent = Agent(name="delegate", model=delegate_model)
triage_model = FakeModel()
triage_model.set_next_output(
triage_model = ScriptedModel()
triage_model.enqueue(
[
get_text_message("triage summary"),
get_handoff_tool_call(delegate_agent),
@@ -749,8 +749,8 @@ async def test_agent_as_tool_streaming_example_collects_events() -> None:
billing_tool.on_invoke_tool = fake_invoke
main_model = FakeModel()
main_model.add_multiple_turn_outputs(
main_model = ScriptedModel()
main_model.extend(
[
[get_function_tool_call("billing_agent", json.dumps({"input": "Need bill"}))],
[get_text_message("Final answer")],
@@ -776,8 +776,8 @@ async def test_agent_as_tool_streaming_example_collects_events() -> None:
@pytest.mark.asyncio
async def test_sandbox_agents_as_tools_example_serializes_structured_reviews() -> None:
pricing_model = FakeModel()
pricing_model.set_next_output(
pricing_model = ScriptedModel()
pricing_model.enqueue(
[
get_final_output_message(
json.dumps(
@@ -793,8 +793,8 @@ async def test_sandbox_agents_as_tools_example_serializes_structured_reviews() -
)
]
)
rollout_model = FakeModel()
rollout_model.set_next_output(
rollout_model = ScriptedModel()
rollout_model.enqueue(
[
get_final_output_message(
json.dumps(
@@ -812,8 +812,8 @@ async def test_sandbox_agents_as_tools_example_serializes_structured_reviews() -
)
]
)
orchestrator_model = FakeModel()
orchestrator_model.add_multiple_turn_outputs(
orchestrator_model = ScriptedModel()
orchestrator_model.extend(
[
[
get_function_tool_call(
@@ -878,7 +878,7 @@ async def test_sandbox_agents_as_tools_example_serializes_structured_reviews() -
assert result.final_output == "Recommendation complete"
outer_second_turn_input = cast(
list[dict[str, Any]],
orchestrator_model.last_turn_args["input"],
orchestrator_model.calls[-1].input,
)
outer_tool_outputs = [
item for item in outer_second_turn_input if item.get("type") == "function_call_output"
@@ -962,8 +962,8 @@ async def test_forcing_tool_use_behaviors_align_with_example() -> None:
return f"{city}: Sunny"
# default: run_llm_again -> model responds after tool call
default_model = FakeModel()
default_model.add_multiple_turn_outputs(
default_model = ScriptedModel()
default_model.extend(
[
[
get_text_message("Tool call coming"),
@@ -986,8 +986,8 @@ async def test_forcing_tool_use_behaviors_align_with_example() -> None:
assert len(default_result.raw_responses) == 2
# first_tool: stop_on_first_tool -> final output from first tool result
first_model = FakeModel()
first_model.set_next_output(
first_model = ScriptedModel()
first_model.enqueue(
[
get_text_message("Tool call coming"),
get_function_tool_call("get_weather", json.dumps({"city": "Paris"})),
@@ -1014,8 +1014,8 @@ async def test_forcing_tool_use_behaviors_align_with_example() -> None:
is_final_output=True, final_output=f"Custom:{results[0].output}"
)
custom_model = FakeModel()
custom_model.set_next_output(
custom_model = ScriptedModel()
custom_model.enqueue(
[
get_text_message("Tool call coming"),
get_function_tool_call("get_weather", json.dumps({"city": "Berlin"})),
@@ -1037,12 +1037,12 @@ async def test_forcing_tool_use_behaviors_align_with_example() -> None:
@pytest.mark.asyncio
async def test_routing_multi_turn_continues_with_handoff_agent() -> None:
"""Mimics routing example multi-turn: first handoff, then continue with delegated agent."""
delegate_model = FakeModel()
delegate_model.set_next_output([get_text_message("Bonjour")])
delegate_model = ScriptedModel()
delegate_model.enqueue([get_text_message("Bonjour")])
delegate_agent = Agent(name="delegate", model=delegate_model)
triage_model = FakeModel()
triage_model.add_multiple_turn_outputs(
triage_model = ScriptedModel()
triage_model.extend(
[
[get_handoff_tool_call(delegate_agent)],
[get_text_message("handoff completed")],
@@ -1055,13 +1055,13 @@ async def test_routing_multi_turn_continues_with_handoff_agent() -> None:
assert first_result.last_agent == delegate_agent
# Next user turn continues with delegate.
delegate_model.set_next_output([get_text_message("Encore?")])
delegate_model.enqueue([get_text_message("Encore?")])
follow_up_input = first_result.to_input_list()
follow_up_input.append({"role": "user", "content": "Encore!"})
second_result = await Runner.run(delegate_agent, follow_up_input)
assert second_result.final_output == "Encore?"
assert delegate_model.last_turn_args["input"] == follow_up_input
assert delegate_model.calls[-1].input == follow_up_input
@pytest.mark.asyncio
@@ -1084,19 +1084,19 @@ async def test_agents_as_tools_conditional_enabling_matches_preference() -> None
]
for preference, expected_tools in scenarios:
spanish_model = FakeModel()
spanish_model.set_next_output([get_text_message("ES hola")])
spanish_model = ScriptedModel()
spanish_model.enqueue([get_text_message("ES hola")])
spanish_agent = Agent(name="spanish", model=spanish_model)
french_model = FakeModel()
french_model.set_next_output([get_text_message("FR bonjour")])
french_model = ScriptedModel()
french_model.enqueue([get_text_message("FR bonjour")])
french_agent = Agent(name="french", model=french_model)
italian_model = FakeModel()
italian_model.set_next_output([get_text_message("IT ciao")])
italian_model = ScriptedModel()
italian_model.enqueue([get_text_message("IT ciao")])
italian_agent = Agent(name="italian", model=italian_model)
orchestrator_model = FakeModel()
orchestrator_model = ScriptedModel()
# Build tool calls only for expected tools to avoid missing-tool errors.
tool_calls = [
get_function_tool_call(
@@ -1106,7 +1106,7 @@ async def test_agents_as_tools_conditional_enabling_matches_preference() -> None
)
for tool_name in sorted(expected_tools)
]
orchestrator_model.add_multiple_turn_outputs([tool_calls, [get_text_message("Done")]])
orchestrator_model.extend([tool_calls, [get_text_message("Done")]])
context = AppContext(language_preference=preference)
@@ -1137,35 +1137,35 @@ async def test_agents_as_tools_conditional_enabling_matches_preference() -> None
assert result.final_output == "Done"
assert (
spanish_model.first_turn_args is not None
bool(spanish_model.calls)
if "respond_spanish" in expected_tools
else spanish_model.first_turn_args is None
else not spanish_model.calls
)
assert (
french_model.first_turn_args is not None
bool(french_model.calls)
if "respond_french" in expected_tools
else french_model.first_turn_args is None
else not french_model.calls
)
assert (
italian_model.first_turn_args is not None
bool(italian_model.calls)
if "respond_italian" in expected_tools
else italian_model.first_turn_args is None
else not italian_model.calls
)
@pytest.mark.asyncio
async def test_agents_as_tools_orchestrator_runs_multiple_translations() -> None:
"""Orchestrator calls multiple translation agent tools then summarizes."""
spanish_model = FakeModel()
spanish_model.set_next_output([get_text_message("ES hola")])
spanish_model = ScriptedModel()
spanish_model.enqueue([get_text_message("ES hola")])
spanish_agent = Agent(name="spanish", model=spanish_model)
french_model = FakeModel()
french_model.set_next_output([get_text_message("FR bonjour")])
french_model = ScriptedModel()
french_model.enqueue([get_text_message("FR bonjour")])
french_agent = Agent(name="french", model=french_model)
orchestrator_model = FakeModel()
orchestrator_model.add_multiple_turn_outputs(
orchestrator_model = ScriptedModel()
orchestrator_model.extend(
[
[
get_function_tool_call(
@@ -1197,8 +1197,8 @@ async def test_agents_as_tools_orchestrator_runs_multiple_translations() -> None
result = await Runner.run(orchestrator, "Hi")
assert result.final_output == "Summary complete"
assert spanish_model.last_turn_args["input"] == [{"content": "Hi", "role": "user"}]
assert french_model.last_turn_args["input"] == [{"content": "Hi", "role": "user"}]
assert spanish_model.calls[-1].input == [{"content": "Hi", "role": "user"}]
assert french_model.calls[-1].input == [{"content": "Hi", "role": "user"}]
assert len(result.raw_responses) == 3
@@ -1209,14 +1209,15 @@ async def test_agents_as_tools_subagent_cancellation_preserves_parent_final_outp
async def _cancel_tool() -> str:
raise asyncio.CancelledError("tool-cancelled")
success_model = FakeModel()
success_model.set_next_output([get_text_message("Status: ok")])
success_model = ScriptedModel()
success_model.enqueue([get_text_message("Status: ok")])
success_agent = Agent(name="status", model=success_model)
observability_model = FakeModel()
observability_model.set_next_output(
observability_model = ScriptedModel()
observability_model.enqueue(
[get_function_tool_call("cancel_tool", "{}", call_id="inner_cancel")]
)
observability_model.enqueue([])
observability_agent = Agent(
name="observability",
model=observability_model,
@@ -1224,8 +1225,8 @@ async def test_agents_as_tools_subagent_cancellation_preserves_parent_final_outp
model_settings=ModelSettings(tool_choice="required"),
)
orchestrator_model = FakeModel()
orchestrator_model.add_multiple_turn_outputs(
orchestrator_model = ScriptedModel()
orchestrator_model.extend(
[
[
get_function_tool_call(
@@ -1257,11 +1258,11 @@ async def test_agents_as_tools_subagent_cancellation_preserves_parent_final_outp
assert result.final_output == "Summary complete"
assert len(result.raw_responses) == 2
assert success_model.last_turn_args["input"] == [{"content": "Hi", "role": "user"}]
assert observability_model.first_turn_args is not None
assert observability_model.first_turn_args["input"] == [{"content": "Hi", "role": "user"}]
assert success_model.calls[-1].input == [{"content": "Hi", "role": "user"}]
assert bool(observability_model.calls)
assert observability_model.calls[0].input == [{"content": "Hi", "role": "user"}]
second_turn_input = cast(list[dict[str, Any]], orchestrator_model.last_turn_args["input"])
second_turn_input = cast(list[dict[str, Any]], orchestrator_model.calls[-1].input)
tool_outputs = [
item for item in second_turn_input if item.get("type") == "function_call_output"
]
@@ -1294,12 +1295,12 @@ async def test_agents_as_tools_streaming_subagent_cancellation_preserves_parent_
async def on_stream(event: AgentToolStreamEvent) -> None:
received_events.append(event)
status_model = FakeModel()
status_model.set_next_output([get_text_message("Status: ok")])
status_model = ScriptedModel()
status_model.enqueue([get_text_message("Status: ok")])
status_agent = Agent(name="status", model=status_model)
observability_model = FakeModel()
observability_model.add_multiple_turn_outputs(
observability_model = ScriptedModel()
observability_model.extend(
[
[
get_function_tool_call("ok_tool", "{}", call_id="inner_ok"),
@@ -1318,8 +1319,8 @@ async def test_agents_as_tools_streaming_subagent_cancellation_preserves_parent_
model_settings=ModelSettings(tool_choice="required"),
)
orchestrator_model = FakeModel()
orchestrator_model.add_multiple_turn_outputs(
orchestrator_model = ScriptedModel()
orchestrator_model.extend(
[
[
get_function_tool_call(
@@ -1356,12 +1357,12 @@ async def test_agents_as_tools_streaming_subagent_cancellation_preserves_parent_
assert result.final_output == "Summary complete"
assert len(result.raw_responses) == 2
assert received_events, "on_stream should confirm the nested streaming path ran"
assert status_model.last_turn_args["input"] == [{"content": "Hi", "role": "user"}]
assert observability_model.last_turn_args is not None
assert status_model.calls[-1].input == [{"content": "Hi", "role": "user"}]
assert bool(observability_model.calls)
nested_second_turn_input = cast(
list[dict[str, Any]],
observability_model.last_turn_args["input"],
observability_model.calls[-1].input,
)
nested_tool_outputs = [
item for item in nested_second_turn_input if item.get("type") == "function_call_output"
@@ -1383,7 +1384,7 @@ async def test_agents_as_tools_streaming_subagent_cancellation_preserves_parent_
outer_second_turn_input = cast(
list[dict[str, Any]],
orchestrator_model.last_turn_args["input"],
orchestrator_model.calls[-1].input,
)
outer_tool_outputs = [
item for item in outer_second_turn_input if item.get("type") == "function_call_output"
@@ -1409,12 +1410,12 @@ async def test_agents_as_tools_failure_error_function_none_reraises_cancelled_er
async def _cancel_tool() -> str:
raise asyncio.CancelledError("tool-cancelled")
status_model = FakeModel()
status_model.set_next_output([get_text_message("Status: ok")])
status_model = ScriptedModel()
status_model.enqueue([get_text_message("Status: ok")])
status_agent = Agent(name="status", model=status_model)
observability_model = FakeModel()
observability_model.set_next_output(
observability_model = ScriptedModel()
observability_model.enqueue(
[get_function_tool_call("cancel_tool", "{}", call_id="inner_cancel")]
)
observability_agent = Agent(
@@ -1426,8 +1427,8 @@ async def test_agents_as_tools_failure_error_function_none_reraises_cancelled_er
model_settings=ModelSettings(tool_choice="required"),
)
orchestrator_model = FakeModel()
orchestrator_model.set_next_output(
orchestrator_model = ScriptedModel()
orchestrator_model.enqueue(
[
get_function_tool_call(
"status_agent",
+18 -18
View File
@@ -8,9 +8,9 @@ import pytest
from typing_extensions import TypedDict
from agents import Agent, RunContextWrapper, RunHooks, Runner, TContext, Tool
from agents.testing import ScriptedModel
from agents.tool_context import ToolContext
from .fake_model import FakeModel
from .test_responses import (
get_final_output_message,
get_function_tool,
@@ -75,7 +75,7 @@ class RunHooksForTests(RunHooks):
@pytest.mark.asyncio
async def test_non_streamed_agent_hooks():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(name="test_2", model=model)
agent_3 = Agent(
@@ -87,12 +87,12 @@ async def test_non_streamed_agent_hooks():
agent_1.handoffs.append(agent_3)
model.set_next_output([get_text_message("user_message")])
model.enqueue([get_text_message("user_message")])
output = await Runner.run(agent_3, input="user_message", hooks=hooks)
assert hooks.events == {"on_agent_start": 1, "on_agent_end": 1}, f"{output}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
[get_text_message("done")],
@@ -103,7 +103,7 @@ async def test_non_streamed_agent_hooks():
assert len(set(hooks.tool_context_ids)) == 1
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -127,7 +127,7 @@ async def test_non_streamed_agent_hooks():
}, f"got unexpected event count: {hooks.events}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
@@ -161,7 +161,7 @@ async def test_non_streamed_agent_hooks():
@pytest.mark.asyncio
async def test_streamed_agent_hooks():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(name="test_2", model=model)
agent_3 = Agent(
@@ -173,14 +173,14 @@ async def test_streamed_agent_hooks():
agent_1.handoffs.append(agent_3)
model.set_next_output([get_text_message("user_message")])
model.enqueue([get_text_message("user_message")])
output = Runner.run_streamed(agent_3, input="user_message", hooks=hooks)
async for _ in output.stream_events():
pass
assert hooks.events == {"on_agent_start": 1, "on_agent_end": 1}, f"{output}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -204,7 +204,7 @@ async def test_streamed_agent_hooks():
}, f"got unexpected event count: {hooks.events}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
@@ -243,7 +243,7 @@ class Foo(TypedDict):
@pytest.mark.asyncio
async def test_structured_output_non_streamed_agent_hooks():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(name="test_2", model=model)
agent_3 = Agent(
@@ -256,12 +256,12 @@ async def test_structured_output_non_streamed_agent_hooks():
agent_1.handoffs.append(agent_3)
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
output = await Runner.run(agent_3, input="user_message", hooks=hooks)
assert hooks.events == {"on_agent_start": 1, "on_agent_end": 1}, f"{output}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -284,7 +284,7 @@ async def test_structured_output_non_streamed_agent_hooks():
}, f"got unexpected event count: {hooks.events}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
@@ -316,7 +316,7 @@ async def test_structured_output_non_streamed_agent_hooks():
@pytest.mark.asyncio
async def test_structured_output_streamed_agent_hooks():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(name="test_2", model=model)
agent_3 = Agent(
@@ -329,14 +329,14 @@ async def test_structured_output_streamed_agent_hooks():
agent_1.handoffs.append(agent_3)
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
output = Runner.run_streamed(agent_3, input="user_message", hooks=hooks)
async for _ in output.stream_events():
pass
assert hooks.events == {"on_agent_start": 1, "on_agent_end": 1}, f"{output}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
@@ -360,7 +360,7 @@ async def test_structured_output_streamed_agent_hooks():
}, f"got unexpected event count: {hooks.events}"
hooks.reset()
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
+111 -127
View File
@@ -24,8 +24,8 @@ from agents import (
from agents.guardrail import input_guardrail, output_guardrail
from agents.result import RunResultStreaming
from agents.run_internal.guardrails import run_input_guardrails, run_input_guardrails_with_queue
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import get_function_tool_call, get_text_message
from .testing_processor import fetch_events
@@ -365,14 +365,14 @@ async def test_parallel_guardrail_runs_concurrently_with_agent():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_agent",
instructions="Reply with 'hello'",
input_guardrails=[parallel_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
result = await Runner.run(agent, "test input")
@@ -380,7 +380,7 @@ async def test_parallel_guardrail_runs_concurrently_with_agent():
assert result.final_output is not None
assert len(result.input_guardrail_results) == 1
assert result.input_guardrail_results[0].output.output_info == "parallel_ok"
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
assert bool(model.calls), "Model should have been called in parallel mode"
@pytest.mark.asyncio
@@ -399,14 +399,14 @@ async def test_parallel_guardrail_runs_concurrently_with_agent_streaming():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="streaming_agent",
instructions="Reply with 'hello'",
input_guardrails=[parallel_check],
model=model,
)
model.set_next_output([get_text_message("hello from stream")])
model.enqueue([get_text_message("hello from stream")])
result = Runner.run_streamed(agent, "test input")
@@ -416,7 +416,7 @@ async def test_parallel_guardrail_runs_concurrently_with_agent_streaming():
assert guardrail_executed is True
assert received_events is True
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
assert bool(model.calls), "Model should have been called in parallel mode"
@pytest.mark.asyncio
@@ -435,21 +435,21 @@ async def test_blocking_guardrail_prevents_agent_execution():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_agent",
instructions="Reply with 'hello'",
input_guardrails=[blocking_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
with pytest.raises(InputGuardrailTripwireTriggered) as exc_info:
await Runner.run(agent, "test input")
assert guardrail_executed is True
assert exc_info.value.guardrail_result.output.output_info == "security_violation"
assert model.first_turn_args is None, "Model should not have been called"
assert not model.calls, "Model should not have been called"
@pytest.mark.asyncio
@@ -468,14 +468,14 @@ async def test_blocking_guardrail_prevents_agent_execution_streaming():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="streaming_agent",
instructions="Reply with a long message",
input_guardrails=[blocking_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
result = Runner.run_streamed(agent, "test input")
@@ -484,7 +484,7 @@ async def test_blocking_guardrail_prevents_agent_execution_streaming():
pass
assert guardrail_executed is True
assert model.first_turn_args is None, "Model should not have been called"
assert not model.calls, "Model should not have been called"
@pytest.mark.asyncio
@@ -516,7 +516,7 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="agent_with_tools",
instructions="Call the fast_tool immediately",
@@ -524,8 +524,8 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution():
input_guardrails=[slow_parallel_check],
model=model,
)
model.set_next_output([get_function_tool_call("fast_tool", arguments="{}")])
model.set_next_output([get_text_message("done")])
model.enqueue([get_function_tool_call("fast_tool", arguments="{}")])
model.enqueue([get_text_message("done")])
with pytest.raises(InputGuardrailTripwireTriggered):
await Runner.run(agent, "trigger guardrail")
@@ -534,7 +534,7 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution():
assert tool_was_executed is True, (
"Expected tool to execute before slow parallel guardrail triggered"
)
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
assert bool(model.calls), "Model should have been called in parallel mode"
@pytest.mark.asyncio
@@ -553,7 +553,7 @@ async def test_parallel_guardrail_trip_cancels_model_task():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
original_get_response = model.get_response
async def slow_get_response(*args, **kwargs):
@@ -573,7 +573,7 @@ async def test_parallel_guardrail_trip_cancels_model_task():
input_guardrails=[tripwire_after_model_starts],
model=model,
)
model.set_next_output([get_text_message("should_not_finish")])
model.enqueue([get_text_message("should_not_finish")])
with patch.object(model, "get_response", side_effect=slow_get_response):
with pytest.raises(InputGuardrailTripwireTriggered):
@@ -600,7 +600,7 @@ async def test_parallel_guardrail_trip_compat_mode_does_not_cancel_model_task():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
original_get_response = model.get_response
async def slow_get_response(*args, **kwargs):
@@ -620,7 +620,7 @@ async def test_parallel_guardrail_trip_compat_mode_does_not_cancel_model_task():
input_guardrails=[tripwire_after_model_starts],
model=model,
)
model.set_next_output([get_text_message("should_finish_without_cancel")])
model.enqueue([get_text_message("should_finish_without_cancel")])
with patch.object(model, "get_response", side_effect=slow_get_response):
with patch(
@@ -663,7 +663,7 @@ async def test_model_error_cancels_parallel_input_guardrail_task():
guardrail_cancelled.set()
raise
model = FakeModel()
model = ScriptedModel()
async def boom_get_response(*args, **kwargs):
# Only blow up once the guardrail is genuinely mid-flight.
@@ -704,7 +704,7 @@ async def test_parallel_guardrail_non_tripwire_error_not_swallowed():
await asyncio.wait_for(model_started.wait(), timeout=1)
raise ValueError("guardrail boom")
model = FakeModel()
model = ScriptedModel()
original_get_response = model.get_response
async def slow_get_response(*args, **kwargs):
@@ -724,7 +724,7 @@ async def test_parallel_guardrail_non_tripwire_error_not_swallowed():
input_guardrails=[raising_parallel_check],
model=model,
)
model.set_next_output([get_text_message("should_not_finish")])
model.enqueue([get_text_message("should_not_finish")])
with patch.object(model, "get_response", side_effect=slow_get_response):
with pytest.raises(ValueError, match="guardrail boom"):
@@ -748,7 +748,7 @@ async def test_parallel_guardrail_error_cancels_streaming_model():
await asyncio.wait_for(model_started.wait(), timeout=1)
raise ValueError("guardrail boom")
model = FakeModel()
model = ScriptedModel()
async def blocking_stream_response(*args, **kwargs):
model_started.set()
@@ -797,7 +797,7 @@ async def test_model_error_before_guardrail_error_preserves_stream_finalization(
await asyncio.wait_for(raise_guardrail_error.wait(), timeout=1)
raise ValueError("guardrail boom")
class BlockingCleanupFakeModel(FakeModel):
class BlockingCleanupScriptedModel(ScriptedModel):
async def _cleanup_on_run_end(self, owner: object) -> None:
model_cleanup_started.set()
try:
@@ -807,8 +807,8 @@ async def test_model_error_before_guardrail_error_preserves_stream_finalization(
model_cleanup_cancelled.set()
raise
model = BlockingCleanupFakeModel(tracing_enabled=True)
model.set_next_output(RuntimeError("model boom"))
model = BlockingCleanupScriptedModel(emit_traces=True)
model.enqueue(RuntimeError("model boom"))
agent = Agent(
name="streaming_model_error_agent",
@@ -875,7 +875,7 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution_streaming():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="agent_with_tools",
instructions="Call the fast_tool immediately",
@@ -883,8 +883,8 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution_streaming():
input_guardrails=[slow_parallel_check],
model=model,
)
model.set_next_output([get_function_tool_call("fast_tool", arguments="{}")])
model.set_next_output([get_text_message("done")])
model.enqueue([get_function_tool_call("fast_tool", arguments="{}")])
model.enqueue([get_text_message("done")])
result = Runner.run_streamed(agent, "trigger guardrail")
@@ -896,7 +896,7 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution_streaming():
assert tool_was_executed is True, (
"Expected tool to execute before slow parallel guardrail triggered"
)
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
assert bool(model.calls), "Model should have been called in parallel mode"
@pytest.mark.asyncio
@@ -922,7 +922,7 @@ async def test_parallel_guardrail_trip_before_tool_execution_stops_streaming_tur
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
original_stream_response = model.stream_response
async def delayed_stream_response(*args, **kwargs):
@@ -939,8 +939,8 @@ async def test_parallel_guardrail_trip_before_tool_execution_stops_streaming_tur
input_guardrails=[tripwire_before_tool_execution],
model=model,
)
model.set_next_output([get_function_tool_call("dangerous_tool", arguments="{}")])
model.set_next_output([get_text_message("done")])
model.enqueue([get_function_tool_call("dangerous_tool", arguments="{}")])
model.enqueue([get_text_message("done")])
with patch.object(model, "stream_response", side_effect=delayed_stream_response):
result = Runner.run_streamed(agent, "trigger guardrail")
@@ -952,7 +952,7 @@ async def test_parallel_guardrail_trip_before_tool_execution_stops_streaming_tur
assert model_started.is_set() is True
assert guardrail_tripped.is_set() is True
assert tool_was_executed is False
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
assert bool(model.calls), "Model should have been called in parallel mode"
@pytest.mark.asyncio
@@ -996,7 +996,7 @@ async def test_parallel_guardrail_trip_with_slow_cancel_sibling_stops_streaming_
slow_cancel_finished.set()
raise
model = FakeModel()
model = ScriptedModel()
original_stream_response = model.stream_response
async def delayed_stream_response(*args, **kwargs):
@@ -1013,8 +1013,8 @@ async def test_parallel_guardrail_trip_with_slow_cancel_sibling_stops_streaming_
input_guardrails=[tripwire_before_tool_execution, slow_to_cancel_guardrail],
model=model,
)
model.set_next_output([get_function_tool_call("dangerous_tool", arguments="{}")])
model.set_next_output([get_text_message("done")])
model.enqueue([get_function_tool_call("dangerous_tool", arguments="{}")])
model.enqueue([get_text_message("done")])
with patch.object(model, "stream_response", side_effect=delayed_stream_response):
result = Runner.run_streamed(agent, "trigger guardrail")
@@ -1033,7 +1033,7 @@ async def test_parallel_guardrail_trip_with_slow_cancel_sibling_stops_streaming_
assert slow_cancel_started.is_set() is True
assert slow_cancel_finished.is_set() is True
assert tool_was_executed is False
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
assert bool(model.calls), "Model should have been called in parallel mode"
@pytest.mark.asyncio
@@ -1059,7 +1059,7 @@ async def test_blocking_guardrail_prevents_tool_execution():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="agent_with_tools",
instructions="Call the dangerous_tool immediately",
@@ -1067,14 +1067,14 @@ async def test_blocking_guardrail_prevents_tool_execution():
input_guardrails=[security_check],
model=model,
)
model.set_next_output([get_function_tool_call("dangerous_tool", arguments="{}")])
model.enqueue([get_function_tool_call("dangerous_tool", arguments="{}")])
with pytest.raises(InputGuardrailTripwireTriggered):
await Runner.run(agent, "trigger guardrail")
assert guardrail_executed is True
assert tool_was_executed is False
assert model.first_turn_args is None, "Model should not have been called"
assert not model.calls, "Model should not have been called"
@pytest.mark.asyncio
@@ -1100,7 +1100,7 @@ async def test_blocking_guardrail_prevents_tool_execution_streaming():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="agent_with_tools",
instructions="Call the dangerous_tool immediately",
@@ -1108,7 +1108,7 @@ async def test_blocking_guardrail_prevents_tool_execution_streaming():
input_guardrails=[security_check],
model=model,
)
model.set_next_output([get_function_tool_call("dangerous_tool", arguments="{}")])
model.enqueue([get_function_tool_call("dangerous_tool", arguments="{}")])
result = Runner.run_streamed(agent, "trigger guardrail")
@@ -1118,7 +1118,7 @@ async def test_blocking_guardrail_prevents_tool_execution_streaming():
assert guardrail_executed is True
assert tool_was_executed is False
assert model.first_turn_args is None, "Model should not have been called"
assert not model.calls, "Model should not have been called"
@pytest.mark.asyncio
@@ -1137,20 +1137,20 @@ async def test_parallel_guardrail_passes_agent_continues():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_agent",
instructions="Reply with 'success'",
input_guardrails=[parallel_check],
model=model,
)
model.set_next_output([get_text_message("success")])
model.enqueue([get_text_message("success")])
result = await Runner.run(agent, "test input")
assert guardrail_executed is True
assert result.final_output is not None
assert model.first_turn_args is not None, "Model should have been called"
assert bool(model.calls), "Model should have been called"
@pytest.mark.asyncio
@@ -1169,14 +1169,14 @@ async def test_parallel_guardrail_passes_agent_continues_streaming():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_agent",
instructions="Reply with 'success'",
input_guardrails=[parallel_check],
model=model,
)
model.set_next_output([get_text_message("success")])
model.enqueue([get_text_message("success")])
result = Runner.run_streamed(agent, "test input")
@@ -1186,7 +1186,7 @@ async def test_parallel_guardrail_passes_agent_continues_streaming():
assert guardrail_executed is True
assert received_events is True
assert model.first_turn_args is not None, "Model should have been called"
assert bool(model.calls), "Model should have been called"
@pytest.mark.asyncio
@@ -1205,20 +1205,20 @@ async def test_blocking_guardrail_passes_agent_continues():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_agent",
instructions="Reply with 'success'",
input_guardrails=[blocking_check],
model=model,
)
model.set_next_output([get_text_message("success")])
model.enqueue([get_text_message("success")])
result = await Runner.run(agent, "test input")
assert guardrail_executed is True
assert result.final_output is not None
assert model.first_turn_args is not None, "Model should have been called after guardrail passed"
assert bool(model.calls), "Model should have been called after guardrail passed"
@pytest.mark.asyncio
@@ -1237,14 +1237,14 @@ async def test_blocking_guardrail_passes_agent_continues_streaming():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_agent",
instructions="Reply with 'success'",
input_guardrails=[blocking_check],
model=model,
)
model.set_next_output([get_text_message("success")])
model.enqueue([get_text_message("success")])
result = Runner.run_streamed(agent, "test input")
@@ -1254,7 +1254,7 @@ async def test_blocking_guardrail_passes_agent_continues_streaming():
assert guardrail_executed is True
assert received_events is True
assert model.first_turn_args is not None, "Model should have been called after guardrail passed"
assert bool(model.calls), "Model should have been called after guardrail passed"
@pytest.mark.asyncio
@@ -1289,7 +1289,7 @@ async def test_mixed_blocking_and_parallel_guardrails():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
original_get_response = model.get_response
@@ -1306,7 +1306,7 @@ async def test_mixed_blocking_and_parallel_guardrails():
input_guardrails=[blocking_check, parallel_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
with patch.object(model, "get_response", side_effect=tracked_get_response):
result = await Runner.run(agent, "test input")
@@ -1324,9 +1324,7 @@ async def test_mixed_blocking_and_parallel_guardrails():
"Model called while parallel guardrail still running"
)
assert parallel_finished.is_set() is True, "Parallel guardrail should have completed"
assert model.first_turn_args is not None, (
"Model should have been called after blocking guardrails passed"
)
assert bool(model.calls), "Model should have been called after blocking guardrails passed"
@pytest.mark.asyncio
@@ -1361,7 +1359,7 @@ async def test_mixed_blocking_and_parallel_guardrails_streaming():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
original_stream_response = model.stream_response
@@ -1379,7 +1377,7 @@ async def test_mixed_blocking_and_parallel_guardrails_streaming():
input_guardrails=[blocking_check, parallel_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
with patch.object(model, "stream_response", side_effect=tracked_stream_response):
result = Runner.run_streamed(agent, "test input")
@@ -1399,9 +1397,7 @@ async def test_mixed_blocking_and_parallel_guardrails_streaming():
"Model called while parallel guardrail still running"
)
assert parallel_finished.is_set() is True, "Parallel guardrail should have completed"
assert model.first_turn_args is not None, (
"Model should have been called after blocking guardrails passed"
)
assert bool(model.calls), "Model should have been called after blocking guardrails passed"
@pytest.mark.asyncio
@@ -1432,7 +1428,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
original_get_response = model.get_response
@@ -1446,7 +1442,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent():
input_guardrails=[first_blocking_check, second_blocking_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
with patch.object(model, "get_response", side_effect=tracked_get_response):
result = await Runner.run(agent, "test input")
@@ -1466,9 +1462,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent():
assert timestamps["second_blocking_end"] <= timestamps["model_called"], (
"Second blocking guardrail must complete before model is called"
)
assert model.first_turn_args is not None, (
"Model should have been called after all blocking guardrails passed"
)
assert bool(model.calls), "Model should have been called after all blocking guardrails passed"
@pytest.mark.asyncio
@@ -1499,7 +1493,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent_streaming():
tripwire_triggered=False,
)
model = FakeModel()
model = ScriptedModel()
original_stream_response = model.stream_response
@@ -1514,7 +1508,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent_streaming():
input_guardrails=[first_blocking_check, second_blocking_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
with patch.object(model, "stream_response", side_effect=tracked_stream_response):
result = Runner.run_streamed(agent, "test input")
@@ -1536,9 +1530,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent_streaming():
assert timestamps["second_blocking_end"] <= timestamps["model_called"], (
"Second blocking guardrail must complete before model is called"
)
assert model.first_turn_args is not None, (
"Model should have been called after all blocking guardrails passed"
)
assert bool(model.calls), "Model should have been called after all blocking guardrails passed"
@pytest.mark.asyncio
@@ -1575,14 +1567,14 @@ async def test_multiple_blocking_guardrails_one_triggers():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="multi_blocking_agent",
instructions="Reply with 'hello'",
input_guardrails=[first_blocking_check, second_blocking_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
with pytest.raises(InputGuardrailTripwireTriggered):
await Runner.run(agent, "test input")
@@ -1593,9 +1585,7 @@ async def test_multiple_blocking_guardrails_one_triggers():
assert "first_blocking_end" in timestamps
assert "second_blocking_start" in timestamps
assert "second_blocking_end" in timestamps
assert model.first_turn_args is None, (
"Model should not have been called when guardrail triggered"
)
assert not model.calls, "Model should not have been called when guardrail triggered"
@pytest.mark.asyncio
@@ -1632,14 +1622,14 @@ async def test_multiple_blocking_guardrails_one_triggers_streaming():
tripwire_triggered=True,
)
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="multi_blocking_agent",
instructions="Reply with 'hello'",
input_guardrails=[first_blocking_check, second_blocking_check],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
result = Runner.run_streamed(agent, "test input")
@@ -1653,9 +1643,7 @@ async def test_multiple_blocking_guardrails_one_triggers_streaming():
assert "first_blocking_end" in timestamps
assert "second_blocking_start" in timestamps
assert "second_blocking_end" in timestamps
assert model.first_turn_args is None, (
"Model should not have been called when guardrail triggered"
)
assert not model.calls, "Model should not have been called when guardrail triggered"
@pytest.mark.asyncio
@@ -1685,22 +1673,22 @@ async def test_guardrail_via_agent_and_run_config_equivalent():
tripwire_triggered=False,
)
model1 = FakeModel()
model1 = ScriptedModel()
agent_with_guardrail = Agent(
name="test_agent",
instructions="Reply with 'hello'",
input_guardrails=[agent_level_check],
model=model1,
)
model1.set_next_output([get_text_message("hello")])
model1.enqueue([get_text_message("hello")])
model2 = FakeModel()
model2 = ScriptedModel()
agent_without_guardrail = Agent(
name="test_agent",
instructions="Reply with 'hello'",
model=model2,
)
model2.set_next_output([get_text_message("hello")])
model2.enqueue([get_text_message("hello")])
run_config = RunConfig(input_guardrails=[config_level_check])
result1 = await Runner.run(agent_with_guardrail, "test input")
@@ -1714,8 +1702,8 @@ async def test_guardrail_via_agent_and_run_config_equivalent():
assert result2.input_guardrail_results[0].output.output_info == "config_level_passed"
assert result1.final_output is not None
assert result2.final_output is not None
assert model1.first_turn_args is not None
assert model2.first_turn_args is not None
assert bool(model1.calls)
assert bool(model2.calls)
@pytest.mark.asyncio
@@ -1760,14 +1748,14 @@ async def test_blocking_guardrail_cancels_remaining_on_trigger():
slow_guardrail_cancelled = True
raise
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_agent",
instructions="Reply with 'hello'",
input_guardrails=[fast_guardrail_that_triggers, slow_guardrail_that_should_be_cancelled],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
with pytest.raises(InputGuardrailTripwireTriggered):
await asyncio.wait_for(Runner.run(agent, "test input"), timeout=5)
@@ -1780,9 +1768,7 @@ async def test_blocking_guardrail_cancels_remaining_on_trigger():
assert slow_guardrail_executed is False, "Slow guardrail should NOT have completed execution"
# Verify agent never started
assert model.first_turn_args is None, (
"Model should not have been called when guardrail triggered"
)
assert not model.calls, "Model should not have been called when guardrail triggered"
@pytest.mark.asyncio
@@ -1827,14 +1813,14 @@ async def test_blocking_guardrail_cancels_remaining_on_trigger_streaming():
slow_guardrail_cancelled = True
raise
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_agent",
instructions="Reply with 'hello'",
input_guardrails=[fast_guardrail_that_triggers, slow_guardrail_that_should_be_cancelled],
model=model,
)
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
result = Runner.run_streamed(agent, "test input")
@@ -1853,9 +1839,7 @@ async def test_blocking_guardrail_cancels_remaining_on_trigger_streaming():
assert slow_guardrail_executed is False, "Slow guardrail should NOT have completed execution"
# Verify agent never started
assert model.first_turn_args is None, (
"Model should not have been called when guardrail triggered"
)
assert not model.calls, "Model should not have been called when guardrail triggered"
@pytest.mark.asyncio
@@ -1882,7 +1866,7 @@ async def test_streaming_input_guardrail_exception_awaits_cancelled_siblings():
await slow_started.wait()
raise RuntimeError("guardrail failed")
agent = Agent(name="test_agent", model=FakeModel())
agent = Agent(name="test_agent", model=ScriptedModel())
context = RunContextWrapper(context=None)
streamed_result = RunResultStreaming(
"test input",
@@ -2021,7 +2005,7 @@ def _ordered_input_guardrails(
]
def _tripwire_agent(model: FakeModel, *, run_in_parallel: bool) -> Agent[Any]:
def _tripwire_agent(model: ScriptedModel, *, run_in_parallel: bool) -> Agent[Any]:
return Agent(
name="guardrail_results_agent",
model=model,
@@ -2039,8 +2023,8 @@ def _result_names(results: list[Any]) -> list[str]:
@pytest.mark.parametrize("run_in_parallel", [False, True])
async def test_input_guardrail_tripwire_reports_results(run_in_parallel: bool):
"""Runner.run() reports every completed guardrail result on the raised tripwire."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
with pytest.raises(InputGuardrailTripwireTriggered) as exc_info:
await Runner.run(_tripwire_agent(model, run_in_parallel=run_in_parallel), "test input")
@@ -2055,8 +2039,8 @@ async def test_input_guardrail_tripwire_reports_results(run_in_parallel: bool):
@pytest.mark.parametrize("run_in_parallel", [False, True])
async def test_input_guardrail_tripwire_reports_results_streamed(run_in_parallel: bool):
"""The streamed path reports the same results, including on the streamed result object."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
result = Runner.run_streamed(
_tripwire_agent(model, run_in_parallel=run_in_parallel), "test input"
@@ -2073,8 +2057,8 @@ async def test_input_guardrail_tripwire_reports_results_streamed(run_in_parallel
def test_input_guardrail_tripwire_reports_results_sync():
"""Runner.run_sync() matches the async entry points."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
with pytest.raises(InputGuardrailTripwireTriggered) as exc_info:
Runner.run_sync(_tripwire_agent(model, run_in_parallel=False), "test input")
@@ -2087,8 +2071,8 @@ def test_input_guardrail_tripwire_reports_results_sync():
@pytest.mark.asyncio
async def test_input_guardrail_results_reported_on_success():
"""Passing guardrails still land on the successful result exactly once."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
agent = Agent(
name="guardrail_results_agent",
model=model,
@@ -2134,8 +2118,8 @@ async def test_input_guardrail_exception_reports_completed_results_streamed(
run_in_parallel: bool,
):
"""A streamed guardrail raising a non-tripwire error still reports earlier results."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
agent = Agent(
name="guardrail_results_agent",
model=model,
@@ -2180,7 +2164,7 @@ def _ordered_output_guardrails(
]
def _output_tripwire_agent(model: FakeModel) -> Agent[Any]:
def _output_tripwire_agent(model: ScriptedModel) -> Agent[Any]:
return Agent(
name="output_guardrail_results_agent",
model=model,
@@ -2191,8 +2175,8 @@ def _output_tripwire_agent(model: FakeModel) -> Agent[Any]:
@pytest.mark.asyncio
async def test_output_guardrail_tripwire_reports_results():
"""Runner.run() reports every completed output guardrail result on the raised tripwire."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
with pytest.raises(OutputGuardrailTripwireTriggered) as exc_info:
await Runner.run(_output_tripwire_agent(model), "test input")
@@ -2206,8 +2190,8 @@ async def test_output_guardrail_tripwire_reports_results():
@pytest.mark.asyncio
async def test_output_guardrail_tripwire_reports_results_streamed():
"""The streamed path reports the same results, including on the streamed result object."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
result = Runner.run_streamed(_output_tripwire_agent(model), "test input")
with pytest.raises(OutputGuardrailTripwireTriggered) as exc_info:
@@ -2222,8 +2206,8 @@ async def test_output_guardrail_tripwire_reports_results_streamed():
def test_output_guardrail_tripwire_reports_results_sync():
"""Runner.run_sync() matches the async entry points."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
with pytest.raises(OutputGuardrailTripwireTriggered) as exc_info:
Runner.run_sync(_output_tripwire_agent(model), "test input")
@@ -2236,8 +2220,8 @@ def test_output_guardrail_tripwire_reports_results_sync():
@pytest.mark.asyncio
async def test_output_guardrail_results_reported_on_success():
"""Passing output guardrails still land on the successful result exactly once."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
agent = Agent(
name="output_guardrail_results_agent",
model=model,
@@ -2270,8 +2254,8 @@ async def test_output_guardrail_exception_reports_completed_results():
@pytest.mark.asyncio
async def test_output_guardrail_exception_reports_completed_results_streamed():
"""A streamed output guardrail raising a non-tripwire error still reports earlier results."""
model = FakeModel()
model.set_next_output([get_text_message("hello")])
model = ScriptedModel()
model.enqueue([get_text_message("hello")])
agent = Agent(
name="output_guardrail_results_agent",
model=model,
+176 -243
View File
@@ -63,8 +63,9 @@ from agents.run_internal.items import (
from agents.run_internal.session_persistence import (
resolve_nested_history_owned_session_item_refs,
)
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .model_test_helpers import get_exact_output_stream_step
from .test_responses import get_function_tool_call, get_handoff_tool_call, get_text_message
from .utils.simple_session import SimpleListSession
@@ -528,16 +529,14 @@ class TestHandoffHistoryDuplicationFix:
@pytest.mark.asyncio
async def test_to_input_list_normalized_uses_filtered_continuation_after_nested_handoff() -> None:
triage_model = FakeModel()
delegate_model = FakeModel()
triage_model = ScriptedModel()
delegate_model = ScriptedModel()
delegate = Agent(name="delegate", model=delegate_model)
triage = Agent(name="triage", model=triage_model, handoffs=[delegate])
triage_model.add_multiple_turn_outputs(
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
)
delegate_model.add_multiple_turn_outputs(
triage_model.extend([[get_text_message("triage summary"), get_handoff_tool_call(delegate)]])
delegate_model.extend(
[
[get_text_message("resolution")],
[get_text_message("followup answer")],
@@ -567,13 +566,13 @@ async def test_to_input_list_normalized_uses_filtered_continuation_after_nested_
assert "function_call" not in normalized_types
assert "function_call_output" not in normalized_types
replay_model = FakeModel()
replay_model = ScriptedModel()
replay_agent = Agent(name="replay", model=replay_model)
replay_model.add_multiple_turn_outputs([[get_text_message("replayed")]])
replay_model.extend([[get_text_message("replayed")]])
replay_result = await Runner.run(replay_agent, input=preserve_all_input)
assert replay_model.first_turn_args is not None
replay_input = replay_model.first_turn_args["input"]
assert bool(replay_model.calls)
replay_input = replay_model.calls[0].input
assert isinstance(replay_input, list)
assert sum(_input_item_text(item) == "triage summary" for item in replay_input) == 1
assert replay_result.final_output == "replayed"
@@ -582,7 +581,7 @@ async def test_to_input_list_normalized_uses_filtered_continuation_after_nested_
follow_up_result = await Runner.run(delegate, input=follow_up_input)
assert follow_up_result.final_output == "followup answer"
assert delegate_model.last_turn_args["input"] == follow_up_input
assert delegate_model.calls[-1].input == follow_up_input
@pytest.mark.asyncio
@@ -590,8 +589,8 @@ async def test_to_input_list_normalized_keeps_delegate_tool_items_after_nested_h
async def lookup_weather(city: str) -> str:
return f"weather:{city}"
triage_model = FakeModel()
delegate_model = FakeModel()
triage_model = ScriptedModel()
delegate_model = ScriptedModel()
delegate = Agent(
name="delegate",
@@ -600,10 +599,8 @@ async def test_to_input_list_normalized_keeps_delegate_tool_items_after_nested_h
)
triage = Agent(name="triage", model=triage_model, handoffs=[delegate])
triage_model.add_multiple_turn_outputs(
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
)
delegate_model.add_multiple_turn_outputs(
triage_model.extend([[get_text_message("triage summary"), get_handoff_tool_call(delegate)]])
delegate_model.extend(
[
[
get_text_message("delegate preamble"),
@@ -659,8 +656,8 @@ async def test_to_input_list_normalized_uses_custom_filter_input_items() -> None
)
)
triage_model = FakeModel()
delegate_model = FakeModel()
triage_model = ScriptedModel()
delegate_model = ScriptedModel()
delegate = Agent(name="delegate", model=delegate_model)
triage = Agent(
@@ -669,10 +666,8 @@ async def test_to_input_list_normalized_uses_custom_filter_input_items() -> None
handoffs=[handoff(delegate, input_filter=keep_messages_only)],
)
triage_model.add_multiple_turn_outputs(
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
)
delegate_model.add_multiple_turn_outputs([[get_text_message("resolution")]])
triage_model.extend([[get_text_message("triage summary"), get_handoff_tool_call(delegate)]])
delegate_model.extend([[get_text_message("resolution")]])
result = await Runner.run(triage, input="user_question")
preserve_all_input = result.to_input_list()
@@ -702,18 +697,16 @@ async def test_non_nested_filtered_handoff_does_not_add_occurrence_lineage(
def identity_filter(data: HandoffInputData) -> HandoffInputData:
return data
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=identity_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
if streamed:
streamed_result = Runner.run_streamed(first_agent, input="start")
@@ -745,18 +738,16 @@ async def test_custom_filter_summary_shape_does_not_claim_equal_session_item() -
input_items=(),
)
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=custom_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(first_agent, input="start")
@@ -766,14 +757,12 @@ async def test_custom_filter_summary_shape_does_not_claim_equal_session_item() -
@pytest.mark.asyncio
async def test_wrapper_reset_does_not_change_nested_history_ownership() -> None:
"""Replay ownership must not depend on wrappers that are current after the run."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
set_conversation_history_wrappers(start="<<START>>", end="<<END>>")
try:
@@ -797,18 +786,16 @@ async def test_public_nested_history_filter_preserves_ownership(chained: bool) -
return remove_all_tools(nest_handoff_history(data))
input_filter = chained_filter if chained else nest_handoff_history
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=input_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(first_agent, input="start")
@@ -824,18 +811,16 @@ async def test_public_nested_history_filter_preserves_ownership_after_deepcopy()
assert not isinstance(nested.input_history, str)
return nested.clone(input_history=deepcopy(nested.input_history))
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=copied_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(first_agent, input="start")
@@ -856,18 +841,16 @@ async def test_public_nested_history_filter_preserves_ownership_after_dict_rebui
)
)
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=rebuilt_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(first_agent, input="start")
@@ -891,18 +874,16 @@ async def test_public_nested_history_filter_rebases_owned_input_after_insertion(
)
return nested.clone(input_history=(inserted, *rebuilt_history))
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=inserting_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
if streamed:
streamed_result = Runner.run_streamed(first_agent, input="start")
@@ -930,18 +911,16 @@ async def test_public_nested_history_filter_data_rebuild_drops_private_ownership
input_items=nested.input_items,
)
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=rebuilt_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(first_agent, input="start")
@@ -956,18 +935,16 @@ async def test_public_nested_history_filter_preserves_ownership_after_new_items_
nested = nest_handoff_history(data)
return nested.clone(new_items=deepcopy(nested.new_items))
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=copied_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(first_agent, input="start")
@@ -987,18 +964,16 @@ async def test_public_nested_history_filter_does_not_own_equal_replacement() ->
)
return nested.clone(new_items=(replacement, *nested.new_items[1:]))
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=replacement_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(first_agent, input="start")
@@ -1060,18 +1035,16 @@ async def test_nested_history_preserves_repeated_run_item_reference_occurrences(
message = data.new_items[0]
return nest_handoff_history(data.clone(new_items=(message, message, *data.new_items[1:])))
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
model=first_model,
handoffs=[handoff(second_agent, input_filter=duplicate_message_filter)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
if streamed:
streamed_result = Runner.run_streamed(first_agent, input="start")
@@ -1091,8 +1064,8 @@ async def test_nested_history_retains_forwarded_pre_handoff_item_provenance(
streamed: bool,
) -> None:
"""Lossless items from earlier turns must retain one replay occurrence."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
tool_search_call = ResponseToolSearchCall(
@@ -1111,13 +1084,14 @@ async def test_nested_history_retains_forwarded_pre_handoff_item_provenance(
tools=[],
type="tool_search_output",
)
first_model.add_multiple_turn_outputs(
first_output = [tool_search_call, tool_search_output]
first_model.extend(
[
[tool_search_call, tool_search_output],
get_exact_output_stream_step(first_output) if streamed else first_output,
[get_handoff_tool_call(second_agent)],
]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
second_model.extend([[get_text_message("done")]])
run_config = RunConfig(nest_handoff_history=True)
run_result: RunResult | RunResultStreaming
@@ -1156,14 +1130,12 @@ async def test_nested_history_retains_forwarded_pre_handoff_item_provenance(
@pytest.mark.asyncio
async def test_to_input_list_during_active_stream_does_not_mutate_input() -> None:
"""Inspecting an active stream must not mutate its eventual public input."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = Runner.run_streamed(
first_agent,
@@ -1186,14 +1158,12 @@ async def test_to_input_list_during_active_stream_does_not_mutate_input() -> Non
@pytest.mark.asyncio
async def test_nested_history_ownership_remaps_after_new_items_insertion() -> None:
"""A caller inserting a public new_items entry must not make ownership drop the new item."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(
first_agent,
@@ -1210,14 +1180,12 @@ async def test_nested_history_ownership_remaps_after_new_items_insertion() -> No
@pytest.mark.asyncio
async def test_nested_history_input_removal_does_not_claim_an_unmarked_equal_occurrence() -> None:
"""An equal replacement input must not retain ownership of the removed occurrence."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(
first_agent,
@@ -1237,14 +1205,12 @@ async def test_nested_history_input_removal_does_not_claim_an_unmarked_equal_occ
@pytest.mark.asyncio
async def test_nested_history_new_item_removal_does_not_claim_an_equal_item() -> None:
"""Removing the owned RunItem must not transfer ownership to an equal RunItem."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(
first_agent,
@@ -1272,14 +1238,12 @@ async def test_nested_history_new_item_removal_does_not_claim_an_equal_item() ->
@pytest.mark.asyncio
async def test_nested_history_ownership_survives_result_new_items_copy() -> None:
"""Copying result run items must not replay a nested session occurrence twice."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(
first_agent,
@@ -1299,14 +1263,12 @@ async def test_nested_history_input_copy_does_not_infer_occurrence_ownership(
streamed: bool,
) -> None:
"""An unmarked public-input copy must remain distinct from its session occurrence."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result: RunResult | RunResultStreaming
if streamed:
@@ -1341,14 +1303,12 @@ async def test_nested_history_input_copy_and_reorder_does_not_infer_ownership(
streamed: bool,
) -> None:
"""Payload equality must not transfer ownership after a copied input reorder."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("owned once"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("owned once"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
run_config = RunConfig(nest_handoff_history=True)
result: RunResult | RunResultStreaming
@@ -1380,14 +1340,12 @@ async def test_result_input_mutation_does_not_change_state_snapshot_ownership(
def approval_tool() -> str:
return "approved"
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model, tools=[approval_tool])
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("owned once"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs(
first_model.extend([[get_text_message("owned once"), get_handoff_tool_call(second_agent)]])
second_model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="approval")],
[get_text_message("done")],
@@ -1438,14 +1396,12 @@ async def test_result_input_mutation_does_not_change_state_snapshot_ownership(
@pytest.mark.asyncio
async def test_nested_history_ownership_revalidates_after_input_removal() -> None:
"""Removing an owned input occurrence must restore its session copy during replay."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
result = await Runner.run(
first_agent,
@@ -1566,9 +1522,9 @@ def test_nested_history_normalizes_forwarded_status_before_ownership() -> None:
@pytest.mark.asyncio
async def test_plain_handoff_preserves_prior_nested_history_ownership(streamed: bool) -> None:
"""A later non-nesting handoff must not clear ownership established by an earlier handoff."""
first_model = FakeModel()
second_model = FakeModel()
final_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
final_model = ScriptedModel()
final_agent = Agent(name="final", model=final_model)
second_agent = Agent(
name="second",
@@ -1576,13 +1532,9 @@ async def test_plain_handoff_preserves_prior_nested_history_ownership(streamed:
handoffs=[handoff(final_agent, nest_handoff_history=False)],
)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("first message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs(
[[get_text_message("second message"), get_handoff_tool_call(final_agent)]]
)
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("first message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("second message"), get_handoff_tool_call(final_agent)]])
final_model.extend([[get_text_message("done")]])
run_config = RunConfig(nest_handoff_history=True)
if streamed:
@@ -1609,9 +1561,9 @@ async def test_non_nested_copying_filter_preserves_prior_nested_history_ownershi
return data
return data.clone(input_history=deepcopy(data.input_history))
first_model = FakeModel()
second_model = FakeModel()
final_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
final_model = ScriptedModel()
final_agent = Agent(name="final", model=final_model)
second_agent = Agent(
name="second",
@@ -1625,13 +1577,9 @@ async def test_non_nested_copying_filter_preserves_prior_nested_history_ownershi
],
)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("first message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs(
[[get_text_message("second message"), get_handoff_tool_call(final_agent)]]
)
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("first message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("second message"), get_handoff_tool_call(final_agent)]])
final_model.extend([[get_text_message("done")]])
run_config = RunConfig(nest_handoff_history=True)
if streamed:
@@ -1660,9 +1608,9 @@ async def test_copied_custom_input_items_keep_session_occurrence_for_later_nesti
def copy_model_items(data: HandoffInputData) -> HandoffInputData:
return data.clone(input_items=deepcopy(data.new_items))
first_model = FakeModel()
second_model = FakeModel()
final_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
final_model = ScriptedModel()
final_agent = Agent(name="final", model=final_model)
second_agent = Agent(name="second", model=second_model, handoffs=[final_agent])
first_agent = Agent(
@@ -1670,11 +1618,9 @@ async def test_copied_custom_input_items_keep_session_occurrence_for_later_nesti
model=first_model,
handoffs=[handoff(second_agent, input_filter=copy_model_items)],
)
first_model.add_multiple_turn_outputs(
[[get_text_message("copied once"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_handoff_tool_call(final_agent)]])
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("copied once"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_handoff_tool_call(final_agent)]])
final_model.extend([[get_text_message("done")]])
run_config = RunConfig(nest_handoff_history=True)
if streamed:
@@ -1697,9 +1643,9 @@ async def test_identity_filter_preserves_prior_nested_history_ownership(streamed
def identity_filter(data: HandoffInputData) -> HandoffInputData:
return data
first_model = FakeModel()
second_model = FakeModel()
final_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
final_model = ScriptedModel()
final_agent = Agent(name="final", model=final_model)
second_agent = Agent(
name="second",
@@ -1707,13 +1653,9 @@ async def test_identity_filter_preserves_prior_nested_history_ownership(streamed
handoffs=[handoff(final_agent, input_filter=identity_filter)],
)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("first message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs(
[[get_text_message("second message"), get_handoff_tool_call(final_agent)]]
)
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("first message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("second message"), get_handoff_tool_call(final_agent)]])
final_model.extend([[get_text_message("done")]])
run_config = RunConfig(nest_handoff_history=True)
if streamed:
@@ -1739,9 +1681,9 @@ async def test_nested_handoff_history_preserves_identical_messages_across_turns(
def continue_work() -> str:
return "continue"
first_model = FakeModel()
second_model = FakeModel()
final_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
final_model = ScriptedModel()
final_agent = Agent(name="final", model=final_model)
second_agent = Agent(
name="second",
@@ -1751,16 +1693,14 @@ async def test_nested_handoff_history_preserves_identical_messages_across_turns(
)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs(
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
second_model.extend(
[
[get_text_message("same"), get_function_tool_call("continue_work", "{}")],
[get_text_message("same"), get_handoff_tool_call(final_agent)],
]
)
final_model.add_multiple_turn_outputs([[get_text_message("same")]])
final_model.extend([[get_text_message("same")]])
if streamed:
streamed_result = Runner.run_streamed(
@@ -1779,7 +1719,7 @@ async def test_nested_handoff_history_preserves_identical_messages_across_turns(
)
replay_input = result.to_input_list()
final_input = final_model.last_turn_args["input"]
final_input = final_model.calls[-1].input
summary = str(cast(dict[str, Any], final_input[0])["content"])
assert summary.count("same") == 2
assert sum(_input_item_text(item) == "same" for item in replay_input) == 4
@@ -1788,15 +1728,13 @@ async def test_nested_handoff_history_preserves_identical_messages_across_turns(
@pytest.mark.asyncio
async def test_explicit_default_handoff_history_mapper_is_honored() -> None:
"""An explicitly configured mapper should own the exact model input."""
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
first_model.add_multiple_turn_outputs(
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
second_model.extend([[get_text_message("done")]])
await Runner.run(
first_agent,
@@ -1807,8 +1745,8 @@ async def test_explicit_default_handoff_history_mapper_is_honored() -> None:
),
)
assert second_model.first_turn_args is not None
second_input = second_model.first_turn_args["input"]
assert bool(second_model.calls)
second_input = second_model.calls[0].input
assert isinstance(second_input, list)
assert len(second_input) == 1
summary = str(cast(dict[str, Any], second_input[0])["content"])
@@ -1824,9 +1762,9 @@ async def test_nested_handoff_history_partition_survives_interruption_resume() -
def approval_tool() -> str:
return "approved"
first_model = FakeModel()
second_model = FakeModel()
final_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
final_model = ScriptedModel()
final_agent = Agent(name="final", model=final_model)
second_agent = Agent(
name="second",
@@ -1837,16 +1775,14 @@ async def test_nested_handoff_history_partition_survives_interruption_resume() -
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
run_config = RunConfig(nest_handoff_history=True)
first_model.add_multiple_turn_outputs(
[[get_text_message("once"), get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs(
first_model.extend([[get_text_message("once"), get_handoff_tool_call(second_agent)]])
second_model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="approval")],
[get_text_message("once"), get_handoff_tool_call(final_agent)],
]
)
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
final_model.extend([[get_text_message("done")]])
interrupted = await Runner.run(first_agent, input="start", run_config=run_config)
assert len(interrupted.interruptions) == 1
@@ -1870,7 +1806,7 @@ async def test_nested_handoff_history_partition_survives_interruption_resume() -
resumed = await Runner.run(first_agent, restored, run_config=run_config)
assert resumed.final_output == "done"
final_input = final_model.last_turn_args["input"]
final_input = final_model.calls[-1].input
summary = str(cast(dict[str, Any], final_input[0])["content"])
assert summary.count("once") == 1
assert sum(_input_item_text(item) == "once" for item in resumed.to_input_list()) == 2
@@ -1893,9 +1829,9 @@ async def test_pending_handoff_in_interrupted_turn_survives_run_state(
def approval_tool() -> str:
return "approved"
first_model = FakeModel()
second_model = FakeModel()
final_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
final_model = ScriptedModel()
final_agent = Agent(name="final", model=final_model)
second_agent = Agent(
name="second",
@@ -1910,8 +1846,8 @@ async def test_pending_handoff_in_interrupted_turn_survives_run_state(
first_handoff.call_id = "first-handoff"
final_handoff = cast(ResponseFunctionToolCall, get_handoff_tool_call(final_agent))
final_handoff.call_id = "final-handoff"
first_model.add_multiple_turn_outputs([[get_text_message("first once"), first_handoff]])
second_model.add_multiple_turn_outputs(
first_model.extend([[get_text_message("first once"), first_handoff]])
second_model.extend(
[
[
get_text_message("second once"),
@@ -1920,7 +1856,7 @@ async def test_pending_handoff_in_interrupted_turn_survives_run_state(
]
]
)
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
final_model.extend([[get_text_message("done")]])
interrupted: RunResult | RunResultStreaming
if streamed:
@@ -1964,7 +1900,7 @@ async def test_pending_handoff_in_interrupted_turn_survives_run_state(
== 1
)
if nest_handoff_history:
final_input = final_model.last_turn_args["input"]
final_input = final_model.calls[-1].input
summary = str(cast(dict[str, Any], final_input[0])["content"])
assert summary.count("first once") == 1
assert summary.count("second once") == 1
@@ -2008,8 +1944,8 @@ async def test_resumed_handoff_persists_all_staged_approval_outputs(
nonlocal handoff_count
handoff_count += 1
source_model = FakeModel()
target_model = FakeModel()
source_model = ScriptedModel()
target_model = ScriptedModel()
target_agent = Agent(name="target", model=target_model)
source_agent = Agent(
name="source",
@@ -2030,8 +1966,8 @@ async def test_resumed_handoff_persists_all_staged_approval_outputs(
second_call.id = "item-second"
handoff_call.id = "item-handoff"
handoff_call.call_id = "handoff"
source_model.add_multiple_turn_outputs([[first_call, second_call, handoff_call]])
target_model.add_multiple_turn_outputs([[get_text_message("done")]])
source_model.extend([[first_call, second_call, handoff_call]])
target_model.extend([[get_text_message("done")]])
run_config = RunConfig(nest_handoff_history=nest_handoff_history)
hooks = RecordingHooks()
@@ -2152,8 +2088,8 @@ async def test_nested_history_resume_to_final_preserves_status_less_ownership(
def approval_tool() -> str:
return "approved"
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model, tools=[approval_tool])
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
run_config = RunConfig(nest_handoff_history=True)
@@ -2164,10 +2100,8 @@ async def test_nested_history_resume_to_final_preserves_status_less_ownership(
status=None,
type="message",
)
first_model.add_multiple_turn_outputs(
[[status_less_message, get_handoff_tool_call(second_agent)]]
)
second_model.add_multiple_turn_outputs(
first_model.extend([[status_less_message, get_handoff_tool_call(second_agent)]])
second_model.extend(
[
[get_function_tool_call("approval_tool", "{}", call_id="approval")],
[get_text_message("done")],
@@ -2211,10 +2145,8 @@ async def test_nested_history_resume_to_final_preserves_status_less_ownership(
assert final_output == "done"
assert sum(_input_item_text(item) == "once" for item in replay_input) == 1
assert all("_agents_nested_history_token" not in item for item in replay_input)
assert second_model.last_turn_args is not None
assert all(
"_agents_nested_history_token" not in item for item in second_model.last_turn_args["input"]
)
assert bool(second_model.calls)
assert all("_agents_nested_history_token" not in item for item in second_model.calls[-1].input)
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
@@ -2234,8 +2166,8 @@ async def test_first_nested_handoff_after_restore_uses_explicit_occurrence_linea
def approval_tool() -> str:
return "approved"
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
second_agent = Agent(name="second", model=second_model)
first_agent = Agent(
name="first",
@@ -2259,17 +2191,18 @@ async def test_first_nested_handoff_after_restore_uses_explicit_occurrence_linea
tools=[],
type="tool_search_output",
)
first_model.add_multiple_turn_outputs(
first_output = [
tool_search_call,
tool_search_output,
get_function_tool_call("approval_tool", "{}", call_id="approval"),
]
first_model.extend(
[
[
tool_search_call,
tool_search_output,
get_function_tool_call("approval_tool", "{}", call_id="approval"),
],
get_exact_output_stream_step(first_output) if streamed else first_output,
[get_handoff_tool_call(second_agent)],
]
)
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
second_model.extend([[get_text_message("done")]])
run_config = RunConfig(nest_handoff_history=True)
interrupted: RunResult | RunResultStreaming
+45 -47
View File
@@ -73,6 +73,7 @@ from agents.run_internal.tool_planning import (
execute_mcp_approval_requests,
)
from agents.run_state import RunState as RunStateClass
from agents.testing import ScriptedModel
from agents.tool import FunctionTool, HostedMCPTool
from agents.tool_guardrails import (
ToolGuardrailFunctionOutput,
@@ -83,7 +84,6 @@ from agents.tool_guardrails import (
)
from agents.usage import Usage
from .fake_model import FakeModel
from .mcp.helpers import FakeMCPServer
from .test_responses import get_text_message
from .utils.hitl import (
@@ -321,12 +321,12 @@ async def test_nested_agent_tool_resumes_after_rejection() -> None:
async def inner_hitl_tool() -> str:
return "ok"
inner_model = FakeModel()
inner_model = ScriptedModel()
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool])
inner_call_first = make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")
inner_call_retry = make_function_tool_call(inner_hitl_tool.name, call_id="inner-2")
inner_final = get_text_message("done")
inner_model.add_multiple_turn_outputs(
inner_model.extend(
[
[inner_call_first],
[inner_call_retry],
@@ -340,12 +340,12 @@ async def test_nested_agent_tool_resumes_after_rejection() -> None:
needs_approval=True,
)
outer_model = FakeModel()
outer_model = ScriptedModel()
outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool])
outer_call = make_function_tool_call(
agent_tool.name, call_id="outer-1", arguments='{"input":"hi"}'
)
outer_model.add_multiple_turn_outputs([[outer_call]])
outer_model.extend([[outer_call]])
first = await Runner.run(outer_agent, "start")
assert first.interruptions, "agent tool should request approval first"
@@ -391,23 +391,23 @@ async def test_changed_nested_parent_fails_before_tool_inventory_callbacks() ->
async def observer() -> str:
return "unused"
inner_model = FakeModel()
inner_model.add_multiple_turn_outputs(
[[make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")]]
)
inner_model = ScriptedModel()
inner_model.extend([[make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")]])
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool])
agent_tool = inner_agent.as_tool(
tool_name="inner_agent_tool",
tool_description="Inner agent tool with HITL",
needs_approval=True,
)
outer_model = FakeModel(
initial_output=[
make_function_tool_call(
agent_tool.name,
call_id="outer-1",
arguments='{"input":"safe"}',
)
outer_model = ScriptedModel(
steps=[
[
make_function_tool_call(
agent_tool.name,
call_id="outer-1",
arguments='{"input":"safe"}',
)
]
]
)
outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool, observer])
@@ -438,9 +438,9 @@ async def test_nested_agent_tool_interruptions_remain_distinct_across_outer_call
async def inner_hitl_tool() -> str:
return "ok"
inner_model = FakeModel()
inner_model = ScriptedModel()
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool])
inner_model.add_multiple_turn_outputs(
inner_model.extend(
[
[make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")],
[make_function_tool_call(inner_hitl_tool.name, call_id="inner-2")],
@@ -453,9 +453,9 @@ async def test_nested_agent_tool_interruptions_remain_distinct_across_outer_call
needs_approval=False,
)
outer_model = FakeModel()
outer_model = ScriptedModel()
outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool])
outer_model.add_multiple_turn_outputs(
outer_model.extend(
[
[
make_function_tool_call(
@@ -488,11 +488,9 @@ async def test_nested_agent_tool_does_not_inherit_parent_approvals() -> None:
async def inner_shared_tool() -> str:
return "inner"
inner_model = FakeModel()
inner_model = ScriptedModel()
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_shared_tool])
inner_model.add_multiple_turn_outputs(
[[make_function_tool_call(inner_shared_tool.name, call_id="dup")]]
)
inner_model.extend([[make_function_tool_call(inner_shared_tool.name, call_id="dup")]])
agent_tool = inner_agent.as_tool(
tool_name="inner_agent_tool",
@@ -500,9 +498,9 @@ async def test_nested_agent_tool_does_not_inherit_parent_approvals() -> None:
needs_approval=False,
)
outer_model = FakeModel()
outer_model = ScriptedModel()
outer_agent = Agent(name="Outer", model=outer_model, tools=[outer_shared_tool, agent_tool])
outer_model.add_multiple_turn_outputs(
outer_model.extend(
[
[make_function_tool_call(outer_shared_tool.name, call_id="dup")],
[
@@ -568,7 +566,7 @@ async def test_resume_does_not_duplicate_pending_shell_approvals() -> None:
call_id = extract_tool_call_id(raw_call)
assert call_id, "shell call must have a call_id"
model.set_next_output([raw_call])
model.enqueue([raw_call])
first = await Runner.run(agent, "run shell")
assert first.interruptions, "shell tool should require approval"
@@ -626,7 +624,8 @@ async def test_route_local_shell_calls_to_remote_shell_tool():
action={"type": "exec", "command": ["echo", "test"], "env": {}}, # type: ignore[arg-type]
status="in_progress",
)
model.set_next_output([local_shell_call])
model.enqueue([local_shell_call])
model.enqueue([])
await Runner.run(agent, "run local shell")
@@ -654,7 +653,7 @@ async def test_preserve_max_turns_when_resuming_from_runresult_state():
tool = function_tool(test_tool, needs_approval=require_approval)
model, agent = make_model_and_agent(tools=[tool])
model.add_multiple_turn_outputs([[make_function_tool_call("test_tool", call_id="call-1")]])
model.extend([[make_function_tool_call("test_tool", call_id="call-1")]])
result1 = await Runner.run(agent, "call test_tool", max_turns=20)
assert result1.interruptions, "should have an interruption"
@@ -662,7 +661,7 @@ async def test_preserve_max_turns_when_resuming_from_runresult_state():
state = approve_first_interruption(result1, always_approve=True)
# Provide 10 more turns (turns 2-11) to ensure we exceed the default 10 but not 20.
model.add_multiple_turn_outputs(
model.extend(
[
[
get_text_message(f"turn {i + 2}"), # Text message first (doesn't finish)
@@ -671,6 +670,7 @@ async def test_preserve_max_turns_when_resuming_from_runresult_state():
for i in range(10)
]
)
model.enqueue([])
result2 = await Runner.run(agent, state)
assert result2 is not None, "Run should complete successfully with max_turns=20 from state"
@@ -687,7 +687,7 @@ async def test_current_turn_not_preserved_in_to_state():
model, agent = make_model_and_agent(tools=[tool])
# Model emits a tool call requiring approval
model.set_next_output([make_function_tool_call("test_tool", call_id="call-1")])
model.enqueue([make_function_tool_call("test_tool", call_id="call-1")])
# First turn with interruption
result1 = await Runner.run(agent, "call test_tool")
@@ -859,7 +859,7 @@ async def test_preserve_persisted_item_counter_when_resuming_streamed_runs():
]
# Set up model to return final output immediately (so the run completes)
model.set_next_output([get_text_message("done")])
model.enqueue([get_text_message("done")])
result = Runner.run_streamed(agent, state)
@@ -910,7 +910,7 @@ async def test_function_needs_approval_invalid_type_raises() -> None:
return "ok"
model, agent = make_model_and_agent(tools=[bad_tool])
model.set_next_output([make_function_tool_call("bad_tool")])
model.enqueue([make_function_tool_call("bad_tool")])
with pytest.raises(UserError, match="needs_approval"):
await Runner.run(agent, "run invalid")
@@ -951,9 +951,7 @@ async def test_callable_function_approval_fails_closed_for_invalid_arguments(
needs_approval=needs_approval,
)
model, agent = make_model_and_agent(tools=[tool])
model.set_next_output(
[make_function_tool_call(tool.name, arguments=arguments, call_id="call-invalid")]
)
model.enqueue([make_function_tool_call(tool.name, arguments=arguments, call_id="call-invalid")])
result = await Runner.run(agent, "send an email")
@@ -986,7 +984,7 @@ async def test_callable_function_approval_receives_valid_object_arguments() -> N
)
arguments = '{"subject": "status update"}'
model, agent = make_model_and_agent(tools=[tool])
model.add_multiple_turn_outputs(
model.extend(
[
[make_function_tool_call(tool.name, arguments=arguments, call_id="call-valid")],
[get_text_message("done")],
@@ -1058,7 +1056,7 @@ async def test_agent_as_tool_with_nested_approvals_propagates() -> None:
spanish_agent.tools = [get_current_timestamp]
# Spanish agent will first request timestamp, then return text.
nested_model.add_multiple_turn_outputs(
nested_model.extend(
[
[make_function_tool_call("get_current_timestamp")],
[get_text_message("hola")],
@@ -1066,7 +1064,7 @@ async def test_agent_as_tool_with_nested_approvals_propagates() -> None:
)
# Orchestrator model will call the spanish agent tool.
orchestrator_model = FakeModel()
orchestrator_model = ScriptedModel()
orchestrator = Agent(
name="orchestrator",
tools=[
@@ -1079,7 +1077,7 @@ async def test_agent_as_tool_with_nested_approvals_propagates() -> None:
model=orchestrator_model,
)
orchestrator_model.add_multiple_turn_outputs(
orchestrator_model.extend(
[
[
make_function_tool_call(
@@ -1132,7 +1130,7 @@ async def test_nested_agent_tool_continuation_runs_outer_callbacks_once(streamed
return "inner output"
nested_agent.tools = [inner_tool]
nested_model.add_multiple_turn_outputs(
nested_model.extend(
[
[
make_function_tool_call(
@@ -1171,8 +1169,8 @@ async def test_nested_agent_tool_continuation_runs_outer_callbacks_once(streamed
outer_tool.tool_output_guardrails = [track_output]
outer_tool.custom_data_extractor = extract_custom_data
outer_model = FakeModel()
outer_model.add_multiple_turn_outputs(
outer_model = ScriptedModel()
outer_model.extend(
[
[
make_function_tool_call(
@@ -1789,9 +1787,9 @@ async def test_execute_path_skips_needs_approval_checker_when_status_resolved()
async def sensitive(value: str) -> str:
return f"ran:{value}"
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="agent", model=model, tools=[sensitive])
model.add_multiple_turn_outputs(
model.extend(
[
[make_function_tool_call(sensitive.name, call_id="call-1", arguments='{"value":"x"}')],
[get_text_message("done")],
@@ -1827,14 +1825,14 @@ async def test_resume_checkpoints_tool_output_before_tool_use_behavior_failure()
def failing_behavior(_ctx: Any, _results: Any) -> Any:
raise RuntimeError("tool use behavior failed")
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="agent",
model=model,
tools=[sensitive],
tool_use_behavior=failing_behavior,
)
model.add_multiple_turn_outputs(
model.extend(
[
[make_function_tool_call(sensitive.name, call_id="call-1", arguments='{"value":"x"}')],
[get_text_message("done")],
+21 -61
View File
@@ -1,24 +1,19 @@
from __future__ import annotations
import json
from collections.abc import AsyncIterator
from dataclasses import dataclass
from typing import Any, cast
import pytest
from openai.types.responses import ResponseFunctionToolCall
from agents import (
Agent,
Model,
ModelResponse,
ModelSettings,
OpenAIConversationsSession,
Runner,
Usage,
function_tool,
)
from agents.items import TResponseInputItem, TResponseStreamEvent
from agents.items import TResponseInputItem
from agents.testing import ModelCall, ModelStep, ScriptedModel, function_call
from tests.test_responses import get_text_message
from tests.utils.hitl import HITL_REJECTION_MSG
from tests.utils.simple_session import SimpleListSession
@@ -69,67 +64,32 @@ class ScenarioResult:
items: list[TResponseInputItem]
class ScenarioModel(Model):
def __init__(self) -> None:
self._counter = 0
def make_scenario_model() -> ScriptedModel:
call_counter = 0
async def get_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Any],
output_schema: Any,
handoffs: list[Any],
tracing: Any,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: Any | None,
) -> ModelResponse:
if input_has_rejection(input):
return ModelResponse(
output=[get_text_message(HITL_REJECTION_MSG)],
usage=Usage(),
response_id="resp-test",
)
tool_choice = model_settings.tool_choice
def respond(call: ModelCall):
nonlocal call_counter
if input_has_rejection(call.input):
return [get_text_message(HITL_REJECTION_MSG)]
tool_choice = call.model_settings.tool_choice
tool_name = tool_choice if isinstance(tool_choice, str) else TOOL_ECHO
self._counter += 1
call_id = f"call_{self._counter}"
query = extract_user_message(input)
tool_call = ResponseFunctionToolCall(
type="function_call",
name=tool_name,
call_id=call_id,
arguments=json.dumps({"query": query}),
)
return ModelResponse(output=[tool_call], usage=Usage(), response_id="resp-test")
call_counter += 1
return [
function_call(
tool_name,
{"query": extract_user_message(call.input)},
call_id=f"call_{call_counter}",
)
]
async def stream_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Any],
output_schema: Any,
handoffs: list[Any],
tracing: Any,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: Any | None,
) -> AsyncIterator[TResponseStreamEvent]:
if False:
yield cast(TResponseStreamEvent, {})
raise RuntimeError("Streaming is not supported in this scenario.")
return ScriptedModel([ModelStep.respond(respond) for _ in range(4)])
@pytest.mark.asyncio
async def test_memory_session_hitl_scenario() -> None:
execute_counts.clear()
session = SimpleListSession(session_id="memory")
model = ScenarioModel()
model = make_scenario_model()
steps = [
ScenarioStep(
@@ -233,7 +193,7 @@ async def test_openai_conversations_session_hitl_scenario() -> None:
rehydrated_session = OpenAIConversationsSession(
conversation_id="conv_test", openai_client=typed_client
)
model = ScenarioModel()
model = make_scenario_model()
steps = [
ScenarioStep(
@@ -280,7 +240,7 @@ async def test_openai_conversations_session_hitl_scenario() -> None:
async def run_scenario_step(
session: Any,
model: ScenarioModel,
model: ScriptedModel,
step: ScenarioStep,
) -> ScenarioResult:
agent = Agent(
+21 -21
View File
@@ -27,8 +27,8 @@ from agents import (
)
from agents.items import TResponseInputItem, TResponseOutputItem
from agents.stream_events import RunItemStreamEvent
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import get_function_tool_call, get_text_message
from .utils.simple_session import SimpleListSession
@@ -62,7 +62,7 @@ def _message_texts(items: list[TResponseInputItem]) -> list[str]:
@pytest.mark.asyncio
async def test_invalid_final_output_raises_without_handler() -> None:
model = FakeModel(initial_output=[get_text_message("not valid json")])
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
with pytest.raises(ModelBehaviorError, match="Invalid JSON"):
@@ -71,7 +71,7 @@ async def test_invalid_final_output_raises_without_handler() -> None:
@pytest.mark.asyncio
async def test_invalid_final_output_handler_returns_validated_fallback() -> None:
model = FakeModel(initial_output=[get_text_message("not valid json")])
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
def handler(data: RunErrorHandlerInput[None]) -> FinalOutput:
@@ -96,7 +96,7 @@ async def test_invalid_final_output_handler_returns_validated_fallback() -> None
@pytest.mark.asyncio
async def test_invalid_final_output_handler_can_skip_fallback_history() -> None:
model = FakeModel(initial_output=[get_text_message("not valid json")])
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
result = await Runner.run(
@@ -119,7 +119,7 @@ async def test_invalid_final_output_handler_rejects_invalid_fallback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False)
model = FakeModel(initial_output=[get_text_message("not valid json")])
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
with pytest.warns(UserWarning, match="Pydantic serializer warnings"):
@@ -133,7 +133,7 @@ async def test_invalid_final_output_handler_rejects_invalid_fallback(
@pytest.mark.asyncio
async def test_invalid_final_output_handler_can_decline_recovery() -> None:
model = FakeModel(initial_output=[get_text_message("not valid json")])
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
with pytest.raises(ModelBehaviorError, match="Invalid JSON"):
@@ -146,7 +146,7 @@ async def test_invalid_final_output_handler_can_decline_recovery() -> None:
@pytest.mark.asyncio
async def test_invalid_final_output_handler_does_not_catch_other_model_behavior_errors() -> None:
model = FakeModel(initial_output=[get_function_tool_call("missing_tool")])
model = ScriptedModel(steps=[[get_function_tool_call("missing_tool")]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
handler_called = False
@@ -170,8 +170,8 @@ async def test_invalid_final_output_handler_does_not_catch_other_model_behavior_
async def test_empty_structured_output_handler_avoids_another_model_turn(
invalid_output: list[TResponseOutputItem],
) -> None:
model = FakeModel()
model.add_multiple_turn_outputs([invalid_output, [get_text_message('{"summary":"unused"}')]])
model = ScriptedModel()
model.extend([invalid_output, [get_text_message('{"summary":"unused"}')]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
def handler(data: RunErrorHandlerInput[None]) -> FinalOutput:
@@ -188,7 +188,7 @@ async def test_empty_structured_output_handler_avoids_another_model_turn(
)
assert result.final_output == FinalOutput(summary="safe fallback")
assert len(model.turn_outputs) == 1
assert model.remaining_steps == 1
@pytest.mark.asyncio
@@ -199,19 +199,19 @@ async def test_empty_structured_output_handler_avoids_another_model_turn(
async def test_empty_structured_output_without_fallback_keeps_existing_next_turn_behavior(
error_handlers: RunErrorHandlers[None] | None,
) -> None:
model = FakeModel()
model.add_multiple_turn_outputs([[], [get_text_message('{"summary":"second turn"}')]])
model = ScriptedModel()
model.extend([[], [get_text_message('{"summary":"second turn"}')]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
result = await Runner.run(agent, input="user_message", error_handlers=error_handlers)
assert result.final_output == FinalOutput(summary="second turn")
assert not model.turn_outputs
assert model.remaining_steps == 0
@pytest.mark.asyncio
async def test_streamed_invalid_final_output_emits_exact_fallback_item() -> None:
model = FakeModel(initial_output=[get_text_message("not valid json")])
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
session = SimpleListSession()
@@ -246,8 +246,8 @@ async def test_streamed_invalid_final_output_emits_exact_fallback_item() -> None
@pytest.mark.asyncio
async def test_streamed_empty_structured_output_handler_avoids_another_model_turn() -> None:
model = FakeModel()
model.add_multiple_turn_outputs([[], [get_text_message('{"summary":"unused"}')]])
model = ScriptedModel()
model.extend([[], [get_text_message('{"summary":"unused"}')]])
agent = Agent(name="test", model=model, output_type=FinalOutput)
result = Runner.run_streamed(
@@ -258,7 +258,7 @@ async def test_streamed_empty_structured_output_handler_avoids_another_model_tur
events = [event async for event in result.stream_events()]
assert result.final_output == FinalOutput(summary="safe fallback")
assert len(model.turn_outputs) == 1
assert model.remaining_steps == 1
assert any(
isinstance(event, RunItemStreamEvent)
and event.name == "message_output_created"
@@ -270,7 +270,7 @@ async def test_streamed_empty_structured_output_handler_avoids_another_model_tur
@pytest.mark.asyncio
async def test_invalid_final_output_fallback_runs_hooks_and_output_guardrails() -> None:
model = FakeModel(initial_output=[get_text_message("not valid json")])
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
hooks = RecordingRunHooks()
guarded_outputs: list[Any] = []
@@ -315,8 +315,8 @@ async def test_invalid_final_output_fallback_does_not_retry_or_replay_tools(
side_effects.append(value)
return f"recorded:{value}"
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
@@ -373,4 +373,4 @@ async def test_invalid_final_output_fallback_does_not_retry_or_replay_tools(
assert final_output == FinalOutput(summary="safe fallback")
assert side_effects == ["once"]
assert len(model.turn_outputs) == 2
assert model.remaining_steps == 2
+12 -9
View File
@@ -28,8 +28,9 @@ from agents import (
from agents.items import ToolCallOutputItem
from agents.run_internal.run_loop import LocalShellAction, ToolRunLocalShellCall
from agents.run_state import RunState
from agents.testing import ScriptedModel
from tests.model_test_helpers import get_response_obj
from .fake_model import FakeModel, get_response_obj
from .test_responses import get_text_message
@@ -47,7 +48,7 @@ class RecordingLocalShellExecutor:
async def _create_serialized_local_shell_state() -> tuple[LocalShellTool, dict[str, Any]]:
tool = LocalShellTool(executor=RecordingLocalShellExecutor(output="shell result"))
initial_model = FakeModel()
initial_model = ScriptedModel()
initial_agent = Agent(name="shell-agent", model=initial_model, tools=[tool])
local_shell_call = LocalShellCall(
id="lsh_test",
@@ -62,7 +63,7 @@ async def _create_serialized_local_shell_state() -> tuple[LocalShellTool, dict[s
status="completed",
type="local_shell_call",
)
initial_model.add_multiple_turn_outputs(
initial_model.extend(
[
[get_text_message("running shell"), local_shell_call],
[get_text_message("shell complete")],
@@ -158,7 +159,7 @@ async def test_runner_executes_local_shell_calls() -> None:
executor = RecordingLocalShellExecutor(output="shell result")
tool = LocalShellTool(executor=executor)
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="shell-agent", model=model, tools=[tool])
action = LocalShellCallAction(
@@ -176,7 +177,7 @@ async def test_runner_executes_local_shell_calls() -> None:
type="local_shell_call",
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("running shell"), local_shell_call],
[get_text_message("shell complete")],
@@ -188,7 +189,8 @@ async def test_runner_executes_local_shell_calls() -> None:
assert len(executor.calls) == 1
request = executor.calls[0]
assert isinstance(request, LocalShellCommandRequest)
assert request.data is local_shell_call
assert request.data == local_shell_call
assert request.data is not local_shell_call
items = result.new_items
assert len(items) == 4
@@ -201,7 +203,8 @@ async def test_runner_executes_local_shell_calls() -> None:
tool_call_item = items[1]
assert tool_call_item.type == "tool_call_item"
assert tool_call_item.raw_item is local_shell_call
assert tool_call_item.raw_item == local_shell_call
assert tool_call_item.raw_item is not local_shell_call
local_shell_output = items[2]
assert isinstance(local_shell_output, ToolCallOutputItem)
@@ -334,8 +337,8 @@ async def test_run_state_preserves_official_local_shell_original_input(
"id": "lsh_output_123",
"output": "shell result",
}
model = FakeModel()
model.add_multiple_turn_outputs([[get_text_message("complete")]])
model = ScriptedModel()
model.extend([[get_text_message("complete")]])
agent = Agent(name="shell-agent", model=model)
result = await Runner.run(agent, input=[original_input])
serialized = json.loads(json.dumps(result.to_state().to_json()))
+30 -32
View File
@@ -18,8 +18,8 @@ from agents import (
UserError,
)
from agents.stream_events import RunItemStreamEvent
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import (
get_function_tool,
get_function_tool_call,
@@ -30,7 +30,7 @@ from .test_responses import (
@pytest.mark.asyncio
async def test_non_streamed_max_turns():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -39,7 +39,7 @@ async def test_non_streamed_max_turns():
func_output = json.dumps({"a": "b"})
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("1"), get_function_tool_call("some_function", func_output, "1")],
[get_text_message("2"), get_function_tool_call("some_function", func_output, "2")],
@@ -54,7 +54,7 @@ async def test_non_streamed_max_turns():
@pytest.mark.asyncio
async def test_non_streamed_max_turns_none_disables_limit():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -63,7 +63,7 @@ async def test_non_streamed_max_turns_none_disables_limit():
func_output = json.dumps({"a": "b"})
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("1"), get_function_tool_call("some_function", func_output, "1")],
[get_text_message("2"), get_function_tool_call("some_function", func_output, "2")],
@@ -81,7 +81,7 @@ async def test_non_streamed_max_turns_none_disables_limit():
@pytest.mark.asyncio
async def test_streamed_max_turns():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -89,7 +89,7 @@ async def test_streamed_max_turns():
)
func_output = json.dumps({"a": "b"})
model.add_multiple_turn_outputs(
model.extend(
[
[
get_text_message("1"),
@@ -121,7 +121,7 @@ async def test_streamed_max_turns():
@pytest.mark.asyncio
async def test_streamed_max_turns_none_disables_limit():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -129,7 +129,7 @@ async def test_streamed_max_turns_none_disables_limit():
)
func_output = json.dumps({"a": "b"})
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("1"), get_function_tool_call("some_function", func_output, "1")],
[get_text_message("2"), get_function_tool_call("some_function", func_output, "2")],
@@ -157,19 +157,19 @@ class FooModel(BaseModel):
@pytest.mark.asyncio
async def test_non_streamed_structured_output_refusal_raises_without_retry():
model = FakeModel(initial_output=[get_refusal_message("I cannot help with that request.")])
model = ScriptedModel(steps=[[get_refusal_message("I cannot help with that request.")]])
agent = Agent(name="test_1", model=model, output_type=FooModel)
with pytest.raises(ModelRefusalError) as exc_info:
await Runner.run(agent, input="user_message", max_turns=3)
assert exc_info.value.refusal == "I cannot help with that request."
assert not model.turn_outputs
assert model.remaining_steps == 0
@pytest.mark.asyncio
async def test_non_streamed_refusal_handler_returns_structured_output():
model = FakeModel(initial_output=[get_refusal_message("I cannot help with that request.")])
model = ScriptedModel(steps=[[get_refusal_message("I cannot help with that request.")]])
agent = Agent(name="test_1", model=model, output_type=FooModel)
def handler(data):
@@ -194,7 +194,7 @@ async def test_non_streamed_refusal_handler_returns_structured_output():
@pytest.mark.asyncio
async def test_non_streamed_refusal_handler_can_skip_history():
model = FakeModel(initial_output=[get_refusal_message("I cannot help with that request.")])
model = ScriptedModel(steps=[[get_refusal_message("I cannot help with that request.")]])
agent = Agent(name="test_1", model=model)
result = await Runner.run(
@@ -214,7 +214,7 @@ async def test_non_streamed_refusal_handler_can_skip_history():
@pytest.mark.asyncio
async def test_streamed_refusal_handler_returns_output():
model = FakeModel(initial_output=[get_refusal_message("I cannot help with that request.")])
model = ScriptedModel(steps=[[get_refusal_message("I cannot help with that request.")]])
agent = Agent(name="test_1", model=model)
result = Runner.run_streamed(
@@ -237,7 +237,7 @@ async def test_streamed_refusal_handler_returns_output():
@pytest.mark.asyncio
async def test_structured_output_non_streamed_max_turns():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -245,7 +245,7 @@ async def test_structured_output_non_streamed_max_turns():
tools=[get_function_tool("tool_1", "result")],
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("tool_1")],
[get_function_tool_call("tool_1")],
@@ -260,7 +260,7 @@ async def test_structured_output_non_streamed_max_turns():
@pytest.mark.asyncio
async def test_structured_output_streamed_max_turns():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -268,7 +268,7 @@ async def test_structured_output_streamed_max_turns():
tools=[get_function_tool("tool_1", "result")],
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("tool_1")],
[get_function_tool_call("tool_1")],
@@ -285,7 +285,7 @@ async def test_structured_output_streamed_max_turns():
@pytest.mark.asyncio
async def test_structured_output_max_turns_handler_invalid_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -303,7 +303,7 @@ async def test_structured_output_max_turns_handler_invalid_output():
@pytest.mark.asyncio
async def test_structured_output_max_turns_handler_pydantic_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -324,7 +324,7 @@ async def test_structured_output_max_turns_handler_pydantic_output():
@pytest.mark.asyncio
async def test_structured_output_max_turns_handler_list_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -344,7 +344,7 @@ async def test_structured_output_max_turns_handler_list_output():
@pytest.mark.asyncio
async def test_non_streamed_max_turns_handler_returns_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test_1", model=model)
result = await Runner.run(
@@ -364,7 +364,7 @@ async def test_non_streamed_max_turns_handler_returns_output():
@pytest.mark.asyncio
async def test_non_streamed_max_turns_handler_skip_history():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test_1", model=model)
result = await Runner.run(
@@ -385,7 +385,7 @@ async def test_non_streamed_max_turns_handler_skip_history():
@pytest.mark.asyncio
async def test_non_streamed_max_turns_handler_raw_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test_1", model=model)
result = await Runner.run(
@@ -401,7 +401,7 @@ async def test_non_streamed_max_turns_handler_raw_output():
@pytest.mark.asyncio
async def test_non_streamed_max_turns_handler_raw_dict_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test_1", model=model)
result = await Runner.run(
@@ -416,7 +416,7 @@ async def test_non_streamed_max_turns_handler_raw_dict_output():
@pytest.mark.asyncio
async def test_streamed_max_turns_handler_returns_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test_1", model=model)
result = Runner.run_streamed(
@@ -439,7 +439,7 @@ async def test_streamed_max_turns_handler_returns_output():
@pytest.mark.asyncio
async def test_streamed_max_turns_handler_pydantic_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -466,7 +466,7 @@ async def test_streamed_max_turns_handler_pydantic_output():
@pytest.mark.asyncio
async def test_streamed_max_turns_handler_list_output():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
@@ -492,15 +492,13 @@ async def test_streamed_max_turns_handler_list_output():
async def _run_max_turns_handler_with_session(streamed: bool) -> list[str]:
"""Run one tool turn, trip max turns, and return the session's persisted item types."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="test_1",
model=model,
tools=[get_function_tool("some_function", "result")],
)
model.add_multiple_turn_outputs(
[[get_function_tool_call("some_function", json.dumps({"a": "b"}))]]
)
model.extend([[get_function_tool_call("some_function", json.dumps({"a": "b"}))]])
session = SQLiteSession("max-turns-handler", ":memory:")
try:
if streamed:
+13 -13
View File
@@ -6,20 +6,20 @@ from pydantic import BaseModel
from agents import Agent, RunContextWrapper, RunErrorDetails, Runner, RunResult
from agents.agent_output import _WRAPPER_DICT_KEY
from agents.testing import ScriptedModel
from agents.util._pretty_print import (
pretty_print_result,
pretty_print_run_error_details,
pretty_print_run_result_streaming,
)
from tests.fake_model import FakeModel
from .test_responses import get_final_output_message, get_text_message
@pytest.mark.asyncio
async def test_pretty_result():
model = FakeModel()
model.set_next_output([get_text_message("Hi there")])
model = ScriptedModel()
model.enqueue([get_text_message("Hi there")])
agent = Agent(name="test_agent", model=model)
result = await Runner.run(agent, input="Hello")
@@ -92,8 +92,8 @@ RunErrorDetails:
@pytest.mark.asyncio
async def test_pretty_run_result_streaming():
model = FakeModel()
model.set_next_output([get_text_message("Hi there")])
model = ScriptedModel()
model.enqueue([get_text_message("Hi there")])
agent = Agent(name="test_agent", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -122,8 +122,8 @@ class Foo(BaseModel):
@pytest.mark.asyncio
async def test_pretty_run_result_structured_output():
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[
get_text_message("Test"),
get_final_output_message(Foo(bar="Hi there").model_dump_json()),
@@ -150,8 +150,8 @@ RunResult:
@pytest.mark.asyncio
async def test_pretty_run_result_streaming_structured_output():
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[
get_text_message("Test"),
get_final_output_message(Foo(bar="Hi there").model_dump_json()),
@@ -184,8 +184,8 @@ RunResultStreaming:
@pytest.mark.asyncio
async def test_pretty_run_result_list_structured_output():
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[
get_text_message("Test"),
get_final_output_message(
@@ -219,8 +219,8 @@ RunResult:
@pytest.mark.asyncio
async def test_pretty_run_result_streaming_list_structured_output():
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[
get_text_message("Test"),
get_final_output_message(
+29 -29
View File
@@ -41,8 +41,8 @@ from agents.items import (
)
from agents.mcp.util import MCPUtil
from agents.run_internal import run_loop
from agents.testing import ScriptedModel
from agents.usage import Usage
from tests.fake_model import FakeModel
from tests.mcp.helpers import FakeMCPServer
from tests.mcp.model_compat import Tool as MCPTool
from tests.test_responses import get_function_tool_call
@@ -76,7 +76,7 @@ def _make_hosted_mcp_list_tools(server_label: str, tool_name: str) -> McpListToo
def test_process_model_response_shell_call_without_tool_raises() -> None:
agent = Agent(name="no-shell", model=FakeModel())
agent = Agent(name="no-shell", model=ScriptedModel())
shell_call = make_shell_call("shell-1")
with pytest.raises(ModelBehaviorError, match="shell tool"):
@@ -111,7 +111,7 @@ def test_process_model_response_dispatches_falsy_shell_tool() -> None:
def test_process_model_response_sets_title_for_local_mcp_function_tool() -> None:
agent = Agent(name="local-mcp", model=FakeModel())
agent = Agent(name="local-mcp", model=ScriptedModel())
mcp_tool = MCPTool(name="search_docs", inputSchema={}, description=None, title="Search Docs")
function_tool = MCPUtil.to_function_tool(
mcp_tool,
@@ -142,7 +142,7 @@ def test_process_model_response_sets_title_for_local_mcp_function_tool() -> None
def test_process_model_response_uses_mcp_list_tools_metadata_for_hosted_mcp_calls() -> None:
agent = Agent(name="hosted-mcp", model=FakeModel())
agent = Agent(name="hosted-mcp", model=ScriptedModel())
hosted_tool = HostedMCPTool(
tool_config=cast(
Any,
@@ -186,7 +186,7 @@ def test_process_model_response_uses_mcp_list_tools_metadata_for_hosted_mcp_call
def test_process_model_response_skips_local_shell_execution_for_hosted_environment() -> None:
shell_tool = ShellTool(environment={"type": "container_auto"})
agent = Agent(name="hosted-shell", model=FakeModel(), tools=[shell_tool])
agent = Agent(name="hosted-shell", model=ScriptedModel(), tools=[shell_tool])
shell_call = make_shell_call("shell-hosted-1")
processed = run_loop.process_model_response(
@@ -213,7 +213,7 @@ def test_process_model_response_sanitizes_shell_call_model_object() -> None:
action=cast(Any, {"commands": ["echo hi"], "timeout_ms": 1000}),
)
shell_tool = ShellTool(environment={"type": "container_auto"})
agent = Agent(name="hosted-shell-model", model=FakeModel(), tools=[shell_tool])
agent = Agent(name="hosted-shell-model", model=ScriptedModel(), tools=[shell_tool])
processed = run_loop.process_model_response(
agent=agent,
@@ -252,7 +252,7 @@ def test_process_model_response_preserves_shell_call_output() -> None:
}
],
}
agent = Agent(name="shell-output", model=FakeModel())
agent = Agent(name="shell-output", model=ScriptedModel())
processed = run_loop.process_model_response(
agent=agent,
@@ -288,7 +288,7 @@ def test_process_model_response_sanitizes_shell_call_output_model_object() -> No
],
),
)
agent = Agent(name="shell-output-model", model=FakeModel())
agent = Agent(name="shell-output-model", model=ScriptedModel())
processed = run_loop.process_model_response(
agent=agent,
@@ -322,7 +322,7 @@ def test_process_model_response_sanitizes_shell_call_output_model_object() -> No
def test_process_model_response_apply_patch_call_without_tool_raises() -> None:
agent = Agent(name="no-apply", model=FakeModel())
agent = Agent(name="no-apply", model=ScriptedModel())
apply_patch_call = make_apply_patch_dict("apply-1", diff="-old\n+new\n")
with pytest.raises(ModelBehaviorError, match="apply_patch tool"):
@@ -338,7 +338,7 @@ def test_process_model_response_apply_patch_call_without_tool_raises() -> None:
def test_process_model_response_sanitizes_apply_patch_call_model_object() -> None:
editor = RecordingEditor()
apply_patch_tool = ApplyPatchTool(editor=editor)
agent = Agent(name="apply-agent-model", model=FakeModel(), tools=[apply_patch_tool])
agent = Agent(name="apply-agent-model", model=ScriptedModel(), tools=[apply_patch_tool])
apply_patch_call = ResponseApplyPatchToolCall(
type="apply_patch_call",
id="ap_call_1",
@@ -380,7 +380,7 @@ def test_process_model_response_sanitizes_apply_patch_call_model_object() -> Non
def test_process_model_response_queues_apply_patch_call() -> None:
editor = RecordingEditor()
apply_patch_tool = ApplyPatchTool(editor=editor)
agent = Agent(name="apply-agent", model=FakeModel(), tools=[apply_patch_tool])
agent = Agent(name="apply-agent", model=ScriptedModel(), tools=[apply_patch_tool])
apply_patch_call = make_apply_patch_dict("apply-1")
processed = run_loop.process_model_response(
@@ -420,7 +420,7 @@ def test_process_model_response_dispatches_falsy_apply_patch_tool() -> None:
def test_process_model_response_queues_hosted_apply_patch_from_custom_tool_call() -> None:
editor = RecordingEditor()
apply_patch_tool = ApplyPatchTool(editor=editor)
agent = Agent(name="apply-agent-custom", model=FakeModel(), tools=[apply_patch_tool])
agent = Agent(name="apply-agent-custom", model=ScriptedModel(), tools=[apply_patch_tool])
custom_call = ResponseCustomToolCall(
type="custom_tool_call",
name="apply_patch",
@@ -456,7 +456,7 @@ def test_process_model_response_queues_custom_tool_call_for_custom_tool() -> Non
on_invoke_tool=lambda _ctx, raw_input: raw_input,
format={"type": "text"},
)
agent = Agent(name="custom-agent", model=FakeModel(), tools=[custom_tool])
agent = Agent(name="custom-agent", model=ScriptedModel(), tools=[custom_tool])
custom_call = ResponseCustomToolCall(
type="custom_tool_call",
name="raw_editor",
@@ -487,7 +487,7 @@ def test_process_model_response_prefers_namespaced_function_over_apply_patch_fal
tools=[function_tool(lambda payload: payload, name_override="apply_patch_lookup")],
)[0]
all_tools: list[Tool] = [namespaced_tool]
agent = Agent(name="billing-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="billing-agent", model=ScriptedModel(), tools=all_tools)
processed = run_loop.process_model_response(
agent=agent,
@@ -511,7 +511,7 @@ def test_process_model_response_prefers_namespaced_function_over_apply_patch_fal
def test_process_model_response_handles_compaction_item() -> None:
agent = Agent(name="compaction-agent", model=FakeModel())
agent = Agent(name="compaction-agent", model=ScriptedModel())
compaction_item = ResponseCompactionItem(
id="comp-1",
encrypted_content="enc",
@@ -537,7 +537,7 @@ def test_process_model_response_handles_compaction_item() -> None:
def test_process_model_response_classifies_tool_search_items() -> None:
agent = Agent(name="tool-search-agent", model=FakeModel())
agent = Agent(name="tool-search-agent", model=ScriptedModel())
tool_search_call = construct_type(
type_=ResponseOutputItem,
value={
@@ -604,7 +604,7 @@ def test_process_model_response_uses_namespace_for_duplicate_function_names() ->
tools=[billing_tool],
)
all_tools: list[Tool] = [*crm_namespace, *billing_namespace]
agent = Agent(name="billing-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="billing-agent", model=ScriptedModel(), tools=all_tools)
processed = run_loop.process_model_response(
agent=agent,
@@ -633,7 +633,7 @@ def test_process_model_response_collapses_synthetic_deferred_namespace_in_tools_
name_override="get_weather",
defer_loading=True,
)
agent = Agent(name="weather-agent", model=FakeModel(), tools=[deferred_tool])
agent = Agent(name="weather-agent", model=ScriptedModel(), tools=[deferred_tool])
processed = run_loop.process_model_response(
agent=agent,
@@ -670,7 +670,7 @@ def test_process_model_response_rejects_bare_name_for_duplicate_namespaced_funct
tools=[billing_tool],
)
all_tools: list[Tool] = [*crm_namespace, *billing_namespace]
agent = Agent(name="billing-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="billing-agent", model=ScriptedModel(), tools=all_tools)
with pytest.raises(ModelBehaviorError, match="Tool lookup_account not found"):
run_loop.process_model_response(
@@ -688,7 +688,7 @@ def test_process_model_response_uses_last_duplicate_top_level_function() -> None
first_tool = function_tool(lambda customer_id: f"first:{customer_id}", name_override="lookup")
second_tool = function_tool(lambda customer_id: f"second:{customer_id}", name_override="lookup")
all_tools: list[Tool] = [first_tool, second_tool]
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
processed = run_loop.process_model_response(
agent=agent,
@@ -707,7 +707,7 @@ def test_process_model_response_rejects_reserved_same_name_namespace_shape() ->
invalid_tool._tool_namespace = "lookup_account"
invalid_tool._tool_namespace_description = "Same-name namespace"
all_tools: list[Tool] = [invalid_tool]
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
with pytest.raises(UserError, match="synthetic namespace `lookup_account.lookup_account`"):
run_loop.process_model_response(
@@ -740,7 +740,7 @@ def test_process_model_response_rejects_qualified_name_collision_with_dotted_top
tools=[function_tool(lambda customer_id: customer_id, name_override="lookup_account")],
)[0]
all_tools: list[Tool] = [dotted_top_level_tool, namespaced_tool]
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
with pytest.raises(UserError, match="qualified name `crm.lookup_account`"):
run_loop.process_model_response(
@@ -771,7 +771,7 @@ def test_process_model_response_prefers_visible_top_level_function_over_deferred
defer_loading=True,
)
all_tools: list[Tool] = [visible_tool, deferred_tool]
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
processed = run_loop.process_model_response(
agent=agent,
@@ -801,7 +801,7 @@ def test_process_model_response_uses_internal_lookup_key_for_deferred_top_level_
defer_loading=True,
)
all_tools: list[Tool] = [visible_tool, deferred_tool]
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
processed = run_loop.process_model_response(
agent=agent,
@@ -832,7 +832,7 @@ def test_process_model_response_preserves_synthetic_namespace_for_deferred_top_l
defer_loading=True,
)
all_tools: list[Tool] = [deferred_tool]
agent = Agent(name="weather-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="weather-agent", model=ScriptedModel(), tools=all_tools)
processed = run_loop.process_model_response(
agent=agent,
@@ -858,10 +858,10 @@ def test_process_model_response_prefers_namespaced_function_over_handoff_name_co
description="Billing tools",
tools=[billing_tool],
)
handoff_target = Agent(name="lookup-agent", model=FakeModel())
handoff_target = Agent(name="lookup-agent", model=ScriptedModel())
lookup_handoff: Handoff = handoff(handoff_target, tool_name_override="lookup_account")
all_tools: list[Tool] = [*billing_namespace]
agent = Agent(name="billing-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="billing-agent", model=ScriptedModel(), tools=all_tools)
processed = run_loop.process_model_response(
agent=agent,
@@ -890,7 +890,7 @@ def test_process_model_response_prefers_namespaced_function_over_handoff_name_co
def test_process_model_response_rejects_mismatched_function_namespace() -> None:
bare_tool = function_tool(lambda customer_id: customer_id, name_override="lookup_account")
all_tools: list[Tool] = [bare_tool]
agent = Agent(name="bare-agent", model=FakeModel(), tools=all_tools)
agent = Agent(name="bare-agent", model=ScriptedModel(), tools=all_tools)
with pytest.raises(ModelBehaviorError, match="crm.lookup_account"):
run_loop.process_model_response(
@@ -911,7 +911,7 @@ def test_process_model_response_rejects_mismatched_function_namespace() -> None:
def test_process_model_response_collects_missing_function_tool_when_opted_in() -> None:
agent = Agent(name="test", model=FakeModel(), tools=[function_tool(lambda: "ok")])
agent = Agent(name="test", model=ScriptedModel(), tools=[function_tool(lambda: "ok")])
missing_call = get_function_tool_call("missing_tool", "{}", call_id="call_missing")
processed = run_loop.process_model_response(
+89 -82
View File
@@ -70,9 +70,10 @@ from agents.memory import SQLiteSession
from agents.models.chatcmpl_converter import Converter as ChatCompletionsConverter
from agents.models.openai_responses import Converter as ResponsesConverter
from agents.run_internal.turn_resolution import process_model_response
from agents.testing import ScriptedModel
from agents.tool_context import ToolContext
from .fake_model import FakeModel
from .model_test_helpers import get_exact_output_stream_step
from .test_responses import get_handoff_tool_call, get_text_message
PROGRAM_CALL_ID = "call_program"
@@ -493,7 +494,7 @@ async def test_schema_backed_direct_tool_preserves_argument_error_formatter() ->
@pytest.mark.asyncio
async def test_runner_preserves_direct_error_for_schema_backed_tool() -> None:
model = FakeModel()
model = ScriptedModel()
direct_call = ResponseFunctionToolCall(
id="function_item",
call_id=FUNCTION_CALL_ID,
@@ -501,7 +502,7 @@ async def test_runner_preserves_direct_error_for_schema_backed_tool() -> None:
arguments='{"sku":"A-1"}',
type="function_call",
)
model.add_multiple_turn_outputs([[direct_call], [get_text_message("inventory lookup failed")]])
model.extend([[direct_call], [get_text_message("inventory lookup failed")]])
@function_tool(allowed_callers=["direct", "programmatic"])
def lookup_inventory(sku: str) -> InventoryOutput:
@@ -526,7 +527,7 @@ async def test_runner_preserves_direct_error_for_schema_backed_tool() -> None:
@pytest.mark.asyncio
async def test_runner_preserves_direct_default_timeout_for_schema_backed_tool() -> None:
model = FakeModel()
model = ScriptedModel()
direct_call = ResponseFunctionToolCall(
id="function_item",
call_id=FUNCTION_CALL_ID,
@@ -534,7 +535,7 @@ async def test_runner_preserves_direct_default_timeout_for_schema_backed_tool()
arguments='{"sku":"A-1"}',
type="function_call",
)
model.add_multiple_turn_outputs([[direct_call], [get_text_message("timed out")]])
model.extend([[direct_call], [get_text_message("timed out")]])
@function_tool(allowed_callers=["direct", "programmatic"], timeout=0.01)
async def lookup_inventory(sku: str) -> InventoryOutput:
@@ -588,8 +589,8 @@ async def test_schema_backed_function_tool_accepts_conforming_custom_error_outpu
@pytest.mark.asyncio
async def test_schema_backed_programmatic_tool_accepts_conforming_custom_timeout_output() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("timeout handled")],
@@ -1190,8 +1191,8 @@ def test_process_model_response_rejects_program_items_without_programmatic_tool(
@pytest.mark.asyncio
async def test_runner_rejects_program_item_without_programmatic_tool() -> None:
model = FakeModel()
model.set_next_output([_program()])
model = ScriptedModel()
model.enqueue([_program()])
agent = Agent(name="inventory", model=model)
with pytest.raises(ModelBehaviorError, match="programmatic_tool_calling tool"):
@@ -1252,8 +1253,8 @@ def test_process_model_response_rejects_program_owned_calls_without_programmatic
@pytest.mark.asyncio
async def test_runner_does_not_execute_program_owned_call_without_programmatic_tool() -> None:
model = FakeModel()
model.set_next_output([_function_call()])
model = ScriptedModel()
model.enqueue([_function_call()])
executed = False
@function_tool(allowed_callers=["programmatic"])
@@ -1500,8 +1501,8 @@ def test_process_model_response_accepts_server_owned_parent_after_incomplete_out
@pytest.mark.asyncio
async def test_runner_does_not_execute_program_owned_call_without_parent_program() -> None:
model = FakeModel()
model.set_next_output([_function_call()])
model = ScriptedModel()
model.enqueue([_function_call()])
executed = False
@function_tool(allowed_callers=["programmatic"])
@@ -1891,12 +1892,12 @@ def test_process_model_response_accepts_allowed_program_owned_shell_output(
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_runner_executes_and_replays_programmatic_function_calls(streamed: bool) -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
[
[_program(), _function_call()],
[_program_output(), get_text_message("42 units are available")],
]
outputs = [
[_program(), _function_call()],
[_program_output(), get_text_message("42 units are available")],
]
model = ScriptedModel(
[get_exact_output_stream_step(output) for output in outputs] if streamed else outputs
)
@function_tool(allowed_callers=["programmatic"])
@@ -1923,9 +1924,9 @@ async def test_runner_executes_and_replays_programmatic_function_calls(streamed:
result = await Runner.run(agent, "Check inventory")
assert result.final_output == "42 units are available"
assert model.first_turn_args is not None
assert model.first_turn_args["model_settings"].tool_choice == "programmatic_tool_calling"
assert model.last_turn_args["model_settings"].tool_choice is None
assert bool(model.calls)
assert model.calls[0].model_settings.tool_choice == "programmatic_tool_calling"
assert model.calls[-1].model_settings.tool_choice is None
function_outputs = [
item
@@ -1941,18 +1942,21 @@ async def test_runner_executes_and_replays_programmatic_function_calls(streamed:
}
assert _caller_dict(raw_output["caller"]) == PROGRAM_CALLER
replayed_output = next(
item
for item in model.last_turn_args["input"]
if isinstance(item, dict) and item.get("type") == "function_call_output"
replayed_output = cast(
dict[str, Any],
next(
item
for item in model.calls[-1].input
if isinstance(item, dict) and item.get("type") == "function_call_output"
),
)
assert _caller_dict(replayed_output["caller"]) == PROGRAM_CALLER
@pytest.mark.asyncio
async def test_typed_programmatic_tool_preserves_input_guardrail_rejection() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("request rejected")],
@@ -1991,8 +1995,8 @@ async def test_typed_programmatic_tool_preserves_input_guardrail_rejection() ->
@pytest.mark.asyncio
async def test_typed_programmatic_tool_preserves_default_timeout_result() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("request timed out")],
@@ -2023,8 +2027,8 @@ async def test_typed_programmatic_tool_preserves_default_timeout_result() -> Non
@pytest.mark.asyncio
async def test_typed_programmatic_tool_preserves_output_guardrail_rejection() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("request rejected")],
@@ -2070,12 +2074,12 @@ async def test_typed_programmatic_tool_preserves_approval_rejection(
serialize_state: bool,
rejection_message: str | None,
) -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
[
[_program(), _function_call()],
[_program_output(), get_text_message("request rejected")],
]
outputs = [
[_program(), _function_call()],
[_program_output(), get_text_message("request rejected")],
]
model = ScriptedModel(
[get_exact_output_stream_step(output) for output in outputs] if streaming else outputs
)
@function_tool(allowed_callers=["programmatic"], needs_approval=True)
@@ -2114,19 +2118,22 @@ async def test_typed_programmatic_tool_preserves_approval_rejection(
expected_message = rejection_message or "Tool execution was not approved."
assert json.loads(function_outputs[0]["output"]) == {"error": expected_message}
assert _caller_dict(function_outputs[0]["caller"]) == PROGRAM_CALLER
assert model.last_turn_args is not None
replayed_output = next(
item
for item in model.last_turn_args["input"]
if isinstance(item, dict) and item.get("type") == "function_call_output"
assert bool(model.calls)
replayed_output = cast(
dict[str, Any],
next(
item
for item in model.calls[-1].input
if isinstance(item, dict) and item.get("type") == "function_call_output"
),
)
assert json.loads(replayed_output["output"]) == {"error": expected_message}
@pytest.mark.asyncio
async def test_rebuilt_mapping_programmatic_approval_preserves_caller() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("done")],
@@ -2164,8 +2171,8 @@ async def test_rebuilt_mapping_programmatic_approval_preserves_caller() -> None:
@pytest.mark.asyncio
async def test_rebuilt_mapping_programmatic_approval_rechecks_caller_permissions() -> None:
model = FakeModel()
model.set_next_output([_program(), _function_call()])
model = ScriptedModel()
model.enqueue([_program(), _function_call()])
executed = False
@function_tool(allowed_callers=["programmatic"], needs_approval=True)
@@ -2197,8 +2204,8 @@ async def test_rebuilt_mapping_programmatic_approval_rechecks_caller_permissions
@pytest.mark.asyncio
@pytest.mark.parametrize("parent_state", ["missing", "completed"])
async def test_rebuilt_programmatic_approval_requires_active_parent(parent_state: str) -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("done")],
@@ -2250,8 +2257,8 @@ async def test_rebuilt_programmatic_approval_requires_active_parent(parent_state
@pytest.mark.asyncio
async def test_typed_programmatic_tool_preserves_pre_approval_guardrail_rejection() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("request rejected")],
@@ -2290,7 +2297,7 @@ async def test_typed_programmatic_tool_preserves_pre_approval_guardrail_rejectio
@pytest.mark.asyncio
async def test_runner_handles_multiple_pauses_from_one_program() -> None:
model = FakeModel()
model = ScriptedModel()
second_call = ResponseFunctionToolCall(
id="function_item_2",
call_id="call_lookup_2",
@@ -2299,7 +2306,7 @@ async def test_runner_handles_multiple_pauses_from_one_program() -> None:
caller=CallerProgram(type="program", caller_id=PROGRAM_CALL_ID),
type="function_call",
)
model.add_multiple_turn_outputs(
model.extend(
[
[_program(), _function_call()],
[_program_output("incomplete"), second_call],
@@ -2334,13 +2341,13 @@ async def test_runner_handles_multiple_pauses_from_one_program() -> None:
assert result.final_output == "84 units are available"
assert calls == ["A-1", "B-2"]
assert model.last_turn_args["model_settings"].tool_choice is None
assert model.calls[-1].model_settings.tool_choice is None
assert len([item for item in result.new_items if isinstance(item, ToolCallOutputItem)]) == 4
@pytest.mark.asyncio
async def test_runner_executes_programmatic_batch_calls_concurrently() -> None:
model = FakeModel()
model = ScriptedModel()
batch_calls = [
ResponseFunctionToolCall(
id=f"function_item_{index}",
@@ -2352,7 +2359,7 @@ async def test_runner_executes_programmatic_batch_calls_concurrently() -> None:
)
for index in range(9)
]
model.add_multiple_turn_outputs(
model.extend(
[
[_program(), *batch_calls],
[_program_output(), get_text_message("batch complete")],
@@ -2398,8 +2405,8 @@ async def test_runner_executes_programmatic_batch_calls_concurrently() -> None:
@pytest.mark.asyncio
async def test_previous_response_id_continuation_sends_only_program_function_output() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("done")],
@@ -2419,8 +2426,8 @@ async def test_previous_response_id_continuation_sends_only_program_function_out
result = await Runner.run(agent, "Check inventory", auto_previous_response_id=True)
assert result.final_output == "done"
assert model.last_turn_args["previous_response_id"] == "resp-789"
last_input = model.last_turn_args["input"]
assert model.calls[-1].previous_response_id == "resp-789"
last_input = model.calls[-1].input
assert isinstance(last_input, list)
assert len(last_input) == 1
function_output = cast(dict[str, Any], last_input[0])
@@ -2433,8 +2440,8 @@ async def test_previous_response_id_continuation_sends_only_program_function_out
async def test_previous_response_id_continuation_accepts_server_owned_program_output(
streamed: bool,
) -> None:
model = FakeModel()
model.set_next_output([_program_output(), get_text_message("done")])
output = [_program_output(), get_text_message("done")]
model = ScriptedModel([get_exact_output_stream_step(output) if streamed else output])
@function_tool(allowed_callers=["programmatic"])
def lookup_inventory(sku: str) -> InventoryOutput:
@@ -2470,13 +2477,13 @@ async def test_previous_response_id_continuation_accepts_server_owned_program_ou
)
assert result.final_output == "done"
assert model.last_turn_args["previous_response_id"] == "response_with_program_parent"
assert model.calls[-1].previous_response_id == "response_with_program_parent"
@pytest.mark.asyncio
async def test_previous_response_id_continuation_accepts_repeated_program_pause() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_function_call()],
[_program_output(), get_text_message("done")],
@@ -2520,8 +2527,8 @@ async def test_previous_response_id_continuation_accepts_repeated_program_pause(
async def test_run_state_round_trip_preserves_server_owned_program_parent(
parent_source: str,
) -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_function_call()],
[_program_output(), get_text_message("done")],
@@ -2566,13 +2573,13 @@ async def test_run_state_round_trip_preserves_server_owned_program_parent(
assert executed is True
assert result.final_output == "done"
assert model.last_turn_args["previous_response_id"] == "resp-789"
assert model.calls[-1].previous_response_id == "resp-789"
@pytest.mark.asyncio
async def test_sqlite_session_round_trip_preserves_program_history_and_caller() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_text_message("done")],
@@ -2614,9 +2621,9 @@ async def test_sqlite_session_round_trip_preserves_program_history_and_caller()
@pytest.mark.asyncio
async def test_nested_handoff_summarizes_complete_programmatic_transcript() -> None:
model = FakeModel()
model = ScriptedModel()
delegate = Agent(name="delegate", model=model)
model.add_multiple_turn_outputs(
model.extend(
[
[_program(), _function_call()],
[_program_output(), get_handoff_tool_call(delegate)],
@@ -2692,8 +2699,8 @@ async def test_non_function_programmatic_outputs_preserve_caller() -> None:
caller = cast(Any, PROGRAM_CALLER)
async def run_tool(tool: Any, tool_call: Any) -> dict[str, Any]:
model = FakeModel()
model.add_multiple_turn_outputs([[_program(), tool_call], [get_text_message("done")]])
model = ScriptedModel()
model.extend([[_program(), tool_call], [get_text_message("done")]])
agent = Agent(
name="tool agent",
model=model,
@@ -2752,13 +2759,13 @@ async def test_non_function_programmatic_outputs_preserve_caller() -> None:
apply_patch_output = await run_tool(
ApplyPatchTool(editor=Editor(), allowed_callers=["programmatic"]),
ResponseApplyPatchToolCall(
id="apply_patch_item",
call_id="call_apply_patch",
operation=OperationCreateFile(type="create_file", path="example.txt", diff="hello"),
status="completed",
type="apply_patch_call",
caller=caller,
),
{
"id": "apply_patch_item",
"call_id": "call_apply_patch",
"operation": {"type": "create_file", "path": "example.txt", "diff": "hello"},
"status": "completed",
"type": "apply_patch_call",
"caller": caller,
},
)
assert _caller_dict(apply_patch_output["caller"]) == PROGRAM_CALLER
+41 -37
View File
@@ -4,13 +4,13 @@ import pytest
from openai.types.responses.response_create_params import ContextManagement, PromptCacheOptions
from agents import Agent, ModelSettings, RunConfig, Runner
from agents.testing import ScriptedModel
from .fake_model import FakeModel, PromptCacheFakeModel
from .test_responses import get_function_tool, get_function_tool_call, get_text_message
from .utils.simple_session import SimpleListSession
def _sent_prompt_cache_key(model: FakeModel, *, first_turn: bool = False) -> str | None:
def _sent_prompt_cache_key(model: ScriptedModel, *, first_turn: bool = False) -> str | None:
model_settings = _sent_model_settings(model, first_turn=first_turn)
extra_args = model_settings.extra_args or {}
value = extra_args.get("prompt_cache_key")
@@ -18,23 +18,27 @@ def _sent_prompt_cache_key(model: FakeModel, *, first_turn: bool = False) -> str
return value
def _sent_model_settings(model: FakeModel, *, first_turn: bool = False) -> ModelSettings:
args = model.first_turn_args if first_turn else model.last_turn_args
assert args is not None
model_settings = args["model_settings"]
def _sent_model_settings(model: ScriptedModel, *, first_turn: bool = False) -> ModelSettings:
call = model.calls[0] if first_turn else model.calls[-1]
model_settings = call.model_settings
assert isinstance(model_settings, ModelSettings)
return model_settings
class DefaultPromptCacheDisabledFakeModel(FakeModel):
class PromptCacheScriptedModel(ScriptedModel):
def _supports_default_prompt_cache_key(self) -> bool:
return True
class DefaultPromptCacheDisabledScriptedModel(ScriptedModel):
def _supports_default_prompt_cache_key(self) -> bool:
return False
@pytest.mark.asyncio
async def test_runner_generates_prompt_cache_key_by_default() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="test", model=model)
await Runner.run(agent, "hi")
@@ -46,21 +50,21 @@ async def test_runner_generates_prompt_cache_key_by_default() -> None:
@pytest.mark.asyncio
async def test_runner_adds_prompt_cache_key_without_adding_model_call_keyword() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="test", model=model)
await Runner.run(agent, "hi")
# PromptCacheFakeModel uses the public Model.get_response() signature. If the runner added
# PromptCacheScriptedModel uses the public Model.get_response() signature. If the runner added
# prompt_cache_key as a direct model-call keyword, this run would fail before this assertion.
assert _sent_prompt_cache_key(model) is not None
@pytest.mark.asyncio
async def test_runner_reuses_generated_prompt_cache_key_across_turns() -> None:
model = PromptCacheFakeModel()
model.add_multiple_turn_outputs(
model = PromptCacheScriptedModel()
model.extend(
[
[get_function_tool_call("lookup", "{}")],
[get_text_message("done")],
@@ -78,8 +82,8 @@ async def test_runner_reuses_generated_prompt_cache_key_across_turns() -> None:
@pytest.mark.asyncio
async def test_runner_skips_generated_prompt_cache_key_when_model_disables_default() -> None:
model = DefaultPromptCacheDisabledFakeModel()
model.set_next_output([get_text_message("done")])
model = DefaultPromptCacheDisabledScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="test", model=model)
await Runner.run(agent, "hi")
@@ -89,8 +93,8 @@ async def test_runner_skips_generated_prompt_cache_key_when_model_disables_defau
@pytest.mark.asyncio
async def test_runner_respects_existing_extra_args_prompt_cache_key() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(
name="test",
model=model,
@@ -106,8 +110,8 @@ async def test_runner_respects_existing_extra_args_prompt_cache_key() -> None:
@pytest.mark.asyncio
async def test_runner_respects_existing_extra_body_prompt_cache_key() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(
name="test",
model=model,
@@ -124,8 +128,8 @@ async def test_runner_respects_existing_extra_body_prompt_cache_key() -> None:
@pytest.mark.asyncio
async def test_runner_generates_prompt_cache_key_with_unrelated_extra_args() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
model_settings = ModelSettings(extra_args={"service_tier": "flex"})
agent = Agent(
name="test",
@@ -146,8 +150,8 @@ async def test_runner_generates_prompt_cache_key_with_unrelated_extra_args() ->
@pytest.mark.asyncio
async def test_runner_preserves_context_management_when_adding_prompt_cache_key() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
context_management: list[ContextManagement] = [
{"type": "compaction", "compact_threshold": 200000}
]
@@ -170,8 +174,8 @@ async def test_runner_preserves_context_management_when_adding_prompt_cache_key(
@pytest.mark.asyncio
async def test_runner_preserves_prompt_cache_options_when_adding_prompt_cache_key() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
prompt_cache_options: PromptCacheOptions = {"mode": "explicit", "ttl": "30m"}
model_settings = ModelSettings(prompt_cache_options=prompt_cache_options)
agent = Agent(name="test", model=model, model_settings=model_settings)
@@ -188,8 +192,8 @@ async def test_runner_preserves_prompt_cache_options_when_adding_prompt_cache_ke
@pytest.mark.asyncio
async def test_runner_skips_generated_key_when_model_settings_has_prompt_cache_keys() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(
name="test",
model=model,
@@ -206,8 +210,8 @@ async def test_runner_skips_generated_key_when_model_settings_has_prompt_cache_k
@pytest.mark.asyncio
async def test_runner_uses_group_id_as_stable_prompt_cache_key_boundary() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="test", model=model)
await Runner.run(agent, "hi", run_config=RunConfig(group_id="thread-123"))
@@ -219,8 +223,8 @@ async def test_runner_uses_group_id_as_stable_prompt_cache_key_boundary() -> Non
@pytest.mark.asyncio
async def test_runner_uses_session_id_as_stable_prompt_cache_key_boundary() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="test", model=model)
session = SimpleListSession(session_id="session-123")
@@ -233,8 +237,8 @@ async def test_runner_uses_session_id_as_stable_prompt_cache_key_boundary() -> N
@pytest.mark.asyncio
async def test_streamed_runner_generates_prompt_cache_key_by_default() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("done")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="test", model=model)
result = Runner.run_streamed(agent, "hi")
@@ -248,8 +252,8 @@ async def test_streamed_runner_generates_prompt_cache_key_by_default() -> None:
@pytest.mark.asyncio
async def test_run_state_preserves_generated_prompt_cache_key_on_resume() -> None:
model = PromptCacheFakeModel()
model.set_next_output([get_text_message("first")])
model = PromptCacheScriptedModel()
model.enqueue([get_text_message("first")])
agent = Agent(name="test", model=model)
first_result = await Runner.run(agent, "hi")
@@ -257,7 +261,7 @@ async def test_run_state_preserves_generated_prompt_cache_key_on_resume() -> Non
state = first_result.to_state()
restored_state = await type(state).from_string(agent, state.to_string())
model.set_next_output([get_text_message("second")])
model.enqueue([get_text_message("second")])
await Runner.run(agent, restored_state)
assert first_key is not None
+11 -11
View File
@@ -1,8 +1,8 @@
import pytest
from agents import Agent, run_demo_loop
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import (
get_function_tool,
get_function_tool_call,
@@ -14,8 +14,8 @@ from .test_responses import (
@pytest.mark.asyncio
async def test_run_demo_loop_conversation(monkeypatch, capsys):
model = FakeModel()
model.add_multiple_turn_outputs([[get_text_message("hello")], [get_text_message("good")]])
model = ScriptedModel()
model.extend([[get_text_message("hello")], [get_text_message("good")]])
agent = Agent(name="test", model=model)
@@ -27,7 +27,7 @@ async def test_run_demo_loop_conversation(monkeypatch, capsys):
output = capsys.readouterr().out
assert "hello" in output
assert "good" in output
assert model.last_turn_args["input"] == [
assert model.calls[-1].input == [
get_text_input_item("Hi"),
get_text_message("hello").model_dump(exclude_unset=True),
get_text_input_item("How are you?"),
@@ -36,7 +36,7 @@ async def test_run_demo_loop_conversation(monkeypatch, capsys):
@pytest.mark.asyncio
async def test_run_demo_loop_streaming(monkeypatch, capsys):
model = FakeModel()
model = ScriptedModel()
target_agent = Agent(name="target", model=model)
agent = Agent(
name="test",
@@ -47,7 +47,7 @@ async def test_run_demo_loop_streaming(monkeypatch, capsys):
# A single user turn that exercises every streamed event branch:
# a tool call, the tool output, a handoff (agent update), then a text answer.
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("foo", "{}")],
[get_handoff_tool_call(target_agent)],
@@ -69,7 +69,7 @@ async def test_run_demo_loop_streaming(monkeypatch, capsys):
@pytest.mark.asyncio
async def test_run_demo_loop_exits_on_eof(monkeypatch, capsys):
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
def raise_eof(_=" > ") -> str:
@@ -80,13 +80,13 @@ async def test_run_demo_loop_exits_on_eof(monkeypatch, capsys):
await run_demo_loop(agent, stream=False)
# The loop should terminate cleanly without ever invoking the model.
assert model.last_turn_args == {}
assert not model.calls
@pytest.mark.asyncio
async def test_run_demo_loop_skips_empty_input(monkeypatch, capsys):
model = FakeModel()
model.add_multiple_turn_outputs([[get_text_message("hello")]])
model = ScriptedModel()
model.extend([[get_text_message("hello")]])
agent = Agent(name="test", model=model)
# Empty lines are ignored; only the non-empty input reaches the runner.
@@ -97,4 +97,4 @@ async def test_run_demo_loop_skips_empty_input(monkeypatch, capsys):
output = capsys.readouterr().out
assert "hello" in output
assert model.last_turn_args["input"] == [get_text_input_item("Hi")]
assert model.calls[-1].input == [get_text_input_item("Hi")]
+6 -6
View File
@@ -6,7 +6,7 @@ from openai.types.responses.response_usage import InputTokensDetails, OutputToke
from agents import ModelBehaviorError, ModelSettings, ModelTracing, OpenAIResponsesModel, trace
from agents.tracing.span_data import ResponseSpanData
from tests import fake_model
from tests import model_test_helpers
from .testing_processor import assert_no_spans, fetch_normalized_spans, fetch_ordered_spans
@@ -49,7 +49,7 @@ class DummyResponse:
def __aiter__(self):
yield ResponseCompletedEvent(
type="response.completed",
response=fake_model.get_response_obj(self.output),
response=model_test_helpers.get_response_obj(self.output),
sequence_number=0,
)
@@ -249,7 +249,7 @@ async def test_stream_response_creates_trace(monkeypatch):
async def __aiter__(self):
yield ResponseCompletedEvent(
type="response.completed",
response=fake_model.get_response_obj([], "dummy-id-123"),
response=model_test_helpers.get_response_obj([], "dummy-id-123"),
sequence_number=0,
)
@@ -322,7 +322,7 @@ async def test_stream_response_failed_or_incomplete_terminal_event_creates_trace
class DummyTerminalEvent:
def __init__(self):
self.type = terminal_event_type
self.response = fake_model.get_response_obj([], "dummy-id-terminal")
self.response = model_test_helpers.get_response_obj([], "dummy-id-terminal")
self.sequence_number = 0
class DummyStream:
@@ -391,7 +391,7 @@ async def test_stream_non_data_tracing_doesnt_set_response_id(monkeypatch):
async def __aiter__(self):
yield ResponseCompletedEvent(
type="response.completed",
response=fake_model.get_response_obj([], "dummy-id-123"),
response=model_test_helpers.get_response_obj([], "dummy-id-123"),
sequence_number=0,
)
@@ -467,7 +467,7 @@ async def test_stream_disabled_tracing_doesnt_create_span(monkeypatch):
async def __aiter__(self):
yield ResponseCompletedEvent(
type="response.completed",
response=fake_model.get_response_obj([], "dummy-id-123"),
response=model_test_helpers.get_response_obj([], "dummy-id-123"),
sequence_number=0,
)
+8 -7
View File
@@ -1,13 +1,14 @@
from __future__ import annotations
from typing import Any, cast
from unittest import mock
import pytest
from agents import Agent, Runner
from agents.run import AgentRunner, set_default_agent_runner
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import get_text_input_item, get_text_message
@@ -16,7 +17,7 @@ async def test_static_run_methods_call_into_default_runner() -> None:
runner = mock.Mock(spec=AgentRunner)
set_default_agent_runner(runner)
agent = Agent(name="test", model=FakeModel())
agent = Agent(name="test", model=ScriptedModel())
await Runner.run(agent, input="test")
runner.run.assert_called_once()
@@ -29,16 +30,16 @@ async def test_static_run_methods_call_into_default_runner() -> None:
@pytest.mark.asyncio
async def test_run_preserves_duplicate_user_messages() -> None:
model = FakeModel()
model.set_next_output([get_text_message("done")])
model = ScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="test", model=model)
input_items = [get_text_input_item("repeat"), get_text_input_item("repeat")]
await Runner.run(agent, input=input_items)
sent_input = model.last_turn_args["input"]
sent_input = model.calls[-1].input
assert isinstance(sent_input, list)
assert len(sent_input) == 2
assert sent_input[0]["content"] == "repeat"
assert sent_input[1]["content"] == "repeat"
assert cast(dict[str, Any], sent_input[0])["content"] == "repeat"
assert cast(dict[str, Any], sent_input[1])["content"] == "repeat"
+25 -23
View File
@@ -19,8 +19,8 @@ from agents.run import __all__ as run_exports
from agents.run_config import SandboxConcurrencyLimits, SandboxRunConfig
from agents.sandbox.manifest import Manifest
from agents.sandbox.snapshot import NoopSnapshotSpec
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import get_text_message
@@ -30,7 +30,7 @@ class DummyProvider(ModelProvider):
def __init__(self, model_to_return: Model | None = None) -> None:
self.last_requested: str | None = None
self.model_to_return: Model = model_to_return or FakeModel()
self.model_to_return: Model = model_to_return or ScriptedModel()
def get_model(self, model_name: str | None) -> Model:
# record the requested model name and return our test model
@@ -119,7 +119,7 @@ def test_run_config_rejects_unknown_first_party_dictionary_fields(
@pytest.mark.asyncio
async def test_runner_accepts_dictionary_run_configuration() -> None:
model = FakeModel(initial_output=[get_text_message("done")])
model = ScriptedModel(steps=[[get_text_message("done")]])
agent = Agent(name="test", model=model)
result = await Runner.run(
@@ -138,8 +138,8 @@ async def test_model_provider_on_run_config_is_used_for_agent_model_name() -> No
provided in the ``RunConfig``, the ``Runner`` should resolve the model using the
``model_provider`` on the ``RunConfig``.
"""
fake_model = FakeModel(initial_output=[get_text_message("from-provider")])
provider = DummyProvider(model_to_return=fake_model)
scripted_model = ScriptedModel(steps=[[get_text_message("from-provider")]])
provider = DummyProvider(model_to_return=scripted_model)
agent = Agent(name="test", model="test-model")
run_config = RunConfig(model_provider=provider)
result = await Runner.run(agent, input="any", run_config=run_config)
@@ -154,8 +154,8 @@ async def test_run_config_model_name_override_takes_precedence() -> None:
When a model name string is set on the RunConfig, then that name should be looked up
using the RunConfig's model_provider, and should override any model on the agent.
"""
fake_model = FakeModel(initial_output=[get_text_message("override-name")])
provider = DummyProvider(model_to_return=fake_model)
scripted_model = ScriptedModel(steps=[[get_text_message("override-name")]])
provider = DummyProvider(model_to_return=scripted_model)
agent = Agent(name="test", model="agent-model")
run_config = RunConfig(model="override-name", model_provider=provider)
result = await Runner.run(agent, input="any", run_config=run_config)
@@ -179,14 +179,15 @@ async def test_run_config_model_name_override_uses_model_specific_default_settin
than the default fallback model.
"""
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", "gpt-5.4-mini")
fake_model = FakeModel(initial_output=[get_text_message("override-name")])
provider = DummyProvider(model_to_return=fake_model)
scripted_model = ScriptedModel(steps=[[get_text_message("override-name")]])
provider = DummyProvider(model_to_return=scripted_model)
agent = Agent(name="test")
run_config = RunConfig(model=model_name, model_provider=provider)
result = await Runner.run(agent, input="any", run_config=run_config)
assert result.final_output == "override-name"
assert fake_model.first_turn_args is not None
model_settings = fake_model.first_turn_args["model_settings"]
assert bool(scripted_model.calls)
model_settings = scripted_model.calls[0].model_settings
assert model_settings.reasoning is not None
assert model_settings.reasoning.effort == reasoning_effort
assert model_settings.verbosity == "low"
@@ -199,8 +200,8 @@ async def test_run_config_model_settings_override_implicit_model_specific_defaul
RunConfig model settings should overlay the implicit defaults for the resolved model name.
"""
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", "gpt-5.4-mini")
fake_model = FakeModel(initial_output=[get_text_message("override-name")])
provider = DummyProvider(model_to_return=fake_model)
scripted_model = ScriptedModel(steps=[[get_text_message("override-name")]])
provider = DummyProvider(model_to_return=scripted_model)
agent = Agent(name="test")
run_config = RunConfig(
model="gpt-5",
@@ -209,8 +210,9 @@ async def test_run_config_model_settings_override_implicit_model_specific_defaul
)
result = await Runner.run(agent, input="any", run_config=run_config)
assert result.final_output == "override-name"
assert fake_model.first_turn_args is not None
model_settings = fake_model.first_turn_args["model_settings"]
assert bool(scripted_model.calls)
model_settings = scripted_model.calls[0].model_settings
assert model_settings.reasoning is not None
assert model_settings.reasoning.effort == "low"
assert model_settings.verbosity == "low"
assert model_settings.temperature == 0.3
@@ -222,11 +224,11 @@ async def test_run_config_model_override_object_takes_precedence() -> None:
When a concrete Model instance is set on the RunConfig, then that instance should be
returned by AgentRunner._get_model regardless of the agent's model.
"""
fake_model = FakeModel(initial_output=[get_text_message("override-object")])
scripted_model = ScriptedModel(steps=[[get_text_message("override-object")]])
agent = Agent(name="test", model="agent-model")
run_config = RunConfig(model=fake_model)
run_config = RunConfig(model=scripted_model)
result = await Runner.run(agent, input="any", run_config=run_config)
# Our FakeModel on the RunConfig should have been used.
# The ScriptedModel on the RunConfig should have been used.
assert result.final_output == "override-object"
@@ -237,13 +239,13 @@ async def test_agent_model_object_is_used_when_present() -> None:
not specify a model override, then that object should be used directly without
consulting the RunConfig's model_provider.
"""
fake_model = FakeModel(initial_output=[get_text_message("from-agent-object")])
scripted_model = ScriptedModel(steps=[[get_text_message("from-agent-object")]])
provider = DummyProvider()
agent = Agent(name="test", model=fake_model)
agent = Agent(name="test", model=scripted_model)
run_config = RunConfig(model_provider=provider)
result = await Runner.run(agent, input="any", run_config=run_config)
# The dummy provider should never have been called, and the output should come from
# the FakeModel on the agent.
# the ScriptedModel on the agent.
assert provider.last_requested is None
assert result.final_output == "from-agent-object"
@@ -352,7 +354,7 @@ def test_tool_name_collision_policy_rejects_invalid_value() -> None:
@pytest.mark.asyncio
async def test_runner_dictionary_rejects_invalid_tool_name_collision_policy() -> None:
model = FakeModel(initial_output=[get_text_message("done")])
model = ScriptedModel(steps=[[get_text_message("done")]])
agent = Agent(name="test", model=model)
with pytest.raises(
@@ -365,4 +367,4 @@ async def test_runner_dictionary_rejects_invalid_tool_name_collision_policy() ->
run_config={"tool_name_collision_policy": cast(Any, "erorr")},
)
assert model.first_turn_args is None
assert not model.calls
+5 -5
View File
@@ -3,16 +3,16 @@ import json
import pytest
from agents import Agent, MaxTurnsExceeded, RunErrorDetails, Runner
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import get_function_tool, get_function_tool_call, get_text_message
@pytest.mark.asyncio
async def test_run_error_includes_data():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model, tools=[get_function_tool("foo", "res")])
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("1"), get_function_tool_call("foo", json.dumps({"a": "b"}))],
[get_text_message("done")],
@@ -29,9 +29,9 @@ async def test_run_error_includes_data():
@pytest.mark.asyncio
async def test_streamed_run_error_includes_data():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model, tools=[get_function_tool("foo", "res")])
model.add_multiple_turn_outputs(
model.extend(
[
[get_text_message("1"), get_function_tool_call("foo", json.dumps({"a": "b"}))],
[get_text_message("done")],
+30 -34
View File
@@ -7,15 +7,14 @@ import pytest
from agents.agent import Agent
from agents.items import ItemHelpers, ModelResponse, TResponseInputItem
from agents.lifecycle import AgentHooks, RunHooks
from agents.models.interface import Model
from agents.run import Runner
from agents.run_context import AgentHookContext, RunContextWrapper, TContext
from agents.run_internal.run_loop import validate_run_hooks
from agents.testing import ModelStep, ScriptedModel
from agents.tool import Tool, function_tool
from agents.tool_context import ToolContext
from tests.test_agent_llm_hooks import AgentHooksForTests
from .fake_model import FakeModel
from .test_responses import (
get_function_tool,
get_function_tool_call,
@@ -89,11 +88,11 @@ class RunHooksForTests(RunHooks):
@pytest.mark.asyncio
async def test_async_run_hooks_with_llm():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
# Simulate a single LLM call producing an output:
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
await Runner.run(agent, input="hello", hooks=hooks)
# Expect one on_agent_start, one on_llm_start, one on_llm_end, and one on_agent_end
assert hooks.events == {
@@ -107,10 +106,10 @@ async def test_async_run_hooks_with_llm():
# test_sync_run_hook_with_llm()
def test_sync_run_hook_with_llm():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
# Simulate a single LLM call producing an output:
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
Runner.run_sync(agent, input="hello", hooks=hooks)
# Expect one on_agent_start, one on_llm_start, one on_llm_end, and one on_agent_end
assert hooks.events == {
@@ -125,10 +124,10 @@ def test_sync_run_hook_with_llm():
@pytest.mark.asyncio
async def test_streamed_run_hooks_with_llm():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
# Simulate a single LLM call producing an output:
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
stream = Runner.run_streamed(agent, input="hello", hooks=hooks)
async for event in stream.stream_events():
@@ -160,13 +159,13 @@ async def test_streamed_run_hooks_with_llm():
async def test_async_run_hooks_with_agent_hooks_with_llm():
hooks = RunHooksForTests()
agent_hooks = AgentHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[], hooks=agent_hooks
)
# Simulate a single LLM call producing an output:
model.set_next_output([get_text_message("hello")])
model.enqueue([get_text_message("hello")])
await Runner.run(agent, input="hello", hooks=hooks)
# Expect one on_agent_start, one on_llm_start, one on_llm_end, and one on_agent_end
assert hooks.events == {
@@ -182,13 +181,13 @@ async def test_async_run_hooks_with_agent_hooks_with_llm():
@pytest.mark.asyncio
async def test_run_hooks_llm_error_non_streaming(monkeypatch):
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
async def boom(*args, **kwargs):
raise RuntimeError("boom")
monkeypatch.setattr(FakeModel, "get_response", boom, raising=True)
monkeypatch.setattr(ScriptedModel, "get_response", boom, raising=True)
with pytest.raises(RuntimeError, match="boom"):
await Runner.run(agent, input="hello", hooks=hooks)
@@ -206,7 +205,7 @@ class DummyAgentHooks(AgentHooks):
@pytest.mark.asyncio
async def test_runner_run_rejects_agent_hooks():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model)
hooks = cast(RunHooks, DummyAgentHooks())
@@ -215,7 +214,7 @@ async def test_runner_run_rejects_agent_hooks():
def test_runner_run_streamed_rejects_agent_hooks():
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="A", model=model)
hooks = cast(RunHooks, DummyAgentHooks())
@@ -228,13 +227,9 @@ def test_validate_run_hooks_rejects_non_hook_objects() -> None:
validate_run_hooks(object())
class BoomModel(Model):
async def get_response(self, *a, **k):
raise AssertionError("get_response should not be called in streaming test")
async def stream_response(self, *a, **k):
yield {"foo": "bar"}
raise RuntimeError("stream blew up")
async def _failing_stream(_call):
yield {"foo": "bar"}
raise RuntimeError("stream blew up")
@pytest.mark.asyncio
@@ -244,7 +239,8 @@ async def test_streamed_run_hooks_llm_error(monkeypatch):
but do NOT emit on_llm_end (current behavior), and the exception propagates.
"""
hooks = RunHooksForTests()
agent = Agent(name="A", model=BoomModel(), tools=[get_function_tool("f", "res")], handoffs=[])
model = ScriptedModel([ModelStep.stream(_failing_stream)])
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
stream = Runner.run_streamed(agent, input="hello", hooks=hooks)
@@ -276,10 +272,10 @@ class RunHooksWithTurnInput(RunHooks):
async def test_run_hooks_receives_turn_input_string():
"""Test that on_agent_start receives turn_input when input is a string."""
hooks = RunHooksWithTurnInput()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("response")])
model.enqueue([get_text_message("response")])
await Runner.run(agent, input="hello world", hooks=hooks)
assert len(hooks.captured_turn_inputs) == 1
@@ -293,7 +289,7 @@ async def test_run_hooks_receives_turn_input_string():
async def test_run_hooks_receives_turn_input_list():
"""Test that on_agent_start receives turn_input when input is a list."""
hooks = RunHooksWithTurnInput()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
input_items: list[Any] = [
@@ -301,7 +297,7 @@ async def test_run_hooks_receives_turn_input_list():
{"role": "user", "content": "second message"},
]
model.set_next_output([get_text_message("response")])
model.enqueue([get_text_message("response")])
await Runner.run(agent, input=input_items, hooks=hooks)
assert len(hooks.captured_turn_inputs) == 1
@@ -315,10 +311,10 @@ async def test_run_hooks_receives_turn_input_list():
async def test_run_hooks_receives_turn_input_streamed():
"""Test that on_agent_start receives turn_input in streamed mode."""
hooks = RunHooksWithTurnInput()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("response")])
model.enqueue([get_text_message("response")])
result = Runner.run_streamed(agent, input="streamed input", hooks=hooks)
async for _ in result.stream_events():
pass
@@ -332,7 +328,7 @@ async def test_run_hooks_receives_turn_input_streamed():
@pytest.mark.asyncio
async def test_run_hooks_count_tool_and_handoff_invocations():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(
@@ -342,7 +338,7 @@ async def test_run_hooks_count_tool_and_handoff_invocations():
tools=[get_function_tool("some_function", "result")],
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
[get_text_message("a_message"), get_handoff_tool_call(agent_1)],
@@ -362,7 +358,7 @@ async def test_run_hooks_count_tool_and_handoff_invocations():
@pytest.mark.asyncio
async def test_streamed_run_hooks_count_tool_and_handoff_invocations():
hooks = RunHooksForTests()
model = FakeModel()
model = ScriptedModel()
agent_1 = Agent(name="test_1", model=model)
agent_2 = Agent(
@@ -372,7 +368,7 @@ async def test_streamed_run_hooks_count_tool_and_handoff_invocations():
tools=[get_function_tool("some_function", "result")],
)
model.add_multiple_turn_outputs(
model.extend(
[
[
get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="call_1"),
@@ -433,10 +429,10 @@ async def test_tool_end_hooks_receive_raw_function_tool_result():
run_hooks = RecordingRunHooks()
agent_hooks = RecordingAgentHooks()
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="test", model=model, tools=[get_metadata], hooks=agent_hooks)
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("get_metadata", "{}")],
[get_text_message("done")],
+2 -2
View File
@@ -30,8 +30,8 @@ from agents.run_internal.run_loop import (
SingleStepResult,
)
from agents.run_state import RunState
from agents.testing import ScriptedModel
from agents.usage import Usage
from tests.fake_model import FakeModel
from tests.test_responses import get_function_tool_call, get_text_message
from tests.utils.hitl import (
make_agent,
@@ -44,7 +44,7 @@ from tests.utils.simple_session import SimpleListSession
@pytest.mark.asyncio
async def test_resolve_interrupted_turn_final_output_short_circuit(monkeypatch) -> None:
agent: Agent[dict[str, str]] = make_agent(model=FakeModel())
agent: Agent[dict[str, str]] = make_agent(model=ScriptedModel())
context_wrapper = make_context_wrapper()
async def fake_execute_tool_plan(*_: object, **__: object):
+100 -194
View File
@@ -4,10 +4,9 @@ from __future__ import annotations
import gc
import importlib
import io
import json
import logging
from collections.abc import AsyncIterator, Callable, Mapping
from collections.abc import Callable, Mapping
from copy import deepcopy
from dataclasses import dataclass
from datetime import datetime
@@ -39,7 +38,7 @@ from openai.types.responses.response_usage import InputTokensDetails
from openai.types.responses.tool_param import Mcp
from pydantic import BaseModel, ValidationError
from agents import Agent, Model, ModelSettings, RunConfig, RunHooks, Runner, handoff, trace
from agents import Agent, ModelSettings, RunConfig, RunHooks, Runner, handoff, trace
from agents._tool_invocation import tool_invocation_identity_and_scope
from agents.computer import Computer
from agents.exceptions import ModelBehaviorError, UserError
@@ -66,7 +65,6 @@ from agents.items import (
ToolSearchOutputItem,
TResponseInputItem,
TResponseOutputItem,
TResponseStreamEvent,
)
from agents.run_context import RunContextWrapper
from agents.run_error_handlers import RunErrorHandlerResult, RunErrorHandlers
@@ -110,9 +108,8 @@ from agents.sandbox import Manifest
from agents.sandbox.capabilities.capability import Capability
from agents.sandbox.entries import BaseEntry, Mount, MountStrategyBase
from agents.sandbox.sandboxes.unix_local import UnixLocalSandboxClient, UnixLocalSandboxSessionState
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
from agents.sandbox.snapshot import LocalSnapshot, NoopSnapshot
from agents.sandbox.types import ExecResult
from agents.sandbox.snapshot import LocalSnapshot
from agents.testing import ModelCall, ModelStep, ScriptedModel, scripted_sandbox_session
from agents.tool import (
ApplyPatchTool,
ComputerTool,
@@ -135,9 +132,7 @@ from agents.tool_guardrails import (
)
from agents.tracing.traces import TraceState
from agents.usage import Usage
from tests.utils.factories import TestSessionState
from .fake_model import FakeModel
from .test_responses import (
get_final_output_message,
get_function_tool_call,
@@ -166,49 +161,6 @@ _NEXT_UNSUPPORTED_SCHEMA_VERSION = f"{_CURRENT_SCHEMA_MAJOR}.{int(_CURRENT_SCHEM
TContext = TypeVar("TContext")
class _IdentitySandboxSession(BaseSandboxSession):
def __init__(self, root: str) -> None:
self.state = TestSessionState(
manifest=Manifest(root=root),
snapshot=NoopSnapshot(id=f"snapshot:{root}"),
)
async def start(self) -> None:
return None
async def stop(self) -> None:
return None
async def shutdown(self) -> None:
return None
async def running(self) -> bool:
return True
async def read(self, path: Path, *, user: object = None) -> Any:
_ = (path, user)
raise AssertionError("read() should not be called")
async def write(self, path: Path, data: io.IOBase, *, user: object = None) -> None:
_ = (path, data, user)
raise AssertionError("write() should not be called")
async def _exec_internal(
self,
*command: Any,
timeout: float | None = None,
) -> ExecResult:
_ = (command, timeout)
raise AssertionError("_exec_internal() should not be called")
async def persist_workspace(self) -> Any:
raise AssertionError("persist_workspace() should not be called")
async def hydrate_workspace(self, data: Any) -> None:
_ = data
raise AssertionError("hydrate_workspace() should not be called")
class _IdentityCapability(Capability):
type: str = "identity"
setting: str
@@ -315,8 +267,8 @@ class TestRunState:
trace_state = FalsyTraceState(trace_id="trace_falsy")
model = FakeModel()
model.set_next_output([get_final_output_message("done")])
model = ScriptedModel()
model.enqueue([get_final_output_message("done")])
result = await Runner.run(Agent(name="test", model=model), "input")
result._trace_state = trace_state
@@ -324,8 +276,8 @@ class TestRunState:
assert isinstance(restored, FalsyTraceState)
assert restored.trace_id == "trace_falsy"
streaming_model = FakeModel()
streaming_model.set_next_output([get_final_output_message("done")])
streaming_model = ScriptedModel()
streaming_model.enqueue([get_final_output_message("done")])
streaming_result = Runner.run_streamed(
Agent(name="streaming-test", model=streaming_model),
"input",
@@ -625,13 +577,21 @@ class TestRunState:
first_alpha_capability = _IdentityCapability(setting="alpha")
first_beta_capability = _IdentityCapability(setting="beta")
first_alpha_capability.bind(_IdentitySandboxSession("/workspace/first-alpha"))
first_beta_capability.bind(_IdentitySandboxSession("/workspace/first-beta"))
first_alpha_capability.bind(
scripted_sandbox_session(manifest=Manifest(root="/workspace/first-alpha"))
)
first_beta_capability.bind(
scripted_sandbox_session(manifest=Manifest(root="/workspace/first-beta"))
)
second_alpha_capability = _IdentityCapability(setting="alpha")
second_beta_capability = _IdentityCapability(setting="beta")
second_alpha_capability.bind(_IdentitySandboxSession("/workspace/second-alpha"))
second_beta_capability.bind(_IdentitySandboxSession("/workspace/second-beta"))
second_alpha_capability.bind(
scripted_sandbox_session(manifest=Manifest(root="/workspace/second-alpha"))
)
second_beta_capability.bind(
scripted_sandbox_session(manifest=Manifest(root="/workspace/second-beta"))
)
first_alpha_signature = _capability_identity_signature(first_alpha_capability)
first_beta_signature = _capability_identity_signature(first_beta_capability)
@@ -707,8 +667,8 @@ class TestRunState:
def approval_tool() -> str:
return "approved"
first_model = FakeModel()
second_model = FakeModel()
first_model = ScriptedModel()
second_model = ScriptedModel()
first = Agent(name="duplicate", model=first_model)
second = Agent(
name="duplicate",
@@ -719,8 +679,8 @@ class TestRunState:
first.handoffs = [second]
second.handoffs = [first]
first_model.add_multiple_turn_outputs([[get_handoff_tool_call(second)]])
second_model.add_multiple_turn_outputs(
first_model.extend([[get_handoff_tool_call(second)]])
second_model.extend(
[[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")]]
)
@@ -1930,7 +1890,7 @@ class TestRunState:
probe_agent = Agent(
name="ApprovalProbeAgent",
model=FakeModel(initial_output=[get_text_message("done")]),
model=ScriptedModel(steps=[[get_text_message("done")]]),
)
await Runner.run(
probe_agent,
@@ -4452,109 +4412,52 @@ class TestDeserializeHelpers:
return True
return False
class ResumeAwareToolModel(Model):
def __init__(
self,
*,
tool_name: str,
tool_arguments: str,
final_text: str,
call_prefix: str,
preceding_tool_name: str | None = None,
) -> None:
self.tool_name = tool_name
self.tool_arguments = tool_arguments
self.final_text = final_text
self.call_prefix = call_prefix
self.preceding_tool_name = preceding_tool_name
self.call_count = 0
def _make_resume_aware_tool_model(
*,
tool_name: str,
tool_arguments: str,
final_text: str,
call_prefix: str,
preceding_tool_name: str | None = None,
) -> ScriptedModel:
tool_call_count = 0
async def get_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Any],
output_schema: Any,
handoffs: list[Any],
tracing: Any,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: Any | None,
) -> ModelResponse:
del (
system_instructions,
model_settings,
tools,
output_schema,
handoffs,
tracing,
previous_response_id,
conversation_id,
prompt,
)
if _has_function_call_output(input):
def _respond(call: ModelCall) -> ModelResponse:
nonlocal tool_call_count
if _has_function_call_output(call.input):
return ModelResponse(
output=[get_text_message(self.final_text)],
output=[get_text_message(final_text)],
usage=Usage(),
response_id=f"{self.call_prefix}-done",
response_id=f"{call_prefix}-done",
)
self.call_count += 1
tool_call_count += 1
output: list[TResponseOutputItem] = []
if self.preceding_tool_name is not None:
if preceding_tool_name is not None:
output.append(
ResponseFunctionToolCall(
type="function_call",
name=self.preceding_tool_name,
call_id=f"{self.call_prefix}-preceding-{self.call_count}",
name=preceding_tool_name,
call_id=f"{call_prefix}-preceding-{tool_call_count}",
arguments="{}",
)
)
output.append(
ResponseFunctionToolCall(
type="function_call",
name=self.tool_name,
call_id=f"{self.call_prefix}-{id(self)}-{self.call_count}",
arguments=self.tool_arguments,
name=tool_name,
call_id=f"{call_prefix}-{id(model)}-{tool_call_count}",
arguments=tool_arguments,
)
)
return ModelResponse(
output=output,
usage=Usage(),
response_id=f"{self.call_prefix}-call-{self.call_count}",
response_id=f"{call_prefix}-call-{tool_call_count}",
)
async def stream_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Any],
output_schema: Any,
handoffs: list[Any],
tracing: Any,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: Any | None,
) -> AsyncIterator[TResponseStreamEvent]:
del (
system_instructions,
input,
model_settings,
tools,
output_schema,
handoffs,
tracing,
previous_response_id,
conversation_id,
prompt,
)
if False:
yield cast(TResponseStreamEvent, {})
raise RuntimeError("Streaming is not supported in this test.")
model = ScriptedModel(ModelStep.respond(_respond) for _ in range(3))
return model
tool_calls: list[str] = []
@@ -4563,7 +4466,7 @@ class TestDeserializeHelpers:
tool_calls.append(text)
return f"approved:{text}"
inner_model = ResumeAwareToolModel(
inner_model = _make_resume_aware_tool_model(
tool_name="inner_sensitive_tool",
tool_arguments=json.dumps({"text": "hello"}),
final_text="inner-complete",
@@ -4575,7 +4478,7 @@ class TestDeserializeHelpers:
tool_name="inner_agent_tool",
tool_description="Inner agent tool",
)
outer_model = ResumeAwareToolModel(
outer_model = _make_resume_aware_tool_model(
tool_name="inner_agent_tool",
tool_arguments=json.dumps({"input": "hello"}),
final_text="outer-complete",
@@ -4628,6 +4531,8 @@ class TestDeserializeHelpers:
assert resumed_result_two.final_output == "outer-complete"
assert resumed_result_two.interruptions == []
assert tool_calls == (["hello", "hello"] if approve_nested_tool else [])
inner_model.assert_complete()
outer_model.assert_complete()
async def test_json_decode_error_handling(self):
"""Test that invalid JSON raises appropriate error."""
@@ -4667,18 +4572,18 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resume_from_run_state(self):
"""Test resuming a run from a RunState."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
# First run - create a state
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input")
# Create RunState from result
state = result1.to_state()
# Resume from state
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = await Runner.run(agent, state)
assert result2.final_output == "Second response"
@@ -4686,16 +4591,16 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resume_from_run_state_does_not_mutate_source_result(self):
"""Resuming from a state must not append to the raw_responses already returned."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input")
assert len(result1.raw_responses) == 1
state = result1.to_state()
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = await Runner.run(agent, state)
# The second run accumulates on top of the first, but the RunResult that was
@@ -4707,15 +4612,15 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resume_does_not_append_to_the_state_it_resumed_from(self):
"""A resumed run must not accumulate its responses into the caller's checkpoint."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input")
state = result1.to_state()
serialized_before = state.to_json()["model_responses"]
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = await Runner.run(agent, state)
assert len(result2.raw_responses) == 2
@@ -4725,22 +4630,22 @@ class TestRunStateResumption:
assert state.to_json()["model_responses"] == serialized_before
# Re-running the same checkpoint therefore replays only its own history.
model.set_next_output([get_text_message("Third response")])
model.enqueue([get_text_message("Third response")])
result3 = await Runner.run(agent, state)
assert len(result3.raw_responses) == 2
@pytest.mark.asyncio
async def test_streamed_resume_does_not_append_to_the_state_it_resumed_from(self):
"""A streamed resume must not accumulate its items into the caller's checkpoint."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input")
state = result1.to_state()
serialized_before = state.to_json()["session_items"]
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = Runner.run_streamed(agent, state)
async for _ in result2.stream_events():
pass
@@ -4750,7 +4655,7 @@ class TestRunStateResumption:
assert state.to_json()["session_items"] == serialized_before
# Without this, the abandoned attempt's message leaks into the replayed history.
model.set_next_output([get_text_message("Third response")])
model.enqueue([get_text_message("Third response")])
result3 = Runner.run_streamed(agent, state)
async for _ in result3.stream_events():
pass
@@ -4760,10 +4665,10 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resumed_max_turns_handler_does_not_append_to_state_items(self):
"""A resumed run that trips max turns must not append to the state's items."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input", max_turns=1)
state = result1.to_state()
serialized_before = state.to_json()["generated_items"]
@@ -4780,15 +4685,15 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_fresh_runs_still_report_their_own_history(self):
"""Boundary: a run that starts without a state is unaffected by the copies."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input")
assert len(result1.raw_responses) == 1
assert len(result1.new_items) == 1
model.set_next_output([get_text_message("Streamed response")])
model.enqueue([get_text_message("Streamed response")])
result2 = Runner.run_streamed(agent, "Second input")
async for _ in result2.stream_events():
pass
@@ -4798,12 +4703,12 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resume_from_run_state_with_context(self):
"""Test resuming a run from a RunState with context override."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
# First run with context
context1 = {"key": "value1"}
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input", context=context1)
# Create RunState from result
@@ -4811,7 +4716,7 @@ class TestRunStateResumption:
# Resume from state with different context (should use new context)
context2 = {"key": "value2"}
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = await Runner.run(agent, state, context=context2)
# New context should be used.
@@ -4823,18 +4728,18 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resume_from_run_state_with_conversation_id(self):
"""Test resuming a run from a RunState with conversation_id."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
# First run
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input", conversation_id="conv123")
# Create RunState from result
state = result1.to_state()
# Resume from state with conversation_id
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = await Runner.run(agent, state, conversation_id="conv123")
assert result2.final_output == "Second response"
@@ -4842,18 +4747,18 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resume_from_run_state_with_previous_response_id(self):
"""Test resuming a run from a RunState with previous_response_id."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
# First run
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input", previous_response_id="resp123")
# Create RunState from result
state = result1.to_state()
# Resume from state with previous_response_id
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = await Runner.run(agent, state, previous_response_id="resp123")
assert result2.final_output == "Second response"
@@ -4861,7 +4766,7 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resume_from_run_state_with_interruption(self):
"""Test resuming a run from a RunState with an interruption."""
model = FakeModel()
model = ScriptedModel()
async def tool_func() -> str:
return "tool_result"
@@ -4875,7 +4780,8 @@ class TestRunStateResumption:
)
# First run - create an interruption
model.set_next_output([get_function_tool_call("test_tool", "{}")])
model.enqueue([get_function_tool_call("test_tool", "{}")])
model.enqueue([])
result1 = await Runner.run(agent, "First input")
# Create RunState from result
@@ -4886,7 +4792,7 @@ class TestRunStateResumption:
state.approve(state.get_interruptions()[0])
# Resume from state - should execute approved tools
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = await Runner.run(agent, state)
assert result2.final_output == "Second response"
@@ -4894,18 +4800,18 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_resume_from_run_state_streamed(self):
"""Test resuming a run from a RunState using run_streamed."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
# First run
model.set_next_output([get_text_message("First response")])
model.enqueue([get_text_message("First response")])
result1 = await Runner.run(agent, "First input")
# Create RunState from result
state = result1.to_state()
# Resume from state using run_streamed
model.set_next_output([get_text_message("Second response")])
model.enqueue([get_text_message("Second response")])
result2 = Runner.run_streamed(agent, state)
events = []
@@ -4920,8 +4826,8 @@ class TestRunStateResumption:
async def test_resume_from_run_state_streamed_uses_context_from_state(self):
"""Test that streaming with RunState uses context from state."""
model = FakeModel()
model.set_next_output([get_text_message("done")])
model = ScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="TestAgent", model=model)
# Create a RunState with context
@@ -4940,8 +4846,8 @@ class TestRunStateResumption:
async def test_resume_from_run_state_streamed_with_context_override(self):
"""Test that streaming uses provided context override when resuming."""
model = FakeModel()
model.set_next_output([get_text_message("done")])
model = ScriptedModel()
model.enqueue([get_text_message("done")])
agent = Agent(name="TestAgent", model=model)
# Create a RunState with context
@@ -4959,7 +4865,7 @@ class TestRunStateResumption:
@pytest.mark.asyncio
async def test_run_result_streaming_to_state_with_interruptions(self):
"""Test RunResultStreaming.to_state() sets _current_step with interruptions."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="TestAgent", model=model)
async def test_tool() -> str:
@@ -4969,7 +4875,7 @@ class TestRunStateResumption:
agent.tools = [tool]
# Create a run that will have interruptions
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("test_tool", json.dumps({}))],
[get_text_message("done")],
@@ -9549,7 +9455,7 @@ async def _interrupted_approval_state_with_tool_input(
return text
model, agent = make_model_and_agent(tools=[needs_ok], name="agent")
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("needs_ok", json.dumps({"text": "one"}), call_id="1")],
[get_final_output_message("done")],
@@ -9647,17 +9553,17 @@ async def test_resume_nested_agent_as_tool_with_context_override() -> None:
output_tokens=3,
total_tokens=20,
)
nested_model = FakeModel()
nested_model.set_hardcoded_usage(nested_turn_usage)
nested_model = ScriptedModel()
nested_model.set_default_usage(nested_turn_usage)
nested_agent = Agent(name="nested", tools=[needs_ok], model=nested_model)
nested_model.add_multiple_turn_outputs(
nested_model.extend(
[
[get_function_tool_call("needs_ok", json.dumps({"text": "one"}), call_id="inner-1")],
[get_final_output_message("nested-done")],
]
)
outer_model = FakeModel()
outer_model = ScriptedModel()
outer = Agent(
name="outer",
tools=[
@@ -9669,7 +9575,7 @@ async def test_resume_nested_agent_as_tool_with_context_override() -> None:
],
model=outer_model,
)
outer_model.add_multiple_turn_outputs(
outer_model.extend(
[
[
get_function_tool_call(
+78 -76
View File
@@ -19,10 +19,11 @@ from agents.run_context import RunContextWrapper
from agents.run_internal.oai_conversation import OpenAIServerConversationTracker
from agents.run_internal.run_steps import NextStepInterruption, NextStepRunAgain
from agents.run_state import CURRENT_SCHEMA_VERSION, RunState
from agents.testing import ScriptedModel
from agents.tool import Tool
from agents.usage import Usage
from .fake_model import FakeModel
from .model_test_helpers import get_exact_output_stream_step
from .test_computer_tool_lifecycle import FakeComputer
from .test_responses import get_function_tool_call, get_text_message
from .utils.simple_session import SimpleListSession
@@ -53,7 +54,7 @@ async def _make_after_turn_state(
*,
session: SimpleListSession | None = None,
auto_previous_response_id: bool = False,
) -> tuple[FakeModel, Agent[Any], RunState[Any], list[str]]:
) -> tuple[ScriptedModel, Agent[Any], RunState[Any], list[str]]:
calls: list[str] = []
@function_tool(name_override="record_destination")
@@ -61,8 +62,8 @@ async def _make_after_turn_state(
calls.append(destination)
return f"recorded:{destination}"
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[
get_function_tool_call(
"record_destination",
@@ -136,13 +137,13 @@ async def test_after_turn_resume_admits_input_after_tool_output_exactly_once() -
session = SimpleListSession()
model, agent, state, calls = await _make_after_turn_state(session=session)
state.add_input("Change the destination to Tokyo")
model.set_next_output([get_text_message("Updated")])
model.enqueue([get_text_message("Updated")])
result = await Runner.run(agent, state, session=session)
assert result.final_output == "Updated"
assert calls == ["Paris"]
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_item_type(item) for item in model_input] == [
"user",
"function_call",
@@ -171,7 +172,7 @@ async def test_after_turn_resume_admits_input_after_tool_output_exactly_once() -
async def test_streamed_resume_matches_pending_input_ordering() -> None:
model, agent, state, calls = await _make_after_turn_state()
state.add_input("Change the destination to Tokyo")
model.set_next_output([get_text_message("Updated")])
model.enqueue([get_text_message("Updated")])
result = Runner.run_streamed(agent, state)
async for _ in result.stream_events():
@@ -179,7 +180,7 @@ async def test_streamed_resume_matches_pending_input_ordering() -> None:
assert result.final_output == "Updated"
assert calls == ["Paris"]
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_item_type(item) for item in model_input] == [
"user",
"function_call",
@@ -199,14 +200,14 @@ async def test_streamed_resume_matches_pending_input_ordering() -> None:
async def test_server_managed_resume_sends_pending_input_as_unsent_delta_once() -> None:
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Change the destination to Tokyo")
model.set_next_output([get_text_message("Updated")])
model.enqueue([get_text_message("Updated")])
result = await Runner.run(agent, state)
assert result.final_output == "Updated"
assert calls == ["Paris"]
assert model.last_turn_args["previous_response_id"] == "resp-789"
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
assert model.calls[-1].previous_response_id == "resp-789"
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_item_type(item) for item in model_input] == ["function_call_output", "user"]
assert [_message_text(item) for item in model_input].count(
"Change the destination to Tokyo"
@@ -243,7 +244,7 @@ async def test_server_managed_resume_sends_identical_late_input_in_later_occurre
) -> None:
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Repeat")
model.set_next_output(
model.enqueue(
[
get_function_tool_call(
"record_destination",
@@ -261,7 +262,7 @@ async def test_server_managed_resume_sends_identical_late_input_in_later_occurre
state = await RunState.from_json(agent, first_resume.to_state().to_json())
admitted_before = next(item for item in state._generated_items if isinstance(item, InputItem))
state.add_input("Repeat")
model.set_next_output([get_text_message("Done")])
model.enqueue([get_text_message("Done")])
if streamed_second_resume:
streamed_result = Runner.run_streamed(agent, state)
@@ -274,7 +275,7 @@ async def test_server_managed_resume_sends_identical_late_input_in_later_occurre
assert final_output == "Done"
assert calls == ["Paris", "Rome"]
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in model_input].count("Repeat") == 1
admitted_after = [item for item in state._generated_items if isinstance(item, InputItem)]
assert [item.input_id for item in admitted_after].count(admitted_before.input_id) == 1
@@ -290,8 +291,8 @@ async def test_unresolved_approval_keeps_pending_input_until_tool_finishes() ->
calls.append(value)
return f"approved:{value}"
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[get_function_tool_call("protected_tool", '{"value":"one"}', call_id="call-protected")]
)
agent = Agent(name="assistant", model=model, tools=[protected_tool])
@@ -305,12 +306,12 @@ async def test_unresolved_approval_keeps_pending_input_until_tool_finishes() ->
assert _message_text(state.pending_input[0]) == "Late input"
state.approve(state.get_interruptions()[0])
model.set_next_output([get_text_message("Done")])
model.enqueue([get_text_message("Done")])
resumed = await Runner.run(agent, state)
assert resumed.final_output == "Done"
assert calls == ["one"]
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_item_type(item) for item in model_input][-2:] == ["function_call_output", "user"]
assert _message_text(model_input[-1]) == "Late input"
@@ -324,8 +325,8 @@ async def test_streamed_after_turn_cancel_keeps_pending_input_for_next_resume()
calls.append(value)
return f"approved:{value}"
model = FakeModel()
model.set_next_output(
model = ScriptedModel()
model.enqueue(
[get_function_tool_call("protected_tool", '{"value":"one"}', call_id="call-protected")]
)
agent = Agent(name="assistant", model=model, tools=[protected_tool])
@@ -342,10 +343,10 @@ async def test_streamed_after_turn_cancel_keeps_pending_input_for_next_resume()
assert calls == ["one"]
assert _message_text(state.pending_input[0]) == "Late input"
model.set_next_output([get_text_message("Done")])
model.enqueue([get_text_message("Done")])
result = await Runner.run(agent, state)
assert result.final_output == "Done"
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in model_input].count("Late input") == 1
@@ -367,13 +368,15 @@ async def test_interruption_without_guaranteed_next_model_rejects_input(
def protected_tool(value: str) -> str:
return value
model = FakeModel(
initial_output=[
get_function_tool_call(
"protected_tool",
'{"value":"one"}',
call_id="call-protected-terminal",
)
model = ScriptedModel(
steps=[
[
get_function_tool_call(
"protected_tool",
'{"value":"one"}',
call_id="call-protected-terminal",
)
]
]
)
agent = Agent(
@@ -412,13 +415,13 @@ async def test_pending_input_guardrail_trip_keeps_input_recoverable() -> None:
agent.input_guardrails = [InputGuardrail(guardrail_function=trip_pending_input)]
state.add_input("Unsafe late input")
model.set_next_output([get_text_message("Must not run")])
queued_outputs = len(model.turn_outputs)
model.enqueue([get_text_message("Must not run")])
queued_outputs = model.remaining_steps
with pytest.raises(InputGuardrailTripwireTriggered):
await Runner.run(agent, state)
assert len(model.turn_outputs) == queued_outputs
assert model.remaining_steps == queued_outputs
assert [[_message_text(item) for item in batch] for batch in guarded_inputs] == [
["Unsafe late input"]
]
@@ -453,7 +456,7 @@ async def test_pending_input_runs_agent_and_run_config_guardrails_on_only_pendin
input_guardrails=[InputGuardrail(guardrail_function=inspect_config_input)]
)
state.add_input("Guard only this")
model.set_next_output([get_text_message("Done")])
model.enqueue([get_text_message("Done")])
result = await Runner.run(agent, state, run_config=run_config)
@@ -494,7 +497,7 @@ async def test_guardrail_retry_persists_successful_turn_with_session(
await Runner.run(agent, state, session=session)
should_trip = False
model.set_next_output([get_text_message("Recovered")])
model.enqueue([get_text_message("Recovered")])
if streamed_retry:
streamed_result = Runner.run_streamed(agent, state, session=session)
async for _event in streamed_result.stream_events():
@@ -519,7 +522,7 @@ async def test_guardrail_retry_persists_successful_turn_with_session(
async def test_failed_model_request_does_not_duplicate_admitted_input_on_resume() -> None:
model, agent, state, _calls = await _make_after_turn_state()
state.add_input("Late input")
model.set_next_output(RuntimeError("model failed"))
model.enqueue(RuntimeError("model failed"))
with pytest.raises(RuntimeError, match="model failed"):
await Runner.run(agent, state)
@@ -534,10 +537,10 @@ async def test_failed_model_request_does_not_duplicate_admitted_input_on_resume(
next(item.input_id for item in state._generated_items if isinstance(item, InputItem))
== admitted_input_id
)
model.set_next_output([get_text_message("Recovered")])
model.enqueue([get_text_message("Recovered")])
result = await Runner.run(agent, state)
assert result.final_output == "Recovered"
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in model_input].count("Late input") == 1
@@ -546,7 +549,7 @@ async def test_failed_model_request_with_session_persists_admitted_input_once()
session = SimpleListSession()
model, agent, state, _calls = await _make_after_turn_state(session=session)
state.add_input("Late input")
model.set_next_output(RuntimeError("model failed"))
model.enqueue(RuntimeError("model failed"))
with pytest.raises(RuntimeError, match="model failed"):
await Runner.run(agent, state, session=session)
@@ -555,10 +558,10 @@ async def test_failed_model_request_with_session_persists_admitted_input_once()
assert [_message_text(item) for item in await session.get_items()].count("Late input") == 1
state = await RunState.from_json(agent, state.to_json())
model.set_next_output([get_text_message("Recovered")])
model.enqueue([get_text_message("Recovered")])
result = await Runner.run(agent, state, session=session)
assert result.final_output == "Recovered"
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in model_input].count("Late input") == 1
assert [_message_text(item) for item in await session.get_items()].count("Late input") == 1
@@ -567,17 +570,17 @@ async def test_failed_model_request_with_session_persists_admitted_input_once()
async def test_failed_server_managed_request_keeps_pending_input_for_retry() -> None:
model, agent, state, _calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Late input")
model.set_next_output(RuntimeError("model failed"))
model.enqueue(RuntimeError("model failed"))
with pytest.raises(RuntimeError, match="model failed"):
await Runner.run(agent, state)
assert _message_text(state.pending_input[0]) == "Late input"
state = await RunState.from_json(agent, state.to_json())
model.set_next_output([get_text_message("Recovered")])
model.enqueue([get_text_message("Recovered")])
result = await Runner.run(agent, state)
assert result.final_output == "Recovered"
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in model_input].count("Late input") == 1
assert state.pending_input == []
@@ -586,7 +589,7 @@ async def test_failed_server_managed_request_keeps_pending_input_for_retry() ->
async def test_server_filter_omission_remains_pending_for_later_nonstream_turn() -> None:
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Late input")
model.add_multiple_turn_outputs(
model.extend(
[
[
get_function_tool_call(
@@ -616,7 +619,7 @@ async def test_server_filter_omission_remains_pending_for_later_nonstream_turn()
assert result.final_output == "Done"
assert calls == ["Paris", "Rome"]
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in model_input].count("Late input") == 1
assert state.pending_input == []
@@ -625,7 +628,7 @@ async def test_server_filter_omission_remains_pending_for_later_nonstream_turn()
async def test_server_filter_omission_survives_streamed_state_round_trip() -> None:
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Late input")
model.set_next_output(
model.enqueue(
[
get_function_tool_call(
"record_destination",
@@ -651,11 +654,11 @@ async def test_server_filter_omission_survives_streamed_state_round_trip() -> No
assert [_message_text(item) for item in state.pending_input] == ["Late input"]
assert not any(isinstance(item, InputItem) for item in state._generated_items)
model.set_next_output([get_text_message("Done")])
model.enqueue([get_text_message("Done")])
result = await Runner.run(agent, state)
assert result.final_output == "Done"
assert calls == ["Paris", "Rome"]
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in model_input].count("Late input") == 1
@@ -664,7 +667,7 @@ async def test_server_filter_omission_survives_streamed_state_round_trip() -> No
async def test_server_filter_reconstructed_pending_rewrite_is_rejected(streamed: bool) -> None:
model, agent, state, _calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Late input")
model.set_next_output([get_text_message("Done")])
model.enqueue([get_text_message("Done")])
def reconstruct_pending(data: CallModelData[Any]) -> ModelInputData:
rewritten = [
@@ -678,7 +681,7 @@ async def test_server_filter_reconstructed_pending_rewrite_is_rejected(streamed:
instructions=data.model_data.instructions,
)
queued_outputs = len(model.turn_outputs)
queued_outputs = model.remaining_steps
run_config = RunConfig(call_model_input_filter=reconstruct_pending)
if streamed:
failed = Runner.run_streamed(agent, state, run_config=run_config)
@@ -689,7 +692,7 @@ async def test_server_filter_reconstructed_pending_rewrite_is_rejected(streamed:
with pytest.raises(UserError, match="cannot safely associate"):
await Runner.run(agent, state, run_config=run_config)
assert len(model.turn_outputs) == queued_outputs
assert model.remaining_steps == queued_outputs
assert [_message_text(item) for item in state.pending_input] == ["Late input"]
@@ -697,7 +700,7 @@ async def test_server_filter_reconstructed_pending_rewrite_is_rejected(streamed:
async def test_server_filter_in_place_pending_rewrite_preserves_occurrence() -> None:
model, agent, state, _calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Late input")
model.set_next_output([get_text_message("Done")])
model.enqueue([get_text_message("Done")])
def rewrite_pending_in_place(data: CallModelData[Any]) -> ModelInputData:
for item in data.model_data.input:
@@ -712,7 +715,7 @@ async def test_server_filter_in_place_pending_rewrite_preserves_occurrence() ->
)
assert result.final_output == "Done"
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in model_input].count("Filtered late input") == 1
assert state.pending_input == []
@@ -747,7 +750,7 @@ async def test_server_response_acceptance_commits_before_hook_failure(
agent_hooks = CountAgentResponseHook()
agent.hooks = agent_hooks
state.add_input("Late input")
model.set_next_output([get_text_message("Accepted")])
model.enqueue([get_text_message("Accepted")])
if streamed_failure:
failed = Runner.run_streamed(agent, state, hooks=FailAfterResponse())
@@ -758,7 +761,7 @@ async def test_server_response_acceptance_commits_before_hook_failure(
with pytest.raises(RuntimeError, match="after response"):
await Runner.run(agent, state, hooks=FailAfterResponse())
accepted_model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
accepted_model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in accepted_model_input].count("Late input") == 1
assert state.pending_input == []
assert isinstance(state._current_step, NextStepInterruption)
@@ -766,12 +769,12 @@ async def test_server_response_acceptance_commits_before_hook_failure(
assert state._current_step.llm_end_hooks_started
assert agent_hooks.call_count == 1
state = await RunState.from_json(agent, state.to_json())
queued_outputs = len(model.turn_outputs)
queued_outputs = model.remaining_steps
recovered = await Runner.run(agent, state)
assert recovered.final_output == "Accepted"
assert agent_hooks.call_count == 1
assert len(model.turn_outputs) == queued_outputs
assert model.remaining_steps == queued_outputs
@pytest.mark.asyncio
@@ -781,7 +784,7 @@ async def test_server_acceptance_commits_before_invocation_validation_failure(
) -> None:
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Late input")
model.set_next_output(
model.enqueue(
[
get_function_tool_call(
"record_destination",
@@ -800,7 +803,7 @@ async def test_server_acceptance_commits_before_invocation_validation_failure(
with pytest.raises(ModelBehaviorError, match="completed tool call ID"):
await Runner.run(agent, state)
accepted_model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
accepted_model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in accepted_model_input].count("Late input") == 1
assert state.pending_input == []
assert isinstance(state._current_step, NextStepInterruption)
@@ -809,10 +812,10 @@ async def test_server_acceptance_commits_before_invocation_validation_failure(
assert calls == ["Paris"]
state = await RunState.from_json(agent, state.to_json())
queued_outputs = len(model.turn_outputs)
queued_outputs = model.remaining_steps
with pytest.raises(UserError, match="accepted model response could not be processed"):
await Runner.run(agent, state)
assert len(model.turn_outputs) == queued_outputs
assert model.remaining_steps == queued_outputs
assert calls == ["Paris"]
@@ -845,18 +848,17 @@ async def test_server_accepted_computer_start_hook_failure_is_not_replayed(
model, agent, state, _calls = await _make_after_turn_state(auto_previous_response_id=True)
agent.tools = [ComputerTool(computer=RecordingComputer())]
state.add_input("Late input")
model.set_next_output(
[
ResponseComputerToolCall(
id="computer-item",
type="computer_call",
action=ActionScreenshot(type="screenshot"),
call_id="computer-call",
pending_safety_checks=[],
status="completed",
)
]
)
output = [
ResponseComputerToolCall(
id="computer-item",
type="computer_call",
action=ActionScreenshot(type="screenshot"),
call_id="computer-call",
pending_safety_checks=[],
status="completed",
)
]
model.enqueue(get_exact_output_stream_step(output) if streamed_failure else output)
hooks = FailComputerStart()
if streamed_failure:
@@ -909,7 +911,7 @@ async def test_server_accepted_tool_side_effect_failure_is_safe(
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
state.add_input("Late input")
model.add_multiple_turn_outputs(
model.extend(
[
[
get_function_tool_call(
@@ -947,13 +949,13 @@ async def test_server_accepted_tool_side_effect_failure_is_safe(
recovered = await Runner.run(agent, state)
assert recovered.final_output == "Recovered"
assert calls == ["Paris", "Rome"]
retry_model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
retry_model_input = cast(list[TResponseInputItem], model.calls[-1].input)
assert [_message_text(item) for item in retry_model_input].count("Late input") == 0
@pytest.mark.asyncio
async def test_terminal_state_rejects_pending_input_without_mutation() -> None:
model = FakeModel(initial_output=[get_text_message("Done")])
model = ScriptedModel(steps=[[get_text_message("Done")]])
agent = Agent(name="assistant", model=model)
result = await Runner.run(agent, "Initial request")
state = result.to_state()
+3 -3
View File
@@ -15,6 +15,7 @@ from agents.run_internal.run_steps import (
SingleStepResult,
)
from agents.run_state import RunState
from agents.testing import ScriptedModel
from agents.tool_guardrails import (
AllowBehavior,
ToolGuardrailFunctionOutput,
@@ -24,12 +25,11 @@ from agents.tool_guardrails import (
ToolOutputGuardrailResult,
)
from agents.usage import Usage
from tests.fake_model import FakeModel
@pytest.mark.asyncio
async def test_runner_resume_preserves_guardrail_results(monkeypatch: pytest.MonkeyPatch) -> None:
agent = Agent(name="agent", model=FakeModel())
agent = Agent(name="agent", model=ScriptedModel())
context_wrapper: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
input_guardrail: InputGuardrail[Any] = InputGuardrail(
@@ -163,7 +163,7 @@ async def test_runner_resume_preserves_guardrail_results_on_reinterruption(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A resumed run that interrupts again must keep the tool guardrail results it carried in."""
agent = Agent(name="agent", model=FakeModel())
agent = Agent(name="agent", model=ScriptedModel())
context_wrapper: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
tool_input_guardrail: ToolInputGuardrail[Any] = ToolInputGuardrail(
+15 -14
View File
@@ -8,9 +8,9 @@ from openai.types.responses.response_usage import InputTokensDetails, OutputToke
from agents import Agent, Runner, Tool, Usage
from agents.items import ToolApprovalItem
from agents.result import RunResult, RunResultStreaming
from agents.testing import ScriptedModel
from agents.usage import serialize_usage
from .fake_model import FakeModel
from .test_responses import get_function_tool, get_function_tool_call, get_text_message
from .testing_processor import SPAN_PROCESSOR_TESTING, fetch_normalized_spans
from .utils.simple_session import SimpleListSession
@@ -97,12 +97,12 @@ async def _run(
@pytest.mark.parametrize("streamed", [False, True])
async def test_fake_model_records_every_model_visible_request_field(streamed: bool) -> None:
model = FakeModel()
model.set_next_output([get_text_message("READY")])
async def test_scripted_model_records_every_model_visible_request_field(streamed: bool) -> None:
model = ScriptedModel()
model.enqueue([get_text_message("READY")])
await _run(Agent(name="request-contract-agent", model=model), streamed=streamed)
assert set(model.last_turn_args) == {
assert set(model.calls[-1].__dataclass_fields__) == {
"system_instructions",
"input",
"model_settings",
@@ -113,6 +113,7 @@ async def test_fake_model_records_every_model_visible_request_field(streamed: bo
"previous_response_id",
"conversation_id",
"prompt",
"streamed",
}
@@ -121,11 +122,11 @@ async def test_streamed_and_nonstreamed_runs_have_matching_semantics(scenario: s
projections: list[dict[str, Any]] = []
for streamed in (False, True):
SPAN_PROCESSOR_TESTING.clear()
model = FakeModel(tracing_enabled=True)
model.set_hardcoded_usage(_detailed_usage())
model = ScriptedModel(emit_traces=True)
model.set_default_usage(_detailed_usage())
tools: list[Tool] = []
if scenario == "function-tool":
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("release_check", "{}", call_id="call-release")],
[get_text_message("READY")],
@@ -133,7 +134,7 @@ async def test_streamed_and_nonstreamed_runs_have_matching_semantics(scenario: s
)
tools = [get_function_tool("release_check", "checked")]
else:
model.set_next_output([get_text_message("READY")])
model.enqueue([get_text_message("READY")])
agent = Agent(name="symmetry-agent", model=model, tools=tools)
session = SimpleListSession(session_id=f"{scenario}-{streamed}")
result = await _run(agent, streamed=streamed, session=session)
@@ -155,8 +156,8 @@ async def test_streamed_and_nonstreamed_runs_have_matching_semantics(scenario: s
async def test_streamed_and_nonstreamed_runs_raise_the_same_exception_class() -> None:
exception_classes: list[type[BaseException]] = []
for streamed in (False, True):
model = FakeModel()
model.set_next_output(RuntimeError("release contract failure"))
model = ScriptedModel()
model.enqueue(RuntimeError("release contract failure"))
agent = Agent(name="symmetry-agent", model=model)
with pytest.raises(RuntimeError) as exc_info:
@@ -170,9 +171,9 @@ async def test_approval_resume_cross_modes_have_matching_semantics() -> None:
projections: list[dict[str, Any]] = []
for start_streamed, resume_streamed in ((True, False), (False, True)):
SPAN_PROCESSOR_TESTING.clear()
model = FakeModel(tracing_enabled=True)
model.set_hardcoded_usage(_detailed_usage())
model.add_multiple_turn_outputs(
model = ScriptedModel(emit_traces=True)
model.set_default_usage(_detailed_usage())
model.extend(
[
[get_function_tool_call("release_check", "{}", call_id="call-release")],
[get_text_message("READY")],
File diff suppressed because it is too large Load Diff
+559
View File
@@ -0,0 +1,559 @@
from __future__ import annotations
import io
from pathlib import Path
from types import MappingProxyType
from typing import Any, cast
import pytest
from agents import RunConfig, Runner
from agents.sandbox import ExecResult, Manifest, SandboxAgent
from agents.sandbox.capabilities import Shell
from agents.sandbox.files import FileEntry
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
from agents.sandbox.session.pty_types import PtyExecUpdate
from agents.testing import (
InvalidSandboxStep,
SandboxCall,
SandboxCallMatcherError,
ScriptedModel,
UnconsumedSandboxSteps,
UnexpectedSandboxCall,
assistant_message,
function_call,
scripted_sandbox_session,
)
class _CallableBytesIO(io.BytesIO):
def __call__(self) -> None:
pass
class _CallableBufferedReader(io.BufferedReader):
def __call__(self) -> None:
pass
def test_scripted_sandbox_exposes_only_configured_scriptable_methods() -> None:
session = scripted_sandbox_session(
[{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)}]
)
assert isinstance(session, BaseSandboxSession)
assert hasattr(session, "exec")
assert not hasattr(session, "read")
assert not hasattr(session, "apply_patch")
assert not hasattr(session, "pty_exec_start")
assert "exec" in dir(session)
assert "read" not in dir(session)
assert "apply_patch" not in dir(session)
assert "pty_exec_start" not in dir(session)
@pytest.mark.asyncio
async def test_scripted_sandbox_cleanup_does_not_advertise_pty_termination() -> None:
session = scripted_sandbox_session()
assert not hasattr(session, "pty_terminate_all")
assert "pty_terminate_all" not in dir(session)
await session.aclose()
async with scripted_sandbox_session() as context_session:
assert await context_session.running() is True
assert await context_session.running() is False
def test_scripted_sandbox_snapshots_manifest_and_derives_pty_support() -> None:
manifest = Manifest(root="/configured")
session = scripted_sandbox_session(
[{"method": "pty_exec_start", "result": None}],
manifest=manifest,
)
manifest.root = "/mutated"
assert session.state.manifest.root == "/configured"
assert session.supports_pty() is True
pty_session = scripted_sandbox_session(
[
{"method": "pty_exec_start", "result": None},
{"method": "pty_write_stdin", "result": None},
]
)
assert pty_session.supports_pty() is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
("configured_method", "missing_method"),
[
("pty_exec_start", "pty_write_stdin"),
("pty_write_stdin", "pty_exec_start"),
],
)
async def test_scripted_sandbox_exposes_pty_methods_as_one_capability(
configured_method: str,
missing_method: str,
) -> None:
session = scripted_sandbox_session([{"method": configured_method, "result": None}])
assert session.supports_pty() is True
assert hasattr(session, "pty_exec_start")
assert hasattr(session, "pty_write_stdin")
assert "pty_exec_start" in dir(session)
assert "pty_write_stdin" in dir(session)
with pytest.raises(UnexpectedSandboxCall) as exc_info:
if missing_method == "pty_exec_start":
await session.pty_exec_start("pwd")
else:
await session.pty_write_stdin(session_id=1, chars="")
assert exc_info.value.actual_method == missing_method
assert exc_info.value.expected_method == configured_method
@pytest.mark.asyncio
async def test_scripted_sandbox_supports_capability_method_inventory() -> None:
read_result = io.BytesIO(b"contents")
list_result: list[FileEntry] = []
pty_start_result = PtyExecUpdate(
process_id=123,
output=b"started",
exit_code=None,
original_token_count=None,
)
pty_write_result = PtyExecUpdate(
process_id=None,
output=b"done",
exit_code=0,
original_token_count=None,
)
session = scripted_sandbox_session(
[
{"method": "read", "result": read_result},
{"method": "write", "result": None},
{"method": "ls", "result": list_result},
{"method": "mkdir", "result": None},
{"method": "rm", "result": None},
{"method": "apply_patch", "result": "Done!"},
{"method": "pty_exec_start", "result": pty_start_result},
{"method": "pty_write_stdin", "result": pty_write_result},
]
)
stream = io.BytesIO(b"payload")
returned_read_result = cast(io.BytesIO, await session.read(Path("in.txt")))
assert returned_read_result is not read_result
assert returned_read_result.getvalue() == b"contents"
await session.write(Path("out.txt"), stream)
assert await session.ls(".") == []
await session.mkdir("new", parents=True)
await session.rm("old", recursive=True)
assert await session.apply_patch({"type": "delete_file", "path": "old.txt"}) == "Done!"
assert (await session.pty_exec_start("sh", tty=True)).process_id == 123
assert (await session.pty_write_stdin(session_id=123, chars="exit")).exit_code == 0
assert [call.method for call in session.calls] == [
"read",
"write",
"ls",
"mkdir",
"rm",
"apply_patch",
"pty_exec_start",
"pty_write_stdin",
]
recorded_stream = cast(io.BytesIO, session.calls[1].args[1])
assert recorded_stream is not stream
assert recorded_stream.getvalue() == b"payload"
assert recorded_stream.tell() == 0
session.assert_complete()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("configured", "original_content"),
[(io.BytesIO(b"before"), b"before"), (io.StringIO("before"), "before")],
)
async def test_scripted_sandbox_snapshots_supported_stream_results(
configured: io.BytesIO | io.StringIO,
original_content: bytes | str,
) -> None:
configured.seek(2)
session = scripted_sandbox_session([{"method": "read", "result": configured}])
configured.seek(0)
configured.write(b"after" if isinstance(configured, io.BytesIO) else "after")
configured.close()
result = cast(io.BytesIO | io.StringIO, await session.read(Path("input.txt")))
assert result is not configured
assert result.tell() == 2
assert result.getvalue() == original_content
session.assert_complete()
@pytest.mark.asyncio
async def test_scripted_sandbox_snapshots_supported_stream_call_arguments() -> None:
session = scripted_sandbox_session([{"method": "write", "result": None}])
source = io.BytesIO(b"before")
source.seek(3)
await session.write(Path("output.bin"), source)
source.seek(0)
source.write(b"after")
source.close()
recorded = cast(io.BytesIO, session.calls[0].args[1])
assert recorded is not source
assert recorded.tell() == 3
assert recorded.getvalue() == b"before"
recorded.write(b"changed")
retained = cast(io.BytesIO, session.calls[0].args[1])
assert retained.tell() == 3
assert retained.getvalue() == b"before"
session.assert_complete()
@pytest.mark.asyncio
async def test_scripted_sandbox_snapshots_callable_supported_streams() -> None:
result_source = _CallableBytesIO(b"result")
call_source = _CallableBytesIO(b"call")
session = scripted_sandbox_session(
[
{"method": "read", "result": result_source},
{"method": "write", "result": None},
]
)
result = cast(io.BytesIO, await session.read(Path("input.bin")))
await session.write(Path("output.bin"), call_source)
assert result is not result_source
assert result.getvalue() == b"result"
recorded = cast(io.BytesIO, session.calls[1].args[1])
assert recorded is not call_source
assert recorded.getvalue() == b"call"
session.assert_complete()
def test_scripted_sandbox_rejects_unsupported_or_closed_stream_results() -> None:
with io.BufferedReader(io.BytesIO(b"payload")) as unsupported:
with pytest.raises(InvalidSandboxStep) as unsupported_info:
scripted_sandbox_session([{"method": "read", "result": unsupported}])
assert unsupported_info.value.reason == "invalid_outcome"
closed = io.BytesIO(b"payload")
closed.close()
with pytest.raises(InvalidSandboxStep) as closed_info:
scripted_sandbox_session([{"method": "read", "result": closed}])
assert closed_info.value.reason == "invalid_outcome"
@pytest.mark.asyncio
async def test_scripted_sandbox_rejects_unsupported_stream_call_before_commit() -> None:
session = scripted_sandbox_session([{"method": "write", "result": None}])
with io.BufferedReader(io.BytesIO(b"payload")) as unsupported:
with pytest.raises(TypeError, match="support only io.BytesIO and io.StringIO"):
await session.write(Path("output.bin"), unsupported)
assert session.calls == ()
assert session.remaining_steps == 1
def test_scripted_sandbox_rejects_callable_unsupported_stream_result() -> None:
with _CallableBufferedReader(io.BytesIO(b"payload")) as unsupported:
with pytest.raises(InvalidSandboxStep) as exc_info:
scripted_sandbox_session([{"method": "read", "result": unsupported}])
assert exc_info.value.reason == "invalid_outcome"
@pytest.mark.parametrize(
"stream_factory",
[
lambda: _CallableBytesIO(b"payload"),
lambda: _CallableBufferedReader(io.BytesIO(b"payload")),
],
)
@pytest.mark.parametrize(
("field", "reason"),
[("match", "invalid_matcher"), ("responder", "invalid_outcome")],
)
def test_scripted_sandbox_rejects_callable_stream_matchers_and_responders(
stream_factory: Any,
field: str,
reason: str,
) -> None:
stream = cast(io.IOBase, stream_factory())
step: dict[str, Any] = {"method": "exec", field: stream}
if field == "match":
step["result"] = ExecResult(stdout=b"", stderr=b"", exit_code=0)
try:
with pytest.raises(InvalidSandboxStep) as exc_info:
scripted_sandbox_session([step])
finally:
stream.close()
assert exc_info.value.reason == reason
assert exc_info.value.input_index == 0
@pytest.mark.asyncio
async def test_scripted_sandbox_rejects_callable_unsupported_stream_call_before_commit() -> None:
session = scripted_sandbox_session([{"method": "write", "result": None}])
with _CallableBufferedReader(io.BytesIO(b"payload")) as unsupported:
with pytest.raises(TypeError, match="support only io.BytesIO and io.StringIO"):
await session.write(Path("output.bin"), unsupported)
assert session.calls == ()
assert session.remaining_steps == 1
@pytest.mark.asyncio
async def test_scripted_sandbox_snapshots_static_results_when_queued() -> None:
configured = ExecResult(stdout=b"before", stderr=b"", exit_code=0)
session = scripted_sandbox_session([{"method": "exec", "result": configured}])
configured.stdout = b"after"
result = await session.exec("pwd")
assert result.stdout == b"before"
session.assert_complete()
@pytest.mark.asyncio
async def test_scripted_sandbox_records_detached_fifo_calls() -> None:
source_operations: list[dict[str, object]] = [{"type": "delete_file", "path": "before.txt"}]
session = scripted_sandbox_session(
[
{"method": "apply_patch", "result": "Done!"},
{"method": "exec", "result": ExecResult(stdout=b"ok", stderr=b"", exit_code=0)},
]
)
assert await session.apply_patch(cast(Any, source_operations)) == "Done!"
source_operations[0]["path"] = "after.txt"
result = await session.exec("pwd", shell=False)
assert result.stdout == b"ok"
assert session.remaining_steps == 0
session.assert_complete()
assert session.calls[0].method == "apply_patch"
assert session.calls[0].args[0] == [{"type": "delete_file", "path": "before.txt"}]
assert session.calls[1] == SandboxCall(
call_index=1,
method="exec",
args=("pwd",),
kwargs=MappingProxyType({"timeout": None, "shell": False, "user": None}),
)
returned_operations = cast(list[dict[str, object]], session.calls[0].args[0])
returned_operations[0]["path"] = "mutated.txt"
assert session.calls[0].args[0] == [{"type": "delete_file", "path": "before.txt"}]
@pytest.mark.asyncio
async def test_scripted_sandbox_snapshot_failure_does_not_commit_call() -> None:
class Uncopyable:
def __deepcopy__(self, memo: dict[int, object]) -> object:
_ = memo
raise RuntimeError("cannot snapshot")
session = scripted_sandbox_session(
[{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)}]
)
with pytest.raises(RuntimeError, match="cannot snapshot"):
await session.exec(cast(Any, Uncopyable()))
assert session.calls == ()
assert session.remaining_steps == 1
@pytest.mark.asyncio
async def test_scripted_sandbox_supports_matchers_responders_and_errors() -> None:
injected_error = RuntimeError("sandbox unavailable")
session = scripted_sandbox_session(
[
{
"method": "exec",
"match": lambda call: call.args == ("pwd",),
"responder": lambda call: ExecResult(
stdout=f"call {len(call.args)}".encode(), stderr=b"", exit_code=0
),
},
{"method": "exec", "error": injected_error},
]
)
assert (await session.exec("pwd")).stdout == b"call 1"
with pytest.raises(RuntimeError) as exc_info:
await session.exec("next")
assert exc_info.value is injected_error
session.assert_complete()
@pytest.mark.asyncio
async def test_scripted_sandbox_reports_structured_payload_free_failures() -> None:
mismatch = scripted_sandbox_session(
[
{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)},
{"method": "read", "result": None},
]
)
with pytest.raises(UnexpectedSandboxCall) as mismatch_info:
await mismatch.read(Path("/secret/payload.txt"))
mismatch_error = mismatch_info.value
assert mismatch_error.call_index == 0
assert "call #1" in str(mismatch_error)
assert mismatch_error.actual_method == "read"
assert mismatch_error.expected_method == "exec"
assert mismatch_error.remaining_steps == 2
assert mismatch.remaining_steps == 2
assert "/secret/payload.txt" not in str(mismatch_error)
await mismatch.exec("retry")
assert mismatch.remaining_steps == 1
rejected = scripted_sandbox_session(
[
{
"method": "exec",
"match": lambda call: call.args == ("expected",),
"result": ExecResult(stdout=b"", stderr=b"", exit_code=0),
}
]
)
with pytest.raises(SandboxCallMatcherError) as rejected_info:
await rejected.exec("secret command")
assert rejected_info.value.call_index == 0
assert "call #1" in str(rejected_info.value)
assert rejected_info.value.method == "exec"
assert "secret command" not in str(rejected_info.value)
assert rejected.remaining_steps == 1
await rejected.exec("expected")
rejected.assert_complete()
extra = scripted_sandbox_session(
[{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)}]
)
await extra.exec("first")
assert hasattr(extra, "exec")
with pytest.raises(UnexpectedSandboxCall) as extra_info:
await extra.exec("second secret")
assert extra_info.value.call_index == 1
assert "call #2" in str(extra_info.value)
assert extra_info.value.expected_method is None
assert extra_info.value.remaining_steps == 0
assert "second secret" not in str(extra_info.value)
@pytest.mark.asyncio
async def test_scripted_sandbox_retains_step_after_matcher_exception() -> None:
matcher_error = RuntimeError("matcher failed")
matcher_calls = 0
def match(_call: SandboxCall) -> bool:
nonlocal matcher_calls
matcher_calls += 1
if matcher_calls == 1:
raise matcher_error
return True
session = scripted_sandbox_session(
[
{
"method": "exec",
"match": match,
"result": ExecResult(stdout=b"ok", stderr=b"", exit_code=0),
}
]
)
with pytest.raises(RuntimeError) as exc_info:
await session.exec("first")
assert exc_info.value is matcher_error
assert session.remaining_steps == 1
assert (await session.exec("retry")).stdout == b"ok"
session.assert_complete()
def test_scripted_sandbox_reports_unconsumed_steps() -> None:
session = scripted_sandbox_session(
[
{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)},
{"method": "read", "result": None},
]
)
with pytest.raises(UnconsumedSandboxSteps) as exc_info:
session.assert_complete()
assert exc_info.value.remaining_steps == 2
assert exc_info.value.pending_methods == ("exec", "read")
@pytest.mark.parametrize(
("step", "reason"),
[
("not a mapping", "invalid_input"),
({"method": "exec", "matc": lambda _call: True, "result": None}, "invalid_input"),
({"method": "unknown", "result": None}, "unknown_method"),
({"method": "exec", "match": "no", "result": None}, "invalid_matcher"),
({"method": "exec"}, "invalid_outcome"),
({"method": "exec", "result": None, "error": RuntimeError()}, "invalid_outcome"),
({"method": "exec", "responder": "no"}, "invalid_outcome"),
({"method": "exec", "error": "no"}, "invalid_outcome"),
],
)
def test_scripted_sandbox_validates_steps_before_use(step: object, reason: str) -> None:
with pytest.raises(InvalidSandboxStep) as exc_info:
scripted_sandbox_session(cast(Any, [step]))
assert exc_info.value.reason == reason
assert exc_info.value.input_index == 0
assert "step #1" in str(exc_info.value)
@pytest.mark.asyncio
async def test_scripted_sandbox_drives_black_box_sandbox_agent_workflow() -> None:
session = scripted_sandbox_session(
[
{
"method": "exec",
"match": lambda call: call.args == ("pwd",),
"result": ExecResult(stdout=b"/workspace\n", stderr=b"", exit_code=0),
}
]
)
model = ScriptedModel(
[
[function_call("exec_command", {"cmd": "pwd"}, call_id="call_1")],
[assistant_message("The workspace is /workspace.")],
]
)
agent = SandboxAgent(
name="Test agent",
model=model,
capabilities=[Shell()],
)
result = await Runner.run(
agent,
"Where am I?",
run_config=RunConfig(sandbox={"session": session}),
)
assert result.final_output == "The workspace is /workspace."
assert len(session.calls) == 1
assert len(model.calls) == 2
session.assert_complete()
model.assert_complete()
+11 -10
View File
@@ -28,9 +28,10 @@ from agents.run_internal.run_loop import get_new_response, run_single_turn_strea
from agents.run_internal.run_steps import NextStepInterruption
from agents.run_internal.tool_use_tracker import AgentToolUseTracker
from agents.stream_events import RunItemStreamEvent
from agents.testing import ScriptedModel
from agents.usage import Usage
from tests.model_test_helpers import get_exact_output_stream_step
from .fake_model import FakeModel
from .test_responses import get_text_message
@@ -815,8 +816,8 @@ def test_prepare_input_does_not_resend_reasoning_item_after_marking_omitted_id_a
@pytest.mark.asyncio
async def test_get_new_response_marks_filtered_input_as_sent() -> None:
model = FakeModel()
model.set_next_output([get_text_message("ok")])
model = ScriptedModel()
model.enqueue([get_text_message("ok")])
agent = Agent(name="test", model=model)
tracker = OpenAIServerConversationTracker(conversation_id="conv4", previous_response_id=None)
context_wrapper: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
@@ -848,15 +849,15 @@ async def test_get_new_response_marks_filtered_input_as_sent() -> None:
None,
)
assert model.last_turn_args["input"] == [item_1]
assert model.calls[-1].input == [item_1]
assert any(item is item_1 for item in tracker.sent_items)
assert all(item is not item_2 for item in tracker.sent_items)
@pytest.mark.asyncio
async def test_run_single_turn_streamed_marks_filtered_input_as_sent() -> None:
model = FakeModel()
model.set_next_output([get_text_message("ok")])
model = ScriptedModel()
model.enqueue([get_text_message("ok")])
agent = Agent(name="test", model=model)
tracker = OpenAIServerConversationTracker(conversation_id="conv6", previous_response_id=None)
context_wrapper: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
@@ -903,13 +904,13 @@ async def test_run_single_turn_streamed_marks_filtered_input_as_sent() -> None:
server_conversation_tracker=tracker,
)
assert model.last_turn_args["input"] == [item_1]
assert model.calls[-1].input == [item_1]
assert tracker.remaining_initial_input == [item_2]
@pytest.mark.asyncio
async def test_run_single_turn_streamed_seeds_hosted_mcp_metadata_from_pre_step_items() -> None:
model = FakeModel()
model = ScriptedModel()
mcp_call = McpCall(
id="mcp_call_1",
arguments="{}",
@@ -918,7 +919,7 @@ async def test_run_single_turn_streamed_seeds_hosted_mcp_metadata_from_pre_step_
type="mcp_call",
status="completed",
)
model.set_next_output([mcp_call])
model.enqueue(get_exact_output_stream_step([mcp_call]))
agent = Agent(name="test", model=model)
hosted_tool = HostedMCPTool(
tool_config=cast(
@@ -977,7 +978,7 @@ async def test_run_single_turn_streamed_seeds_hosted_mcp_metadata_from_pre_step_
all_tools=[hosted_tool],
)
assert model.last_turn_args["input"] == [item_1]
assert model.calls[-1].input == [item_1]
tool_call_events: list[ToolCallItem] = []
while not streamed_result._event_queue.empty():
+3 -3
View File
@@ -6,8 +6,8 @@ from agents.agent import Agent
from agents.exceptions import ModelBehaviorError
from agents.items import ToolCallOutputItem
from agents.run_internal import run_loop
from agents.testing import ScriptedModel
from agents.tool import ShellCallOutcome, ShellCommandOutput
from tests.fake_model import FakeModel
def test_coerce_shell_call_reads_max_output_length() -> None:
@@ -122,7 +122,7 @@ def test_serialize_shell_output_emits_canonical_outcome() -> None:
def test_shell_rejection_payload_preserves_missing_exit_code() -> None:
agent = Agent(name="tester", model=FakeModel())
agent = Agent(name="tester", model=ScriptedModel())
raw_item = {
"type": "shell_call_output",
"call_id": "call-1",
@@ -148,7 +148,7 @@ def test_shell_rejection_payload_preserves_missing_exit_code() -> None:
def test_shell_output_preserves_zero_exit_code() -> None:
agent = Agent(name="tester", model=FakeModel())
agent = Agent(name="tester", model=ScriptedModel())
raw_item = {
"type": "shell_call_output",
"call_id": "call-2",
+38 -38
View File
@@ -10,8 +10,8 @@ import pytest
from agents import Agent, Runner, SQLiteSession
from agents.agent_output import AgentOutputSchema
from agents.stream_events import StreamEvent
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .test_responses import (
get_function_tool,
get_function_tool_call,
@@ -23,7 +23,7 @@ from .test_responses import (
@pytest.mark.asyncio
async def test_soft_cancel_completes_turn():
"""Verify soft cancel waits for turn to complete."""
model = FakeModel()
model = ScriptedModel([[]])
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -44,7 +44,7 @@ async def test_soft_cancel_completes_turn():
async def test_soft_cancel_vs_immediate():
"""Compare soft cancel vs immediate cancel behavior."""
# Immediate cancel
model1 = FakeModel()
model1 = ScriptedModel([[]])
agent1 = Agent(name="A1", model=model1)
result1 = Runner.run_streamed(agent1, input="Hello")
immediate_events = []
@@ -54,7 +54,7 @@ async def test_soft_cancel_vs_immediate():
result1.cancel(mode="immediate")
# Soft cancel
model2 = FakeModel()
model2 = ScriptedModel([[]])
agent2 = Agent(name="A2", model=model2)
result2 = Runner.run_streamed(agent2, input="Hello")
soft_events = []
@@ -72,14 +72,14 @@ async def test_soft_cancel_vs_immediate():
@pytest.mark.asyncio
async def test_soft_cancel_with_tool_calls():
"""Verify tool calls execute before soft cancel stops."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="Assistant",
model=model,
tools=[get_function_tool("calc", "42")],
)
model.add_multiple_turn_outputs(
model.extend(
[
[
get_text_message("Let me calculate"),
@@ -109,7 +109,7 @@ async def test_soft_cancel_with_tool_calls():
@pytest.mark.asyncio
async def test_soft_cancel_saves_session():
"""Verify session is saved properly with soft cancel."""
model = FakeModel()
model = ScriptedModel([[], []])
agent = Agent(name="Assistant", model=model)
session = SQLiteSession("test_soft_cancel_session")
@@ -136,7 +136,7 @@ async def test_soft_cancel_saves_session():
@pytest.mark.asyncio
async def test_soft_cancel_tracks_usage():
"""Verify usage is tracked for completed turn."""
model = FakeModel()
model = ScriptedModel([[]])
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -145,7 +145,7 @@ async def test_soft_cancel_tracks_usage():
if event.type == "raw_response_event":
result.cancel(mode="after_turn")
# Usage should be tracked (FakeModel tracks requests even if tokens are 0)
# Usage should be tracked (ScriptedModel tracks requests even if tokens are 0)
assert result.context_wrapper.usage.requests > 0
@@ -153,7 +153,7 @@ async def test_soft_cancel_tracks_usage():
@pytest.mark.parametrize("consumer_suspensions", [0, 1, 3])
async def test_soft_cancel_stops_next_turn(consumer_suspensions: int):
"""Verify soft cancel prevents next turn from starting."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="Assistant",
model=model,
@@ -161,7 +161,7 @@ async def test_soft_cancel_stops_next_turn(consumer_suspensions: int):
)
# Set up multi-turn scenario
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("tool1", "{}")],
[get_text_message("Turn 2")],
@@ -188,13 +188,13 @@ async def test_soft_cancel_stops_next_turn(consumer_suspensions: int):
@pytest.mark.asyncio
async def test_soft_cancel_stops_next_turn_with_short_lived_anext_tasks():
"""Per-event tasks must not acknowledge a turn before the caller handles its event."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="Assistant",
model=model,
tools=[get_function_tool("tool1", "result1")],
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("tool1", "{}")],
[get_text_message("Turn 2")],
@@ -218,8 +218,8 @@ async def test_soft_cancel_stops_next_turn_with_short_lived_anext_tasks():
@pytest.mark.asyncio
async def test_streamed_run_completes_without_an_event_consumer():
"""Turn acknowledgement must not block a run whose events are not consumed."""
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("tool1", "{}")],
[get_text_message("Turn 2")],
@@ -242,8 +242,8 @@ async def test_streamed_run_completes_without_an_event_consumer():
@pytest.mark.asyncio
async def test_closing_stream_consumer_releases_turn_acknowledgement():
"""Closing an iterator must not deadlock while a turn awaits its consumer."""
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("tool1", "{}")],
[get_text_message("Turn 2")],
@@ -271,8 +271,8 @@ async def test_closing_stream_consumer_releases_turn_acknowledgement():
@pytest.mark.asyncio
async def test_cancelled_stream_consumer_releases_turn_acknowledgement():
"""Cancelling a consumer suspended after yield must release the completed turn."""
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("tool1", "{}")],
[get_text_message("Turn 2")],
@@ -315,8 +315,8 @@ async def test_cancelled_stream_consumer_releases_turn_acknowledgement():
@pytest.mark.asyncio
async def test_immediate_cancel_releases_turn_acknowledgement():
"""Immediate cancellation must cancel a run waiting for streamed event acknowledgement."""
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("tool1", "{}")],
[get_text_message("Turn 2")],
@@ -342,7 +342,7 @@ async def test_immediate_cancel_releases_turn_acknowledgement():
@pytest.mark.asyncio
async def test_cancel_mode_backward_compatibility():
"""Verify default behavior unchanged."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -363,7 +363,7 @@ async def test_cancel_mode_backward_compatibility():
@pytest.mark.asyncio
async def test_soft_cancel_idempotent():
"""Verify calling cancel multiple times is safe."""
model = FakeModel()
model = ScriptedModel([[]])
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -382,7 +382,7 @@ async def test_soft_cancel_idempotent():
@pytest.mark.asyncio
async def test_soft_cancel_before_streaming():
"""Verify soft cancel before streaming starts."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -398,7 +398,7 @@ async def test_soft_cancel_before_streaming():
@pytest.mark.asyncio
async def test_soft_cancel_mixed_modes():
"""Verify changing cancel mode behaves correctly."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -418,7 +418,7 @@ async def test_soft_cancel_mixed_modes():
@pytest.mark.asyncio
async def test_soft_cancel_explicit_immediate_mode():
"""Test explicit immediate mode behaves same as default."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -439,7 +439,7 @@ async def test_soft_cancel_explicit_immediate_mode():
@pytest.mark.asyncio
async def test_soft_cancel_with_multiple_tool_calls():
"""Verify soft cancel works with multiple tool calls in one turn."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="Assistant",
model=model,
@@ -450,7 +450,7 @@ async def test_soft_cancel_with_multiple_tool_calls():
)
# Turn with multiple tool calls
model.add_multiple_turn_outputs(
model.extend(
[
[
get_function_tool_call("tool1", "{}", call_id="tool_1"),
@@ -479,14 +479,14 @@ async def test_soft_cancel_with_multiple_tool_calls():
@pytest.mark.asyncio
async def test_soft_cancel_preserves_state():
"""Verify soft cancel preserves all result state correctly."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="Assistant",
model=model,
tools=[get_function_tool("tool1", "result")],
)
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("tool1", "{}")],
[get_text_message("Done")],
@@ -509,7 +509,7 @@ async def test_soft_cancel_preserves_state():
@pytest.mark.asyncio
async def test_immediate_cancel_clears_queues():
"""Verify immediate cancel clears queues as expected."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -528,7 +528,7 @@ async def test_immediate_cancel_clears_queues():
@pytest.mark.asyncio
async def test_soft_cancel_does_not_clear_queues_immediately():
"""Verify soft cancel does NOT clear queues immediately."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="Assistant", model=model)
result = Runner.run_streamed(agent, input="Hello")
@@ -551,7 +551,7 @@ async def test_soft_cancel_with_handoff():
"""Verify soft cancel after handoff saves the handoff turn."""
from agents import Handoff
model = FakeModel()
model = ScriptedModel()
# Create two agents with handoff
agent2 = Agent(name="Agent2", model=model)
@@ -574,7 +574,7 @@ async def test_soft_cancel_with_handoff():
)
# Setup: Agent1 does handoff, Agent2 responds
model.add_multiple_turn_outputs(
model.extend(
[
# Agent1's turn - triggers handoff
[get_function_tool_call(Handoff.default_tool_name(agent2), "{}")],
@@ -610,7 +610,7 @@ async def test_soft_cancel_waits_for_handoff_event_consumption_before_next_turn(
"""A suspended handoff consumer can stop the run before the delegate model starts."""
second_request_started = asyncio.Event()
class HandoffModel(FakeModel):
class HandoffModel(ScriptedModel):
def __init__(self) -> None:
super().__init__()
self.request_count = 0
@@ -625,7 +625,7 @@ async def test_soft_cancel_waits_for_handoff_event_consumption_before_next_turn(
model = HandoffModel()
delegate = Agent(name="Delegate", model=model, output_type=int)
triage = Agent(name="Triage", model=model, handoffs=[delegate])
model.add_multiple_turn_outputs(
model.extend(
[
[get_handoff_tool_call(delegate)],
[get_text_message("Delegate response")],
@@ -666,7 +666,7 @@ async def test_soft_cancel_waits_for_handoff_event_consumption_before_next_turn(
@pytest.mark.asyncio
async def test_soft_cancel_with_session_and_multiple_turns():
"""Verify soft cancel with session across multiple turns."""
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="Assistant",
model=model,
@@ -677,7 +677,7 @@ async def test_soft_cancel_with_session_and_multiple_turns():
await session.clear_session()
# Setup 3 turns
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("tool1", "{}", call_id="tool_1")],
[get_function_tool_call("tool1", "{}", call_id="tool_2")],
+43 -34
View File
@@ -51,10 +51,11 @@ from agents.items import (
ToolSearchOutputItem,
)
from agents.run_internal.streaming import stream_step_items_to_queue, stream_step_result_to_queue
from agents.testing import ScriptedModel
from .fake_model import FakeModel
from .mcp.helpers import FakeMCPServer
from .mcp.model_compat import Tool as MCPTool
from .model_test_helpers import get_exact_output_stream_step
from .test_responses import get_function_tool_call, get_handoff_tool_call, get_text_message
@@ -88,14 +89,14 @@ async def foo() -> str:
@pytest.mark.asyncio
async def test_stream_events_main():
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="Joker",
model=model,
tools=[foo],
)
model.add_multiple_turn_outputs(
model.extend(
[
# First turn: a message and tool call
[
@@ -127,7 +128,7 @@ async def test_stream_events_main():
@pytest.mark.asyncio
async def test_stream_events_tool_called_includes_local_mcp_title() -> None:
model = FakeModel()
model = ScriptedModel()
server = FakeMCPServer(
tools=[
MCPTool(
@@ -140,7 +141,7 @@ async def test_stream_events_tool_called_includes_local_mcp_title() -> None:
)
agent = Agent(name="MCPAgent", model=model, mcp_servers=[server])
model.add_multiple_turn_outputs(
model.extend(
[
[get_function_tool_call("search_docs", "{}")],
[get_text_message("done")],
@@ -287,11 +288,11 @@ async def test_stream_events_main_with_handoff():
english_agent = Agent(
name="EnglishAgent",
instructions="You only speak English.",
model=FakeModel(),
model=ScriptedModel([[]]),
)
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_text_message("Hello"),
@@ -341,14 +342,14 @@ async def test_complete_streaming_events():
- Function call with arguments delta/done events
- Message output with content_part and text delta/done events
"""
model = FakeModel()
model = ScriptedModel()
agent = Agent(
name="TestAgent",
model=model,
tools=[foo],
)
model.add_multiple_turn_outputs(
model.extend(
[
[
get_reasoning_item(),
@@ -481,8 +482,8 @@ async def test_complete_streaming_events():
@pytest.mark.asyncio
async def test_tool_call_event_preserves_order_before_later_reasoning_item() -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call("foo", '{"arg": "value"}'),
@@ -511,12 +512,14 @@ async def test_tool_call_event_preserves_order_before_later_reasoning_item() ->
async def test_handoff_event_preserves_order_before_later_reasoning_item() -> None:
english_agent = Agent(
name="EnglishAgent",
model=FakeModel(initial_output=[get_text_message("Done")]),
model=ScriptedModel(steps=[[get_text_message("Done")]]),
)
model = FakeModel(
initial_output=[
get_handoff_tool_call(english_agent),
get_reasoning_item(),
model = ScriptedModel(
steps=[
[
get_handoff_tool_call(english_agent),
get_reasoning_item(),
]
]
)
triage_agent = Agent(name="TriageAgent", model=model, handoffs=[english_agent])
@@ -542,12 +545,14 @@ async def test_handoff_filter_copy_does_not_duplicate_streamed_model_items() ->
english_agent = Agent(
name="EnglishAgent",
model=FakeModel(initial_output=[get_text_message("Done")]),
model=ScriptedModel(steps=[[get_text_message("Done")]]),
)
model = FakeModel(
initial_output=[
get_text_message("Transferring"),
get_handoff_tool_call(english_agent),
model = ScriptedModel(
steps=[
[
get_text_message("Transferring"),
get_handoff_tool_call(english_agent),
]
]
)
triage_agent = Agent(
@@ -572,7 +577,7 @@ async def test_handoff_filter_copy_does_not_duplicate_streamed_model_items() ->
@pytest.mark.asyncio
async def test_stream_events_emit_tool_search_items() -> None:
model = FakeModel()
model = ScriptedModel()
agent = Agent(name="ToolSearchAgent", model=model)
tool_search_call = cast(
ResponseOutputItem,
@@ -616,8 +621,12 @@ async def test_stream_events_emit_tool_search_items() -> None:
},
),
)
model.add_multiple_turn_outputs(
[[tool_search_call, tool_search_output, get_text_message("Done")]]
model.extend(
[
get_exact_output_stream_step(
[tool_search_call, tool_search_output, get_text_message("Done")]
)
]
)
result = Runner.run_streamed(agent, input="Search for CRM order tools")
@@ -641,10 +650,10 @@ async def test_stream_events_emit_tool_search_items() -> None:
@pytest.mark.asyncio
async def test_streamed_handoff_call_is_not_emitted_as_tool_called():
"""A handoff call streams only as `handoff_requested`, never also as `tool_called`."""
english_agent = Agent(name="EnglishAgent", model=FakeModel())
english_agent = Agent(name="EnglishAgent", model=ScriptedModel([[]]))
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_handoff_tool_call(english_agent)],
[get_text_message("Done")],
@@ -673,10 +682,10 @@ async def test_streamed_handoff_call_is_not_emitted_as_tool_called():
@pytest.mark.asyncio
async def test_streamed_tool_call_alongside_handoff_still_emits_tool_called():
"""A real tool call in the same turn as a handoff keeps its `tool_called` event."""
english_agent = Agent(name="EnglishAgent", model=FakeModel())
english_agent = Agent(name="EnglishAgent", model=ScriptedModel([[]]))
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call("foo", '{"a": "b"}', call_id="tool_call"),
@@ -712,10 +721,10 @@ async def test_streamed_tool_call_alongside_handoff_still_emits_tool_called():
@pytest.mark.asyncio
async def test_streamed_handoff_item_events_match_new_items():
"""Streamed run item events stay in sync with the items recorded on the result."""
english_agent = Agent(name="EnglishAgent", model=FakeModel())
english_agent = Agent(name="EnglishAgent", model=ScriptedModel([[]]))
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_text_message("Transferring"), get_handoff_tool_call(english_agent)],
[get_text_message("Done")],
+17 -17
View File
@@ -10,7 +10,7 @@ from openai.types.responses import ResponseCompletedEvent
from agents import Agent, GuardrailFunctionOutput, InputGuardrail, RunContextWrapper, Runner
from agents.exceptions import InputGuardrailTripwireTriggered
from agents.items import TResponseInputItem
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
from tests.testing_processor import fetch_events, fetch_ordered_spans
@@ -49,8 +49,8 @@ async def test_input_guardrail_results_follow_completion_order():
output_info={"delay": FAST_GUARDRAIL_DELAY}, tripwire_triggered=False
)
model = FakeModel()
model.set_next_output([get_text_message("Final response")])
model = ScriptedModel()
model.enqueue([get_text_message("Final response")])
agent = Agent(
name="TimingAgentOrder",
@@ -81,8 +81,8 @@ async def test_run_streamed_input_guardrail_timing_is_consistent(guardrail_delay
"""
# Arrange: Agent with a single text output and a delayed input guardrail
model = FakeModel()
model.set_next_output([get_text_message("Final response")])
model = ScriptedModel()
model.enqueue([get_text_message("Final response")])
agent = Agent(
name="TimingAgent",
@@ -122,8 +122,8 @@ async def test_run_streamed_input_guardrail_sequences_match_between_fast_and_slo
"""Run twice with fast vs slow input guardrail and compare event sequences exactly."""
async def run_once(delay: float) -> list[str]:
model = FakeModel()
model.set_next_output([get_text_message("Final response")])
model = ScriptedModel()
model.enqueue([get_text_message("Final response")])
agent = Agent(
name="TimingAgent",
model=model,
@@ -148,8 +148,8 @@ async def test_run_streamed_input_guardrail_sequences_match_between_fast_and_slo
async def test_run_streamed_input_guardrail_tripwire_raises(guardrail_delay: float):
"""Guardrail tripwire must raise from stream_events regardless of timing."""
model = FakeModel()
model.set_next_output([get_text_message("Final response")])
model = ScriptedModel()
model.enqueue([get_text_message("Final response")])
agent = Agent(
name="TimingAgentTrip",
@@ -173,11 +173,11 @@ async def test_run_streamed_input_guardrail_tripwire_raises(guardrail_delay: flo
)
class SlowCompleteFakeModel(FakeModel):
"""A FakeModel that delays just before emitting ResponseCompletedEvent in streaming."""
class SlowCompleteScriptedModel(ScriptedModel):
"""A ScriptedModel that delays just before emitting ResponseCompletedEvent in streaming."""
def __init__(self, delay_seconds: float, tracing_enabled: bool = True):
super().__init__(tracing_enabled=tracing_enabled)
def __init__(self, delay_seconds: float, emit_traces: bool = True):
super().__init__(emit_traces=emit_traces)
self._delay_seconds = delay_seconds
async def stream_response(self, *args, **kwargs):
@@ -206,8 +206,8 @@ def _iso(s: str | None) -> datetime:
async def test_parent_span_and_trace_finish_after_slow_input_guardrail():
"""Agent span and trace finish after guardrail when guardrail completes last."""
model = FakeModel(tracing_enabled=True)
model.set_next_output([get_text_message("Final response")])
model = ScriptedModel(emit_traces=True)
model.enqueue([get_text_message("Final response")])
agent = Agent(
name="TimingAgentTrace",
model=model,
@@ -240,8 +240,8 @@ async def test_parent_span_and_trace_finish_after_slow_input_guardrail():
async def test_parent_span_and_trace_finish_after_slow_model():
"""Agent span and trace finish after model when model completes last."""
model = SlowCompleteFakeModel(delay_seconds=SLOW_GUARDRAIL_DELAY, tracing_enabled=True)
model.set_next_output([get_text_message("Final response")])
model = SlowCompleteScriptedModel(delay_seconds=SLOW_GUARDRAIL_DELAY, emit_traces=True)
model.enqueue([get_text_message("Final response")])
agent = Agent(
name="TimingAgentTrace",
model=model,
+42 -106
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import json
from collections.abc import AsyncIterator
from typing import Any
import pytest
from openai.types.responses import (
@@ -13,75 +12,24 @@ from openai.types.responses import (
)
from agents import Agent, Runner
from agents.agent_output import AgentOutputSchemaBase
from agents.handoffs import Handoff
from agents.items import TResponseInputItem, TResponseOutputItem, TResponseStreamEvent
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
from agents.tool import Tool, function_tool
from agents.items import TResponseOutputItem, TResponseStreamEvent
from agents.testing import ModelStep, ScriptedModel
from agents.tool import function_tool
from .fake_model import FakeModel, get_response_obj
from .model_test_helpers import get_response_obj
from .test_responses import get_final_output_message, get_function_tool_call
class TerminalOutputStreamModel(FakeModel):
def __init__(self) -> None:
super().__init__()
self.terminal_turn_outputs: list[list[TResponseOutputItem]] = []
def add_terminal_turn_outputs(
self,
outputs: list[list[TResponseOutputItem]],
) -> None:
self.terminal_turn_outputs.extend(outputs)
def get_next_terminal_output(self) -> list[TResponseOutputItem]:
if not self.terminal_turn_outputs:
return []
return self.terminal_turn_outputs.pop(0)
async def stream_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None = None,
conversation_id: str | None = None,
prompt: Any | None = None,
) -> AsyncIterator[TResponseStreamEvent]:
turn_args = {
"system_instructions": system_instructions,
"input": input,
"model_settings": model_settings,
"tools": tools,
"output_schema": output_schema,
"previous_response_id": previous_response_id,
"conversation_id": conversation_id,
}
if self.first_turn_args is None:
self.first_turn_args = turn_args.copy()
self.last_turn_args = turn_args
streamed_output = self.get_next_output()
if isinstance(streamed_output, Exception):
raise streamed_output
terminal_response = get_response_obj(
self.get_next_terminal_output(),
usage=self.hardcoded_usage,
)
def _stream_step(
streamed_output: list[TResponseOutputItem],
terminal_output: list[TResponseOutputItem],
) -> ModelStep:
async def events(_call) -> AsyncIterator[TResponseStreamEvent]:
terminal_response = get_response_obj(terminal_output)
sequence_number = 0
yield ResponseCreatedEvent(
type="response.created",
response=terminal_response,
sequence_number=sequence_number,
type="response.created", response=terminal_response, sequence_number=sequence_number
)
sequence_number += 1
@@ -107,6 +55,8 @@ class TerminalOutputStreamModel(FakeModel):
sequence_number=sequence_number,
)
return ModelStep.stream(events)
@pytest.mark.asyncio
async def test_streamed_runner_backfills_empty_terminal_output_before_step_resolution() -> None:
@@ -117,22 +67,17 @@ async def test_streamed_runner_backfills_empty_terminal_output_before_step_resol
return "tool_result"
tool = function_tool(test_tool, name_override="foo")
model = TerminalOutputStreamModel()
model = ScriptedModel(
[
_stream_step(
[get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call-1")],
[],
),
_stream_step([get_final_output_message("done")], [get_final_output_message("done")]),
]
)
agent = Agent(name="test", model=model, tools=[tool])
model.add_multiple_turn_outputs(
[
[get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call-1")],
[get_final_output_message("done")],
]
)
model.add_terminal_turn_outputs(
[
[],
[get_final_output_message("done")],
]
)
result = Runner.run_streamed(agent, input="test")
async for _ in result.stream_events():
pass
@@ -151,20 +96,16 @@ async def test_streamed_runner_preserves_populated_terminal_output() -> None:
return "tool_result"
tool = function_tool(test_tool, name_override="foo")
model = TerminalOutputStreamModel()
model = ScriptedModel(
[
_stream_step(
[get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call-1")],
[get_final_output_message("done")],
)
]
)
agent = Agent(name="test", model=model, tools=[tool])
model.add_multiple_turn_outputs(
[
[get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call-1")],
]
)
model.add_terminal_turn_outputs(
[
[get_final_output_message("done")],
]
)
result = Runner.run_streamed(agent, input="test")
async for _ in result.stream_events():
pass
@@ -188,25 +129,20 @@ async def test_streamed_runner_backfills_multiple_tool_calls_in_order() -> None:
foo = function_tool(foo_tool, name_override="foo")
bar = function_tool(bar_tool, name_override="bar")
model = TerminalOutputStreamModel()
model = ScriptedModel(
[
_stream_step(
[
get_function_tool_call("foo", json.dumps({"a": "first"}), call_id="call-1"),
get_function_tool_call("bar", json.dumps({"b": "second"}), call_id="call-2"),
],
[],
),
_stream_step([get_final_output_message("done")], [get_final_output_message("done")]),
]
)
agent = Agent(name="test", model=model, tools=[foo, bar])
model.add_multiple_turn_outputs(
[
[
get_function_tool_call("foo", json.dumps({"a": "first"}), call_id="call-1"),
get_function_tool_call("bar", json.dumps({"b": "second"}), call_id="call-2"),
],
[get_final_output_message("done")],
]
)
model.add_terminal_turn_outputs(
[
[],
[get_final_output_message("done")],
]
)
result = Runner.run_streamed(agent, input="test")
async for _ in result.stream_events():
pass
+3 -3
View File
@@ -10,7 +10,7 @@ from agents.items import ToolCallOutputItem
from agents.run import AgentRunner
from agents.run_context import RunContextWrapper
from agents.run_state import RunState
from tests.fake_model import FakeModel
from agents.testing import ScriptedModel
from tests.test_responses import get_text_message
@@ -20,8 +20,8 @@ async def test_run_streamed_resume_omits_tool_output_in_log_when_dont_log(
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True)
model = FakeModel()
model.set_next_output([get_text_message("ok")])
model = ScriptedModel()
model.enqueue([get_text_message("ok")])
agent = Agent(name="log-agent", model=model)
context_wrapper: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
state = RunState(
+74 -128
View File
@@ -7,7 +7,7 @@ were emitted with empty arguments during streaming (Issue #1629).
import json
from collections.abc import AsyncIterator
from typing import Any, cast
from typing import cast
import pytest
from openai.types.responses import (
@@ -18,116 +18,54 @@ from openai.types.responses import (
)
from agents import Agent, Runner, function_tool
from agents.agent_output import AgentOutputSchemaBase
from agents.handoffs import Handoff
from agents.items import TResponseInputItem, TResponseOutputItem, TResponseStreamEvent
from agents.model_settings import ModelSettings
from agents.models.interface import Model, ModelTracing
from agents.items import TResponseOutputItem, TResponseStreamEvent
from agents.stream_events import RunItemStreamEvent
from agents.tool import Tool
from agents.tracing import generation_span
from agents.testing import ModelStep, ScriptedModel
from tests.model_test_helpers import get_response_obj
from .fake_model import get_response_obj
from .test_responses import get_function_tool_call
class StreamingFakeModel(Model):
"""A fake model that actually emits streaming events to test our streaming fix."""
def _split_argument_step(output: list[TResponseOutputItem]) -> ModelStep:
async def events(_call) -> AsyncIterator[TResponseStreamEvent]:
sequence_number = 0
def __init__(self):
self.turn_outputs: list[list[TResponseOutputItem]] = []
self.last_turn_args: dict[str, Any] = {}
# Emit each output item with proper streaming events.
for item in output:
if isinstance(item, ResponseFunctionToolCall):
# First emit an added event with empty arguments, as the API does before deltas.
empty_args_item = ResponseFunctionToolCall(
id=item.id,
call_id=item.call_id,
type=item.type,
name=item.name,
arguments="",
)
def set_next_output(self, output: list[TResponseOutputItem]):
self.turn_outputs.append(output)
yield ResponseOutputItemAddedEvent(
item=empty_args_item,
output_index=0,
type="response.output_item.added",
sequence_number=sequence_number,
)
sequence_number += 1
def get_next_output(self) -> list[TResponseOutputItem]:
if not self.turn_outputs:
return []
return self.turn_outputs.pop(0)
# Then emit the completed item with its final arguments.
yield ResponseOutputItemDoneEvent(
item=item,
output_index=0,
type="response.output_item.done",
sequence_number=sequence_number,
)
sequence_number += 1
async def get_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: Any | None,
):
raise NotImplementedError("Use stream_response instead")
yield ResponseCompletedEvent(
type="response.completed",
response=get_response_obj(output),
sequence_number=sequence_number,
)
async def stream_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None = None,
conversation_id: str | None = None,
prompt: Any | None = None,
) -> AsyncIterator[TResponseStreamEvent]:
"""Stream events that simulate real OpenAI streaming behavior for tool calls."""
self.last_turn_args = {
"system_instructions": system_instructions,
"input": input,
"model_settings": model_settings,
"tools": tools,
"output_schema": output_schema,
"previous_response_id": previous_response_id,
"conversation_id": conversation_id,
}
with generation_span(disabled=True) as _:
output = self.get_next_output()
sequence_number = 0
# Emit each output item with proper streaming events
for item in output:
if isinstance(item, ResponseFunctionToolCall):
# First: emit ResponseOutputItemAddedEvent with EMPTY arguments
# (this simulates the real streaming behavior that was causing the bug)
empty_args_item = ResponseFunctionToolCall(
id=item.id,
call_id=item.call_id,
type=item.type,
name=item.name,
arguments="", # EMPTY - this is the bug condition!
)
yield ResponseOutputItemAddedEvent(
item=empty_args_item,
output_index=0,
type="response.output_item.added",
sequence_number=sequence_number,
)
sequence_number += 1
# Then: emit ResponseOutputItemDoneEvent with COMPLETE arguments
yield ResponseOutputItemDoneEvent(
item=item, # This has the complete arguments
output_index=0,
type="response.output_item.done",
sequence_number=sequence_number,
)
sequence_number += 1
# Finally: emit completion
yield ResponseCompletedEvent(
type="response.completed",
response=get_response_obj(output),
sequence_number=sequence_number,
)
return ModelStep.stream(events)
@function_tool
@@ -146,7 +84,7 @@ def format_message(name: str, message: str, urgent: bool = False) -> str:
@pytest.mark.asyncio
async def test_streaming_tool_call_arguments_not_empty():
"""Test that tool_called events contain non-empty arguments during streaming."""
model = StreamingFakeModel()
model = ScriptedModel()
agent = Agent(
name="TestAgent",
model=model,
@@ -155,12 +93,14 @@ async def test_streaming_tool_call_arguments_not_empty():
# Set up a tool call with arguments
expected_arguments = '{"a": 5, "b": 3}'
model.set_next_output(
[
get_function_tool_call("calculate_sum", expected_arguments, "call_123"),
]
model.enqueue(
_split_argument_step(
[
get_function_tool_call("calculate_sum", expected_arguments, "call_123"),
]
)
)
model.enqueue([])
result = Runner.run_streamed(agent, input="Add 5 and 3")
tool_called_events = []
@@ -212,7 +152,7 @@ async def test_streaming_tool_call_arguments_not_empty():
@pytest.mark.asyncio
async def test_streaming_tool_call_arguments_complex():
"""Test streaming tool calls with complex arguments including strings and booleans."""
model = StreamingFakeModel()
model = ScriptedModel()
agent = Agent(
name="TestAgent",
model=model,
@@ -223,12 +163,14 @@ async def test_streaming_tool_call_arguments_complex():
expected_arguments = (
'{"name": "Alice", "message": "Your meeting is starting soon", "urgent": true}'
)
model.set_next_output(
[
get_function_tool_call("format_message", expected_arguments, "call_456"),
]
model.enqueue(
_split_argument_step(
[
get_function_tool_call("format_message", expected_arguments, "call_456"),
]
)
)
model.enqueue([])
result = Runner.run_streamed(agent, input="Format a message for Alice")
tool_called_events = []
@@ -265,7 +207,7 @@ async def test_streaming_tool_call_arguments_complex():
@pytest.mark.asyncio
async def test_streaming_multiple_tool_calls_arguments():
"""Test that multiple tool calls in streaming all have proper arguments."""
model = StreamingFakeModel()
model = ScriptedModel()
agent = Agent(
name="TestAgent",
model=model,
@@ -273,15 +215,17 @@ async def test_streaming_multiple_tool_calls_arguments():
)
# Set up multiple tool calls
model.set_next_output(
[
get_function_tool_call("calculate_sum", '{"a": 10, "b": 20}', "call_1"),
get_function_tool_call(
"format_message", '{"name": "Bob", "message": "Test"}', "call_2"
),
]
model.enqueue(
_split_argument_step(
[
get_function_tool_call("calculate_sum", '{"a": 10, "b": 20}', "call_1"),
get_function_tool_call(
"format_message", '{"name": "Bob", "message": "Test"}', "call_2"
),
]
)
)
model.enqueue([])
result = Runner.run_streamed(agent, input="Do some calculations")
tool_called_events = []
@@ -324,7 +268,7 @@ async def test_streaming_multiple_tool_calls_arguments():
@pytest.mark.asyncio
async def test_streaming_tool_call_with_empty_arguments():
"""Test that tool calls with legitimately empty arguments still work correctly."""
model = StreamingFakeModel()
model = ScriptedModel()
@function_tool
def get_current_time() -> str:
@@ -338,12 +282,14 @@ async def test_streaming_tool_call_with_empty_arguments():
)
# Tool call with empty arguments (legitimate case)
model.set_next_output(
[
get_function_tool_call("get_current_time", "{}", "call_time"),
]
model.enqueue(
_split_argument_step(
[
get_function_tool_call("get_current_time", "{}", "call_time"),
]
)
)
model.enqueue([])
result = Runner.run_streamed(agent, input="What time is it?")
tool_called_events = []
+195 -165
View File
@@ -46,7 +46,7 @@ from agents._tool_invocation import (
)
from agents.editor import ApplyPatchOperation, ApplyPatchResult
from agents.exceptions import ModelBehaviorError, UserError
from agents.items import ModelResponse, ToolApprovalItem
from agents.items import ModelResponse, ToolApprovalItem, TResponseOutputItem
from agents.lifecycle import RunHooks
from agents.models.interface import Model, ModelProvider
from agents.run_context import RunContextWrapper
@@ -59,9 +59,10 @@ from agents.run_internal.tool_execution import (
from agents.run_internal.tool_planning import _collect_runs_by_approval
from agents.run_state import RunState
from agents.stream_events import RunItemStreamEvent
from agents.testing import ScriptedModel
from agents.tool import Tool
from agents.tool_context import ToolContext
from tests.fake_model import FakeModel
from tests.model_test_helpers import get_exact_output_stream_step
from tests.test_computer_tool_lifecycle import FakeComputer
from tests.test_responses import get_function_tool_call, get_handoff_tool_call, get_text_message
from tests.utils.hitl import make_apply_patch_dict, make_shell_call, make_state_with_interruptions
@@ -205,8 +206,8 @@ async def test_completed_apply_patch_fallback_run_state_round_trip(fallback_type
call_id="patch_0",
arguments=json.dumps(operation),
)
model = FakeModel(initial_output=[call])
model.set_next_output([get_text_message("done")])
model = ScriptedModel(steps=[[call]])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[tool])
result = await Runner.run(agent, "update the file")
@@ -236,10 +237,8 @@ async def test_streamed_function_apply_patch_replay_emits_one_tool_called_event(
call_id="patch_0",
arguments=json.dumps(operation),
)
model = FakeModel()
model.add_multiple_turn_outputs(
[[call], [call.model_copy(deep=True)], [get_text_message("done")]]
)
model = ScriptedModel()
model.extend([[call], [call.model_copy(deep=True)], [get_text_message("done")]])
agent = Agent(
name="agent",
model=model,
@@ -305,8 +304,8 @@ async def test_custom_named_shell_sticky_approval_applies_to_fresh_call_ids() ->
first_call = make_shell_call("call_0", commands=["echo first"])
second_call = make_shell_call("call_1", commands=["echo second"])
model = FakeModel()
model.add_multiple_turn_outputs([[first_call], [second_call], [get_text_message("done")]])
model = ScriptedModel()
model.extend([[first_call], [second_call], [get_text_message("done")]])
tool = ShellTool(
executor=execute,
name="safe_shell",
@@ -337,7 +336,7 @@ async def test_custom_named_shell_replacement_rejects_completed_call_id_replay()
return "ok"
call = make_shell_call("call_0", commands=["echo safe"])
model = FakeModel(initial_output=[call])
model = ScriptedModel(steps=[[call]])
original_tool = ShellTool(
executor=run_first,
name="safe_shell",
@@ -348,7 +347,7 @@ async def test_custom_named_shell_replacement_rejects_completed_call_id_replay()
first = await Runner.run(agent, "run command")
state = first.to_state()
state.approve(first.interruptions[0])
model.set_next_output([get_text_message("done")])
model.enqueue([get_text_message("done")])
completed = await Runner.run(agent, state)
agent.tools = [
@@ -358,7 +357,7 @@ async def test_custom_named_shell_replacement_rejects_completed_call_id_replay()
needs_approval=False,
)
]
model.set_next_output([cast(Any, dict(cast(dict[str, Any], call)))])
model.enqueue([cast(Any, dict(cast(dict[str, Any], call)))])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, completed.to_state())
@@ -376,8 +375,8 @@ async def test_schema_1_13_custom_named_shell_skips_exact_completed_replay() ->
return "ok"
call = make_shell_call("legacy-shell", commands=["echo safe"])
model = FakeModel(initial_output=[call])
model.set_next_output([get_text_message("done")])
model = ScriptedModel(steps=[[call]])
model.enqueue([get_text_message("done")])
agent = Agent(
name="agent",
model=model,
@@ -401,7 +400,7 @@ async def test_schema_1_13_custom_named_shell_skips_exact_completed_replay() ->
restored_record = restored._context._tool_invocations["legacy-shell"]
assert restored_record.approval_scope == expected_identity[2]
assert restored_record.fingerprint == expected_identity[3]
model.add_multiple_turn_outputs(
model.extend(
[
[call],
[get_text_message("done again")],
@@ -499,8 +498,8 @@ def _build_scenario(
executed.append(value)
return value
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
@@ -549,14 +548,16 @@ async def test_empty_custom_tool_call_id_fails_before_approval_or_execution() ->
needs_approval=True,
on_approval=approve,
)
model = FakeModel(
initial_output=[
ResponseCustomToolCall(
type="custom_tool_call",
name=tool.name,
call_id="",
input="changed",
)
model = ScriptedModel(
steps=[
[
ResponseCustomToolCall(
type="custom_tool_call",
name=tool.name,
call_id="",
input="changed",
)
]
]
)
agent = Agent(name="agent", model=model, tools=[tool])
@@ -576,15 +577,17 @@ async def test_empty_unresolved_function_call_id_fails_before_error_formatter()
formatter_calls.append(args.tool_name)
return "error"
model = FakeModel(
initial_output=[
ResponseFunctionToolCall(
id="item_0",
type="function_call",
name="missing",
arguments="{}",
call_id="",
)
model = ScriptedModel(
steps=[
[
ResponseFunctionToolCall(
id="item_0",
type="function_call",
name="missing",
arguments="{}",
call_id="",
)
]
]
)
agent = Agent(name="agent", model=model)
@@ -627,8 +630,8 @@ async def test_bound_call_id_with_missing_arguments_fails_before_error_formatter
name="missing",
call_id="shared",
)
model = FakeModel()
model.add_multiple_turn_outputs([[valid_call], [malformed_replacement]])
model = ScriptedModel()
model.extend([[valid_call], [malformed_replacement]])
agent = Agent(name="agent", model=model, tools=[record_value])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
@@ -654,15 +657,17 @@ async def test_empty_handoff_call_id_fails_before_handoff_callback() -> None:
tool_name_override="route",
on_handoff=lambda _context: handoff_calls.append("route"),
)
model = FakeModel(
initial_output=[
ResponseFunctionToolCall(
id="item_0",
type="function_call",
name="route",
arguments="{}",
call_id="",
)
model = ScriptedModel(
steps=[
[
ResponseFunctionToolCall(
id="item_0",
type="function_call",
name="route",
arguments="{}",
call_id="",
)
]
]
)
agent = Agent(name="agent", model=model, handoffs=[route])
@@ -689,8 +694,8 @@ async def test_failed_handoff_hook_does_not_commit_output_or_repeat_callback() -
hook_calls.append(to_agent.name)
raise RuntimeError("handoff hook failed")
model = FakeModel(initial_output=[call])
model.set_next_output([call.model_copy(deep=True)])
model = ScriptedModel(steps=[[call]])
model.enqueue([call.model_copy(deep=True)])
agent = Agent(name="source", model=model, handoffs=[target])
context = RunContextWrapper(context=None)
hooks = FailingHooks()
@@ -763,8 +768,8 @@ async def test_non_approval_identical_siblings_execute_once(
'{"value":"safe"}',
call_id="call_0",
)
model = FakeModel(initial_output=[duplicate, duplicate.model_copy(deep=True)])
model.set_next_output([get_text_message("done")])
model = ScriptedModel(steps=[[duplicate, duplicate.model_copy(deep=True)]])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[record_value])
result = await _run(
@@ -795,8 +800,8 @@ async def test_non_approval_completed_replay_does_not_execute_again(
executed.append(value)
return value
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
[get_function_tool_call("record_value", '{ "value" : "safe" }', call_id="call_0")],
@@ -838,8 +843,8 @@ async def test_non_approval_changed_completed_call_id_fails_before_execution(
executed.append(value)
return value
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
[get_function_tool_call("record_value", '{"value":"changed"}', call_id="call_0")],
@@ -877,28 +882,32 @@ async def test_nested_agent_tool_approval_allows_outer_call_id_collision(
executed.append(value)
return value
inner_model = FakeModel(
initial_output=[
get_function_tool_call(
"inner_sensitive_tool",
'{"value":"safe"}',
call_id="shared",
)
inner_model = ScriptedModel(
steps=[
[
get_function_tool_call(
"inner_sensitive_tool",
'{"value":"safe"}',
call_id="shared",
)
]
]
)
inner_model.set_next_output([get_text_message("inner done")])
inner_model.enqueue([get_text_message("inner done")])
inner_agent = Agent(name="inner", model=inner_model, tools=[inner_sensitive_tool])
outer_model = FakeModel(
initial_output=[
get_function_tool_call(
"nested_agent",
'{"input":"hello"}',
call_id="shared",
)
outer_model = ScriptedModel(
steps=[
[
get_function_tool_call(
"nested_agent",
'{"input":"hello"}',
call_id="shared",
)
]
]
)
outer_model.set_next_output([get_text_message("outer done")])
outer_model.enqueue([get_text_message("outer done")])
outer_agent = Agent(
name="outer",
model=outer_model,
@@ -936,8 +945,8 @@ async def test_parent_approval_does_not_authorize_independent_nested_run(
executed.append(value)
return value
inner_model = FakeModel()
inner_model.add_multiple_turn_outputs(
inner_model = ScriptedModel()
inner_model.extend(
[
[get_function_tool_call("sensitive", '{"value":"same"}', call_id="shared")],
[get_text_message("inner done")],
@@ -945,8 +954,8 @@ async def test_parent_approval_does_not_authorize_independent_nested_run(
)
inner_agent = Agent(name="inner", model=inner_model, tools=[sensitive])
outer_model = FakeModel()
outer_model.add_multiple_turn_outputs(
outer_model = ScriptedModel()
outer_model.extend(
[
[get_function_tool_call("sensitive", '{"value":"same"}', call_id="shared")],
[
@@ -1016,13 +1025,15 @@ async def test_streamed_validated_items_precede_llm_end_hook_failure() -> None:
executed.append(value)
return value
model = FakeModel(
initial_output=[
get_function_tool_call(
"record_value",
'{"value":"safe"}',
call_id="call_0",
)
model = ScriptedModel(
steps=[
[
get_function_tool_call(
"record_value",
'{"value":"safe"}',
call_id="call_0",
)
]
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
@@ -1048,8 +1059,8 @@ async def test_non_approval_failed_tool_body_does_not_reexecute() -> None:
raise RuntimeError("failed after side effect")
call = get_function_tool_call("perform_side_effect", "{}", call_id="call_0")
model = FakeModel()
model.add_multiple_turn_outputs([[call], [call.model_copy(deep=True)]])
model = ScriptedModel()
model.extend([[call], [call.model_copy(deep=True)]])
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
context = RunContextWrapper(context=None)
@@ -1082,8 +1093,8 @@ async def test_non_approval_custom_tool_identical_siblings_execute_once() -> Non
call_id="call_0",
input="safe",
)
model = FakeModel(initial_output=[duplicate, duplicate.model_copy(deep=True)])
model.set_next_output([get_text_message("done")])
model = ScriptedModel(steps=[[duplicate, duplicate.model_copy(deep=True)]])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[tool])
result = await Runner.run(agent, "edit text")
@@ -1107,10 +1118,12 @@ async def test_changed_same_id_siblings_fail_before_approval_callback(
async def record_value(value: str) -> str:
return value
model = FakeModel(
initial_output=[
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
model = ScriptedModel(
steps=[
[
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
]
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
@@ -1140,7 +1153,9 @@ async def test_response_processing_error_still_invokes_llm_end(
self.llm_end_calls += 1
hooks = CountingHooks()
model = FakeModel(initial_output=[make_shell_call("call_0", commands=["echo safe"])])
output: list[TResponseOutputItem] = [make_shell_call("call_0", commands=["echo safe"])]
step = get_exact_output_stream_step(output) if mode == "streamed" else output
model = ScriptedModel(steps=[step])
agent = Agent(name="agent", model=model)
with pytest.raises(ModelBehaviorError, match="without a shell tool"):
@@ -1171,19 +1186,21 @@ async def test_processing_error_with_changed_call_id_still_suppresses_llm_end(
async def record_value(value: str) -> str:
return value
model = FakeModel()
model.add_multiple_turn_outputs(
outputs: list[list[TResponseOutputItem]] = [
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="shared")],
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="shared")],
[
get_function_tool_call(
"record_value",
'{"value":"changed"}',
call_id="shared",
),
make_shell_call("shell_0", commands=["echo safe"]),
],
]
get_function_tool_call(
"record_value",
'{"value":"changed"}',
call_id="shared",
),
make_shell_call("shell_0", commands=["echo safe"]),
],
]
model = ScriptedModel(
[get_exact_output_stream_step(output) for output in outputs]
if mode == "streamed"
else outputs
)
hooks = CountingHooks()
agent = Agent(name="agent", model=model, tools=[record_value])
@@ -1223,7 +1240,9 @@ async def test_processing_error_with_repeated_uncanonical_id_suppresses_llm_end(
call_id="shared",
)
hooks = CountingHooks()
model = FakeModel(initial_output=[first_call, second_call])
output: list[TResponseOutputItem] = [first_call, second_call]
step = get_exact_output_stream_step(output) if mode == "streamed" else output
model = ScriptedModel(steps=[step])
agent = Agent(name="agent", model=model)
with pytest.raises(ModelBehaviorError, match="one response"):
@@ -1244,10 +1263,12 @@ async def test_changed_same_id_siblings_fail_before_non_approval_execution(
executed.append(value)
return value
model = FakeModel(
initial_output=[
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
model = ScriptedModel(
steps=[
[
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
]
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
@@ -1295,9 +1316,11 @@ async def test_changed_computer_safety_checks_fail_before_same_response_effects(
],
}
)
output: list[TResponseOutputItem] = [first_call, changed_call]
step = get_exact_output_stream_step(output) if mode == "streamed" else output
agent = Agent(
name="computer-agent",
model=FakeModel(initial_output=[first_call, changed_call]),
model=ScriptedModel(steps=[step]),
tools=[tool],
)
@@ -1345,8 +1368,12 @@ async def test_changed_computer_safety_checks_fail_before_completed_replay_effec
],
}
)
model = FakeModel()
model.add_multiple_turn_outputs([[first_call], [changed_call]])
outputs: list[list[TResponseOutputItem]] = [[first_call], [changed_call]]
model = ScriptedModel(
[get_exact_output_stream_step(output) for output in outputs]
if mode == "streamed"
else outputs
)
agent = Agent(name="computer-agent", model=model, tools=[tool])
with pytest.raises(ModelBehaviorError, match="completed tool call ID"):
@@ -1389,8 +1416,8 @@ async def test_computer_hook_failure_does_not_repeat_side_effect() -> None:
pending_safety_checks=[],
status="completed",
)
model = FakeModel()
model.add_multiple_turn_outputs([[call], [call.model_copy(deep=True)]])
model = ScriptedModel()
model.extend([[call], [call.model_copy(deep=True)]])
agent = Agent(name="computer-agent", model=model, tools=[tool])
context = RunContextWrapper(context=None)
hooks = FailOnceHooks()
@@ -1413,8 +1440,8 @@ async def test_exact_replay_drops_tied_reasoning_item() -> None:
executed.append(value)
return value
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
[
@@ -1437,7 +1464,7 @@ async def test_exact_replay_drops_tied_reasoning_item() -> None:
assert resumed.final_output == "done"
assert executed == ["safe"]
model_input = model.last_turn_args["input"]
model_input = model.calls[-1].input
assert isinstance(model_input, list)
assert not any(item.get("id") == "rs_replay" for item in model_input)
assert (
@@ -1458,8 +1485,8 @@ async def test_streamed_exact_replay_does_not_emit_tied_reasoning_item() -> None
executed.append(value)
return value
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
[
@@ -1539,9 +1566,7 @@ async def test_failed_output_guardrail_does_not_reexecute_approved_call() -> Non
executed.append("ran")
return "sensitive"
model = FakeModel(
initial_output=[get_function_tool_call("record_value", "{}", call_id="call_0")]
)
model = ScriptedModel(steps=[[get_function_tool_call("record_value", "{}", call_id="call_0")]])
agent = Agent(name="agent", model=model, tools=[record_value])
first = await Runner.run(agent, "record a value")
state = first.to_state()
@@ -1566,8 +1591,8 @@ async def test_failed_approved_tool_body_does_not_reexecute() -> None:
attempts.append("ran")
raise RuntimeError("failed after side effect")
model = FakeModel(
initial_output=[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]
model = ScriptedModel(
steps=[[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]]
)
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
first = await Runner.run(agent, "run it")
@@ -1596,8 +1621,8 @@ async def test_cancelled_approved_tool_body_does_not_reexecute() -> None:
await keep_running.wait()
return "done"
model = FakeModel(
initial_output=[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]
model = ScriptedModel(
steps=[[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]]
)
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
first = await Runner.run(agent, "run it")
@@ -1621,7 +1646,7 @@ async def test_failed_approved_agent_tool_start_does_not_reexecute() -> None:
hook_calls: list[str] = []
inner_agent = Agent(
name="inner",
model=FakeModel(initial_output=[get_text_message("inner done")]),
model=ScriptedModel(steps=[[get_text_message("inner done")]]),
)
agent_tool = inner_agent.as_tool(
tool_name="delegate",
@@ -1640,8 +1665,8 @@ async def test_failed_approved_agent_tool_start_does_not_reexecute() -> None:
hook_calls.append("ran")
raise RuntimeError("failed after side effect")
outer_model = FakeModel(
initial_output=[get_function_tool_call("delegate", '{"input":"hi"}', call_id="call_0")]
outer_model = ScriptedModel(
steps=[[get_function_tool_call("delegate", '{"input":"hi"}', call_id="call_0")]]
)
outer_agent = Agent(name="outer", model=outer_model, tools=[agent_tool])
first = await Runner.run(outer_agent, "delegate")
@@ -1690,13 +1715,15 @@ async def test_failed_parallel_tool_end_hook_checkpoints_outputs_in_model_order(
self.failed = True
raise RuntimeError("second end hook failed")
model = FakeModel(
initial_output=[
get_function_tool_call("first_tool", "{}", call_id="call_first"),
get_function_tool_call("second_tool", "{}", call_id="call_second"),
model = ScriptedModel(
steps=[
[
get_function_tool_call("first_tool", "{}", call_id="call_first"),
get_function_tool_call("second_tool", "{}", call_id="call_second"),
]
]
)
model.set_next_output([get_text_message("done")])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[first_tool, second_tool])
hooks = FailSecondHookOnce()
@@ -1712,7 +1739,7 @@ async def test_failed_parallel_tool_end_hook_checkpoints_outputs_in_model_order(
assert resumed.final_output == "done"
assert sorted(executed) == ["first", "second"]
model_input = model.last_turn_args["input"]
model_input = model.calls[-1].input
assert isinstance(model_input, list)
output_call_ids = [
item["call_id"]
@@ -1739,8 +1766,8 @@ async def test_identical_approval_bound_siblings_execute_once(
json.dumps({"value": "safe"}),
call_id="call-duplicate",
)
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[[duplicated_call, duplicated_call.model_copy(deep=True)], [get_text_message("done")]]
)
run_config = RunConfig(model_provider=_ScriptedProvider(model))
@@ -1829,13 +1856,15 @@ async def test_serialized_sticky_identical_siblings_execute_once() -> None:
'{"value":1}',
call_id="call_0",
)
model = FakeModel(
initial_output=[
duplicate,
duplicate.model_copy(deep=True),
model = ScriptedModel(
steps=[
[
duplicate,
duplicate.model_copy(deep=True),
]
]
)
model.set_next_output([get_text_message("done")])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[record_value])
first = await Runner.run(agent, "record a value")
@@ -2061,8 +2090,8 @@ async def test_changed_tool_under_approved_call_id_fails_before_second_tool_star
executed.append(f"second:{value}")
return value
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
@@ -2116,8 +2145,8 @@ async def test_approved_call_id_reused_for_another_invocation_type_fails_before_
on_invoke_tool=invoke_custom,
format={"type": "text"},
)
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
@@ -2172,8 +2201,8 @@ async def test_changed_missing_tool_under_approved_call_id_fails_before_sibling_
executed.append("sibling")
return "sibling"
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
@@ -2224,8 +2253,8 @@ def _build_serialized_replay_scenario(
executed.append(f"gate:{value}")
return value
model = FakeModel()
model.add_multiple_turn_outputs(
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
@@ -2316,8 +2345,8 @@ async def test_serialized_completed_approval_skips_exact_replay(
provider = run_config.model_provider
assert isinstance(provider, _ScriptedProvider)
model = provider.model
assert isinstance(model, FakeModel)
model_input = model.last_turn_args["input"]
assert isinstance(model, ScriptedModel)
model_input = model.calls[-1].input
assert isinstance(model_input, list)
for call_id in ("call_0", "call_1"):
calls = [
@@ -2368,16 +2397,18 @@ async def test_legacy_schema_historical_sticky_call_id_is_not_reexecuted(
call_id="call_0",
),
)
model = FakeModel(
initial_output=[
get_function_tool_call(
"record_value",
json.dumps({"value": replay_value}),
call_id="call_0",
)
model = ScriptedModel(
steps=[
[
get_function_tool_call(
"record_value",
json.dumps({"value": replay_value}),
call_id="call_0",
)
]
]
)
model.set_next_output([get_text_message("done")])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[record_value])
context: RunContextWrapper[Any] = RunContextWrapper(context=None)
context.approve_tool(
@@ -2738,7 +2769,7 @@ async def test_unbindable_mcp_callback_request_requires_manual_reapproval() -> N
)
agent = Agent(
name="mcp-approval-agent",
model=FakeModel(initial_output=[request]),
model=ScriptedModel(steps=[[request]]),
tools=[mcp_tool],
)
@@ -2746,7 +2777,8 @@ async def test_unbindable_mcp_callback_request_requires_manual_reapproval() -> N
assert callback_calls == 0
assert len(result.interruptions) == 1
assert result.interruptions[0].raw_item is request
assert result.interruptions[0].raw_item == request
assert result.interruptions[0].raw_item is not request
@pytest.mark.parametrize("always_approve", [False, True], ids=["per-call", "sticky"])
@@ -2828,10 +2860,8 @@ async def test_runner_omits_completed_mcp_approval_request_replay(
server_label="test_server",
arguments='{"limit":1,"query":"safe"}',
)
model = FakeModel()
model.add_multiple_turn_outputs(
[[first_request], [replayed_request], [get_text_message("done")]]
)
model = ScriptedModel()
model.extend([[first_request], [replayed_request], [get_text_message("done")]])
agent = Agent(name="mcp-approval-agent", model=model, tools=[mcp_tool])
first = await Runner.run(agent, "lookup")
@@ -2845,7 +2875,7 @@ async def test_runner_omits_completed_mcp_approval_request_replay(
assert result.final_output == "done"
assert callback_calls == int(with_callback)
model_input = model.last_turn_args["input"]
model_input = model.calls[-1].input
assert isinstance(model_input, list)
replay_items = [
item

Some files were not shown because too many files have changed in this diff Show More