feat: add scripted model test utilities (#4362)
This commit is contained in:
@@ -188,6 +188,8 @@ The OpenAI Agents Python repository provides the Python Agents SDK, examples, an
|
||||
|
||||
Before submitting changes, ensure relevant checks pass and extend tests when you touch code.
|
||||
|
||||
For provider-neutral agent workflow tests, prefer `ScriptedModel` from `agents.testing` over adding a new mock or fake `Model`. Prefer `ScriptedRealtimeModel` from `agents.realtime.testing` for Realtime session tests, the scripted utilities from `agents.voice.testing` for Voice pipeline tests, and `scripted_sandbox_session()` from `agents.testing` for deterministic Sandbox session calls. Keep a specialized test double only when the test specifically requires provider-wire conversion, malformed streams, controlled suspension or concurrency, or an exact cancellation or lifecycle boundary that the scripted utilities cannot preserve; document that boundary in the test.
|
||||
|
||||
Before adding or changing async, retry, timeout, subprocess, PTY, warning, or xdist-sensitive tests, read [Performance and determinism](tests/README.md#performance-and-determinism) and preserve the applicable behavioral and lifecycle coverage while optimizing execution.
|
||||
|
||||
When `$code-change-verification` applies, run it to execute the required verification stack from the repository root. Rerun the full stack after applying fixes.
|
||||
|
||||
@@ -1712,7 +1712,7 @@ async def validate_historical_resume_behavior(
|
||||
|
||||
from agents import Agent, Runner, RunState, function_tool
|
||||
from agents.items import ToolCallOutputItem, TResponseOutputItem
|
||||
from integration_tests._fake_model import QueuedFakeModel
|
||||
from agents.testing import ModelStep, ScriptedModel
|
||||
|
||||
invocation_count = 0
|
||||
if feature == "canonical_invocation_identity":
|
||||
@@ -1771,7 +1771,9 @@ async def validate_historical_resume_behavior(
|
||||
],
|
||||
)
|
||||
model_turns.append([final_message])
|
||||
model = QueuedFakeModel(model_turns)
|
||||
model = ScriptedModel(
|
||||
[ModelStep(output=turn, response_id="queued-fake-response") for turn in model_turns]
|
||||
)
|
||||
agent = Agent(name="compat-agent", model=model, tools=[tool])
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
restored = await RunState.from_json(agent, payload)
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from copy import deepcopy
|
||||
from typing import Any, cast
|
||||
|
||||
from openai.types.responses.response_prompt_param import ResponsePromptParam
|
||||
|
||||
from agents.agent_output import AgentOutputSchemaBase
|
||||
from agents.handoffs import Handoff
|
||||
from agents.items import (
|
||||
ModelResponse,
|
||||
TResponseInputItem,
|
||||
TResponseOutputItem,
|
||||
TResponseStreamEvent,
|
||||
)
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import Model, ModelTracing
|
||||
from agents.tool import Tool
|
||||
from agents.usage import Usage
|
||||
|
||||
|
||||
class QueuedFakeModel(Model):
|
||||
"""Deterministic non-streaming model for installed-distribution contracts."""
|
||||
|
||||
def __init__(self, turns: Sequence[Sequence[TResponseOutputItem]]) -> None:
|
||||
self._turns = [list(turn) for turn in turns]
|
||||
self.requests: list[dict[str, Any]] = []
|
||||
|
||||
def _record_request(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: ResponsePromptParam | None,
|
||||
) -> None:
|
||||
self.requests.append(
|
||||
{
|
||||
"system_instructions": system_instructions,
|
||||
"input": deepcopy(input),
|
||||
"model_settings": model_settings,
|
||||
"tools": list(tools),
|
||||
"output_schema": output_schema,
|
||||
"handoffs": list(handoffs),
|
||||
"tracing": tracing,
|
||||
"previous_response_id": previous_response_id,
|
||||
"conversation_id": conversation_id,
|
||||
"prompt": deepcopy(prompt),
|
||||
}
|
||||
)
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: ResponsePromptParam | None,
|
||||
) -> ModelResponse:
|
||||
self._record_request(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
tracing,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
prompt,
|
||||
)
|
||||
if not self._turns:
|
||||
raise AssertionError("QueuedFakeModel received an unexpected model request")
|
||||
return ModelResponse(
|
||||
output=self._turns.pop(0),
|
||||
usage=Usage(requests=1),
|
||||
response_id="queued-fake-response",
|
||||
)
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: ResponsePromptParam | None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
self._record_request(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
tracing,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
prompt,
|
||||
)
|
||||
if False:
|
||||
yield cast(TResponseStreamEvent, None)
|
||||
raise AssertionError("QueuedFakeModel does not support streaming")
|
||||
@@ -28,8 +28,8 @@ from agents.sandbox.session import (
|
||||
SandboxSessionEvent,
|
||||
)
|
||||
from agents.sandbox.snapshot import NoopSnapshotSpec
|
||||
from agents.testing import ModelStep, ScriptedModel, UnexpectedModelCall
|
||||
from integration_tests._contract_support import _redaction_observables
|
||||
from integration_tests._fake_model import QueuedFakeModel
|
||||
from integration_tests.conftest import skip_or_fail
|
||||
|
||||
pytestmark = pytest.mark.security
|
||||
@@ -204,7 +204,9 @@ async def test_runner_owned_local_sandbox_cannot_inspect_trusted_client_credenti
|
||||
turns: list[list[TResponseOutputItem]] = [[tool_call]]
|
||||
if not fail_after_inspection:
|
||||
turns.append([final_message])
|
||||
model = QueuedFakeModel(turns)
|
||||
model = ScriptedModel(
|
||||
[ModelStep(output=turn, response_id="queued-fake-response") for turn in turns]
|
||||
)
|
||||
client = _CredentialOwningDockerClient(trusted_credential=sentinel)
|
||||
nested_mount_source = tmp_path / "nested-mount-probe"
|
||||
nested_mount_source.mkdir()
|
||||
@@ -234,8 +236,8 @@ async def test_runner_owned_local_sandbox_cannot_inspect_trusted_client_credenti
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
if fail_after_inspection:
|
||||
with pytest.raises(
|
||||
AssertionError,
|
||||
match="QueuedFakeModel received an unexpected model request",
|
||||
UnexpectedModelCall,
|
||||
match="no scripted steps remain",
|
||||
) as exc_info:
|
||||
await Runner.run(
|
||||
agent,
|
||||
@@ -277,24 +279,12 @@ async def test_runner_owned_local_sandbox_cannot_inspect_trusted_client_credenti
|
||||
pass
|
||||
docker_client.close()
|
||||
|
||||
expected_model_request_fields = {
|
||||
"system_instructions",
|
||||
"input",
|
||||
"model_settings",
|
||||
"tools",
|
||||
"output_schema",
|
||||
"handoffs",
|
||||
"tracing",
|
||||
"previous_response_id",
|
||||
"conversation_id",
|
||||
"prompt",
|
||||
}
|
||||
assert model.requests
|
||||
assert all(set(request) == expected_model_request_fields for request in model.requests)
|
||||
model_requests = repr(model.requests)
|
||||
assert model.calls
|
||||
assert all(call.streamed is False for call in model.calls)
|
||||
model_requests = repr(model.calls)
|
||||
model_visible_tool_outputs: list[object] = []
|
||||
for request in model.requests:
|
||||
model_input = request["input"]
|
||||
for call in model.calls:
|
||||
model_input = call.input
|
||||
if not isinstance(model_input, list):
|
||||
continue
|
||||
model_visible_tool_outputs.extend(
|
||||
|
||||
@@ -217,22 +217,11 @@ async def test_voice_pipeline_surfaces_tts_failures_without_hanging(
|
||||
from agents.voice import (
|
||||
AudioInput,
|
||||
SingleAgentVoiceWorkflow,
|
||||
TTSModel,
|
||||
TTSModelSettings,
|
||||
VoicePipeline,
|
||||
VoiceStreamEventLifecycle,
|
||||
)
|
||||
from agents.voice.events import VoiceStreamEventError
|
||||
|
||||
class FailingTTSModel(TTSModel):
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return "failing-packaged-tts"
|
||||
|
||||
async def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]:
|
||||
del text, settings
|
||||
raise RuntimeError("Packaged TTS synthesis failed.")
|
||||
yield b"" # pragma: no cover
|
||||
from agents.voice.testing import ScriptedTTSModel
|
||||
|
||||
agent: Agent[Any] = Agent(
|
||||
name="Packaged failing voice workflow agent",
|
||||
@@ -242,7 +231,10 @@ async def test_voice_pipeline_surfaces_tts_failures_without_hanging(
|
||||
)
|
||||
pipeline = VoicePipeline(
|
||||
workflow=SingleAgentVoiceWorkflow(agent),
|
||||
tts_model=FailingTTSModel(),
|
||||
tts_model=ScriptedTTSModel(
|
||||
[RuntimeError("Packaged TTS synthesis failed.")],
|
||||
model_name="failing-packaged-tts",
|
||||
),
|
||||
config={"tracing_disabled": True},
|
||||
)
|
||||
audio = np.frombuffer(integration_pcm_audio, dtype=np.int16).copy()
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
"""Deterministic Realtime model transport for session tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from collections import deque
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TypeAlias, cast
|
||||
|
||||
from typing_extensions import Required, TypedDict
|
||||
|
||||
from ..models._trace import sanitize_url_for_trace
|
||||
from .config import RealtimeSessionModelSettings
|
||||
from .model import (
|
||||
RealtimeModel,
|
||||
RealtimeModelConfig,
|
||||
RealtimeModelListener,
|
||||
RealtimePlaybackTracker,
|
||||
)
|
||||
from .model_events import RealtimeModelErrorEvent, RealtimeModelEvent, RealtimeModelExceptionEvent
|
||||
from .model_inputs import RealtimeModelSendEvent, RealtimeModelSendSessionUpdate
|
||||
|
||||
|
||||
class RealtimeScriptError(Exception):
|
||||
"""Base exception for an invalid or incompletely consumed Realtime script."""
|
||||
|
||||
|
||||
class UnexpectedRealtimeSend(RealtimeScriptError):
|
||||
"""Raised when an outbound event does not match the next scripted step."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
actual: RealtimeModelSendEvent,
|
||||
expected: RealtimeSendMatcher | None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.actual = actual
|
||||
self.expected = expected
|
||||
|
||||
|
||||
class UnconsumedRealtimeSteps(RealtimeScriptError):
|
||||
"""Raised when a test finishes before consuming every configured send step."""
|
||||
|
||||
def __init__(self, message: str, *, remaining_steps: int) -> None:
|
||||
super().__init__(message)
|
||||
self.remaining_steps = remaining_steps
|
||||
|
||||
|
||||
RealtimeSendMatcher: TypeAlias = (
|
||||
RealtimeModelSendEvent | type[RealtimeModelSendEvent] | Callable[[RealtimeModelSendEvent], bool]
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RealtimeStep:
|
||||
"""One expected outbound event and the normalized inbound events it triggers."""
|
||||
|
||||
expect: RealtimeSendMatcher
|
||||
emit: Sequence[RealtimeModelEvent] = field(default_factory=tuple)
|
||||
error: Exception | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
frozen_emit = tuple(self.emit)
|
||||
if frozen_emit and self.error is not None:
|
||||
raise ValueError("A RealtimeStep cannot define both emit events and an error.")
|
||||
object.__setattr__(self, "emit", frozen_emit)
|
||||
|
||||
|
||||
class RealtimeConnectCall(TypedDict, total=False):
|
||||
"""A credential-free snapshot of one Realtime connection call."""
|
||||
|
||||
api_key_provided: Required[bool]
|
||||
headers_provided: Required[bool]
|
||||
url: str
|
||||
initial_model_settings: RealtimeSessionModelSettings
|
||||
playback_tracker: RealtimePlaybackTracker
|
||||
call_id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RealtimeDeliveryResult:
|
||||
done: asyncio.Future[BaseException | None]
|
||||
pending_deliveries: int = 1
|
||||
error: BaseException | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _QueuedRealtimeDelivery:
|
||||
events: Sequence[RealtimeModelEvent]
|
||||
error: Exception | None
|
||||
result: _RealtimeDeliveryResult
|
||||
committed_close_calls: int
|
||||
|
||||
|
||||
class ScriptedRealtimeModel(RealtimeModel):
|
||||
"""An in-memory, listener-based Realtime transport with deterministic send steps."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
steps: Iterable[RealtimeStep] = (),
|
||||
*,
|
||||
connect_events: Iterable[RealtimeModelEvent] = (),
|
||||
connect_error: Exception | None = None,
|
||||
close_error: Exception | None = None,
|
||||
strict: bool = True,
|
||||
) -> None:
|
||||
connect_event_values = tuple(connect_events)
|
||||
if connect_event_values and connect_error is not None:
|
||||
raise ValueError(
|
||||
"A ScriptedRealtimeModel cannot define both connect events and a connect error."
|
||||
)
|
||||
self._steps = [_snapshot_realtime_step(step) for step in steps]
|
||||
self._connect_events = tuple(_snapshot_model_event(event) for event in connect_event_values)
|
||||
self._connect_error = connect_error
|
||||
self._close_error = close_error
|
||||
self._strict = strict
|
||||
self._listeners: list[RealtimeModelListener] = []
|
||||
self._send_lock = asyncio.Lock()
|
||||
self._delivery_queue: deque[_QueuedRealtimeDelivery] = deque()
|
||||
self._delivery_worker: asyncio.Task[None] | None = None
|
||||
self._active_delivery_result: _RealtimeDeliveryResult | None = None
|
||||
self._connect_calls: list[RealtimeConnectCall] = []
|
||||
self._sent_events: list[RealtimeModelSendEvent] = []
|
||||
self.connected = False
|
||||
self.closed = False
|
||||
self.close_calls = 0
|
||||
|
||||
@property
|
||||
def listeners(self) -> tuple[RealtimeModelListener, ...]:
|
||||
"""Return the currently registered listeners."""
|
||||
return tuple(self._listeners)
|
||||
|
||||
@property
|
||||
def connect_calls(self) -> tuple[RealtimeConnectCall, ...]:
|
||||
"""Return detached snapshots of recorded connection calls."""
|
||||
return tuple(_clone_connect_call(call) for call in self._connect_calls)
|
||||
|
||||
@property
|
||||
def sent_events(self) -> tuple[RealtimeModelSendEvent, ...]:
|
||||
"""Return detached snapshots of recorded outbound events."""
|
||||
return tuple(self._snapshot_send_event(event) for event in self._sent_events)
|
||||
|
||||
@property
|
||||
def remaining_steps(self) -> int:
|
||||
"""Return the number of expected outbound sends that remain."""
|
||||
return len(self._steps)
|
||||
|
||||
async def connect(self, options: RealtimeModelConfig) -> None:
|
||||
if self.connected or self._delivery_worker is not None:
|
||||
raise AssertionError("Already connected")
|
||||
self._connect_calls.append(_snapshot_connect_call(options))
|
||||
if self._connect_error is not None:
|
||||
raise self._connect_error
|
||||
self.connected = True
|
||||
self.closed = False
|
||||
try:
|
||||
async with self._send_lock:
|
||||
self._ensure_emit_allowed()
|
||||
event_snapshots = tuple(
|
||||
_snapshot_model_event(event) for event in self._connect_events
|
||||
)
|
||||
result, reentrant = self._queue_delivery_locked(events=event_snapshots)
|
||||
await self._finish_queued_delivery(result, reentrant)
|
||||
except BaseException:
|
||||
self.connected = False
|
||||
self.closed = True
|
||||
raise
|
||||
|
||||
def add_listener(self, listener: RealtimeModelListener) -> None:
|
||||
if listener not in self._listeners:
|
||||
self._listeners.append(listener)
|
||||
|
||||
def remove_listener(self, listener: RealtimeModelListener) -> None:
|
||||
if listener in self._listeners:
|
||||
self._listeners.remove(listener)
|
||||
|
||||
async def send_event(self, event: RealtimeModelSendEvent) -> None:
|
||||
result, reentrant = await self._commit_send(event)
|
||||
await self._finish_queued_delivery(result, reentrant)
|
||||
|
||||
async def send_event_if(
|
||||
self,
|
||||
event: RealtimeModelSendEvent,
|
||||
send_if: Callable[[], bool],
|
||||
) -> bool:
|
||||
async with self._send_lock:
|
||||
self._ensure_sendable()
|
||||
if not send_if():
|
||||
return False
|
||||
result, reentrant = self._commit_send_locked(event)
|
||||
await self._finish_queued_delivery(result, reentrant)
|
||||
return True
|
||||
|
||||
async def emit(self, *events: RealtimeModelEvent) -> None:
|
||||
"""Deliver normalized model events to all current listeners in order."""
|
||||
async with self._send_lock:
|
||||
self._ensure_emit_allowed()
|
||||
event_snapshots = tuple(_snapshot_model_event(event) for event in events)
|
||||
result, reentrant = self._queue_delivery_locked(events=event_snapshots)
|
||||
await self._finish_queued_delivery(result, reentrant)
|
||||
|
||||
async def _broadcast_events(
|
||||
self,
|
||||
events: Sequence[RealtimeModelEvent],
|
||||
*,
|
||||
committed_close_calls: int,
|
||||
) -> None:
|
||||
for event in events:
|
||||
listeners = tuple(self._listeners)
|
||||
for listener in listeners:
|
||||
if self.closed or self.close_calls != committed_close_calls:
|
||||
return
|
||||
await listener.on_event(event)
|
||||
|
||||
async def close(self) -> None:
|
||||
self.close_calls += 1
|
||||
if self.closed:
|
||||
return
|
||||
self.connected = False
|
||||
self.closed = True
|
||||
if self._close_error is not None:
|
||||
raise self._close_error
|
||||
|
||||
def assert_complete(self) -> None:
|
||||
"""Raise when expected outbound send steps remain unconsumed."""
|
||||
if self._steps:
|
||||
raise UnconsumedRealtimeSteps(
|
||||
f"{len(self._steps)} scripted Realtime step(s) were not consumed.",
|
||||
remaining_steps=len(self._steps),
|
||||
)
|
||||
|
||||
async def _commit_send(
|
||||
self, event: RealtimeModelSendEvent
|
||||
) -> tuple[_RealtimeDeliveryResult, bool]:
|
||||
async with self._send_lock:
|
||||
self._ensure_sendable()
|
||||
return self._commit_send_locked(event)
|
||||
|
||||
def _commit_send_locked(
|
||||
self, event: RealtimeModelSendEvent
|
||||
) -> tuple[_RealtimeDeliveryResult, bool]:
|
||||
event_snapshot = self._snapshot_send_event(event)
|
||||
step = self._pop_matching_step(event, actual_snapshot=event_snapshot)
|
||||
self._sent_events.append(event_snapshot)
|
||||
return self._queue_delivery_locked(
|
||||
events=step.emit if step is not None else (),
|
||||
error=step.error if step is not None else None,
|
||||
)
|
||||
|
||||
def _queue_delivery_locked(
|
||||
self,
|
||||
*,
|
||||
events: Sequence[RealtimeModelEvent],
|
||||
error: Exception | None = None,
|
||||
) -> tuple[_RealtimeDeliveryResult, bool]:
|
||||
current_task = asyncio.current_task()
|
||||
if current_task is None:
|
||||
raise RuntimeError("A scripted Realtime send requires an active asyncio task.")
|
||||
active_result = self._active_delivery_result
|
||||
if current_task is self._delivery_worker and active_result is not None:
|
||||
reentrant = True
|
||||
result = active_result
|
||||
result.pending_deliveries += 1
|
||||
else:
|
||||
reentrant = False
|
||||
result = _RealtimeDeliveryResult(done=asyncio.get_running_loop().create_future())
|
||||
self._delivery_queue.append(
|
||||
_QueuedRealtimeDelivery(
|
||||
events=tuple(events),
|
||||
error=error,
|
||||
result=result,
|
||||
committed_close_calls=self.close_calls,
|
||||
)
|
||||
)
|
||||
self._ensure_delivery_worker_locked()
|
||||
return result, reentrant
|
||||
|
||||
async def _finish_queued_delivery(
|
||||
self,
|
||||
result: _RealtimeDeliveryResult,
|
||||
reentrant: bool,
|
||||
) -> None:
|
||||
if reentrant:
|
||||
return
|
||||
error = await asyncio.shield(result.done)
|
||||
if error is not None:
|
||||
raise error
|
||||
|
||||
def _ensure_delivery_worker_locked(self) -> None:
|
||||
if self._delivery_worker is None or self._delivery_worker.done():
|
||||
self._delivery_worker = asyncio.create_task(self._drain_committed_sends())
|
||||
|
||||
async def _drain_committed_sends(self) -> None:
|
||||
while True:
|
||||
async with self._send_lock:
|
||||
if not self._delivery_queue:
|
||||
self._active_delivery_result = None
|
||||
self._delivery_worker = None
|
||||
return
|
||||
delivery = self._delivery_queue.popleft()
|
||||
self._active_delivery_result = delivery.result
|
||||
error = await self._deliver_queued_events(delivery)
|
||||
result = delivery.result
|
||||
if error is not None and result.error is None:
|
||||
result.error = error
|
||||
result.pending_deliveries -= 1
|
||||
if result.pending_deliveries == 0 and not result.done.done():
|
||||
result.done.set_result(result.error)
|
||||
|
||||
async def _deliver_queued_events(
|
||||
self, delivery: _QueuedRealtimeDelivery
|
||||
) -> BaseException | None:
|
||||
try:
|
||||
if delivery.error is not None:
|
||||
raise delivery.error
|
||||
self._ensure_emit_allowed(delivery.committed_close_calls)
|
||||
await self._broadcast_events(
|
||||
delivery.events,
|
||||
committed_close_calls=delivery.committed_close_calls,
|
||||
)
|
||||
except BaseException as error:
|
||||
return error
|
||||
return None
|
||||
|
||||
def _ensure_sendable(self) -> None:
|
||||
if not self.connected or self.closed:
|
||||
raise RealtimeScriptError(
|
||||
"Cannot send an event while the scripted model is disconnected."
|
||||
)
|
||||
|
||||
def _ensure_emit_allowed(
|
||||
self,
|
||||
committed_close_calls: int | None = None,
|
||||
) -> None:
|
||||
if (
|
||||
not self.connected
|
||||
or self.closed
|
||||
or (committed_close_calls is not None and self.close_calls != committed_close_calls)
|
||||
):
|
||||
raise RealtimeScriptError(
|
||||
"Cannot emit events while the scripted model is disconnected."
|
||||
)
|
||||
|
||||
def _pop_matching_step(
|
||||
self,
|
||||
event: RealtimeModelSendEvent,
|
||||
*,
|
||||
actual_snapshot: RealtimeModelSendEvent,
|
||||
) -> RealtimeStep | None:
|
||||
if not self._steps:
|
||||
if not self._strict:
|
||||
return None
|
||||
raise UnexpectedRealtimeSend(
|
||||
"Unexpected Realtime send: no scripted steps remain.",
|
||||
actual=actual_snapshot,
|
||||
expected=None,
|
||||
)
|
||||
step = self._steps[0]
|
||||
if not _matches(step.expect, event):
|
||||
if not self._strict:
|
||||
return None
|
||||
raise UnexpectedRealtimeSend(
|
||||
"Unexpected Realtime send: event did not match the next scripted expectation.",
|
||||
actual=actual_snapshot,
|
||||
expected=_snapshot_realtime_expectation(step.expect),
|
||||
)
|
||||
return self._steps.pop(0)
|
||||
|
||||
@staticmethod
|
||||
def _snapshot_send_event(event: RealtimeModelSendEvent) -> RealtimeModelSendEvent:
|
||||
return _snapshot_send_event(event)
|
||||
|
||||
|
||||
def _snapshot_realtime_step(step: RealtimeStep) -> RealtimeStep:
|
||||
return RealtimeStep(
|
||||
expect=_snapshot_realtime_expectation(step.expect),
|
||||
emit=tuple(_snapshot_model_event(event) for event in step.emit),
|
||||
error=step.error,
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_realtime_expectation(expectation: RealtimeSendMatcher) -> RealtimeSendMatcher:
|
||||
if isinstance(expectation, type) or callable(expectation):
|
||||
return expectation
|
||||
return _snapshot_send_event(expectation)
|
||||
|
||||
|
||||
def _snapshot_send_event(event: RealtimeModelSendEvent) -> RealtimeModelSendEvent:
|
||||
if isinstance(event, RealtimeModelSendSessionUpdate):
|
||||
return RealtimeModelSendSessionUpdate(
|
||||
session_settings=_snapshot_model_settings(event.session_settings)
|
||||
)
|
||||
return copy.deepcopy(event)
|
||||
|
||||
|
||||
def _snapshot_model_event(event: RealtimeModelEvent) -> RealtimeModelEvent:
|
||||
if isinstance(event, RealtimeModelExceptionEvent):
|
||||
return RealtimeModelExceptionEvent(
|
||||
exception=event.exception,
|
||||
context=copy.deepcopy(event.context),
|
||||
)
|
||||
if isinstance(event, RealtimeModelErrorEvent) and isinstance(event.error, Exception):
|
||||
return RealtimeModelErrorEvent(error=event.error)
|
||||
return copy.deepcopy(event)
|
||||
|
||||
|
||||
def _matches(expectation: RealtimeSendMatcher, event: RealtimeModelSendEvent) -> bool:
|
||||
if isinstance(expectation, type):
|
||||
return isinstance(event, expectation)
|
||||
if callable(expectation):
|
||||
return expectation(_snapshot_send_event(event))
|
||||
return event == expectation
|
||||
|
||||
|
||||
def _snapshot_connect_call(options: RealtimeModelConfig) -> RealtimeConnectCall:
|
||||
snapshot: RealtimeConnectCall = {
|
||||
"api_key_provided": "api_key" in options,
|
||||
"headers_provided": "headers" in options,
|
||||
}
|
||||
if "url" in options:
|
||||
snapshot["url"] = sanitize_url_for_trace(options["url"])
|
||||
if "initial_model_settings" in options:
|
||||
snapshot["initial_model_settings"] = _snapshot_model_settings(
|
||||
options["initial_model_settings"]
|
||||
)
|
||||
if "playback_tracker" in options:
|
||||
snapshot["playback_tracker"] = options["playback_tracker"]
|
||||
if "call_id" in options:
|
||||
snapshot["call_id"] = options["call_id"]
|
||||
return snapshot
|
||||
|
||||
|
||||
def _clone_connect_call(call: RealtimeConnectCall) -> RealtimeConnectCall:
|
||||
snapshot: RealtimeConnectCall = {
|
||||
"api_key_provided": call["api_key_provided"],
|
||||
"headers_provided": call["headers_provided"],
|
||||
}
|
||||
if "url" in call:
|
||||
snapshot["url"] = call["url"]
|
||||
if "initial_model_settings" in call:
|
||||
snapshot["initial_model_settings"] = _snapshot_model_settings(
|
||||
call["initial_model_settings"]
|
||||
)
|
||||
if "playback_tracker" in call:
|
||||
snapshot["playback_tracker"] = call["playback_tracker"]
|
||||
if "call_id" in call:
|
||||
snapshot["call_id"] = call["call_id"]
|
||||
return snapshot
|
||||
|
||||
|
||||
def _snapshot_model_settings(
|
||||
settings: RealtimeSessionModelSettings,
|
||||
) -> RealtimeSessionModelSettings:
|
||||
snapshot = cast(
|
||||
RealtimeSessionModelSettings,
|
||||
copy.deepcopy(
|
||||
{key: value for key, value in settings.items() if key not in {"tools", "handoffs"}}
|
||||
),
|
||||
)
|
||||
if "tools" in settings:
|
||||
snapshot["tools"] = list(settings["tools"])
|
||||
if "handoffs" in settings:
|
||||
snapshot["handoffs"] = list(settings["handoffs"])
|
||||
return snapshot
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RealtimeConnectCall",
|
||||
"RealtimeScriptError",
|
||||
"RealtimeStep",
|
||||
"ScriptedRealtimeModel",
|
||||
"UnconsumedRealtimeSteps",
|
||||
"UnexpectedRealtimeSend",
|
||||
]
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Deterministic test doubles for Agents SDK workflows."""
|
||||
|
||||
from .model import (
|
||||
InvalidModelStep,
|
||||
ModelCall,
|
||||
ModelScriptError,
|
||||
ModelStep,
|
||||
ModelStepSpec,
|
||||
ScriptedModel,
|
||||
UnconsumedModelSteps,
|
||||
UnexpectedModelCall,
|
||||
assistant_message,
|
||||
function_call,
|
||||
)
|
||||
from .sandbox import (
|
||||
InvalidSandboxStep,
|
||||
SandboxCall,
|
||||
SandboxCallMatcherError,
|
||||
SandboxScriptError,
|
||||
SandboxStepSpec,
|
||||
UnconsumedSandboxSteps,
|
||||
UnexpectedSandboxCall,
|
||||
scripted_sandbox_session,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"InvalidModelStep",
|
||||
"ModelCall",
|
||||
"ModelScriptError",
|
||||
"ModelStep",
|
||||
"ModelStepSpec",
|
||||
"InvalidSandboxStep",
|
||||
"SandboxCall",
|
||||
"SandboxCallMatcherError",
|
||||
"SandboxScriptError",
|
||||
"SandboxStepSpec",
|
||||
"ScriptedModel",
|
||||
"UnconsumedModelSteps",
|
||||
"UnexpectedModelCall",
|
||||
"UnconsumedSandboxSteps",
|
||||
"UnexpectedSandboxCall",
|
||||
"assistant_message",
|
||||
"function_call",
|
||||
"scripted_sandbox_session",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,570 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
import io
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Literal, TypeAlias, cast, get_args
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from ..editor import ApplyPatchOperation
|
||||
from ..sandbox.apply_patch import PatchFormat
|
||||
from ..sandbox.files import FileEntry
|
||||
from ..sandbox.manifest import Manifest
|
||||
from ..sandbox.session.base_sandbox_session import BaseSandboxSession
|
||||
from ..sandbox.session.pty_types import PtyExecUpdate
|
||||
from ..sandbox.session.sandbox_session_state import SandboxSessionState
|
||||
from ..sandbox.snapshot import NoopSnapshot
|
||||
from ..sandbox.types import ExecResult, User
|
||||
|
||||
SandboxMethod = Literal[
|
||||
"apply_patch",
|
||||
"exec",
|
||||
"ls",
|
||||
"mkdir",
|
||||
"pty_exec_start",
|
||||
"pty_write_stdin",
|
||||
"read",
|
||||
"rm",
|
||||
"write",
|
||||
]
|
||||
SandboxStepReason = Literal["invalid_input", "unknown_method", "invalid_matcher", "invalid_outcome"]
|
||||
|
||||
_SCRIPTABLE_METHODS: frozenset[str] = frozenset(get_args(SandboxMethod))
|
||||
_PTY_METHODS: frozenset[SandboxMethod] = frozenset({"pty_exec_start", "pty_write_stdin"})
|
||||
_UNSUPPORTED_OPTIONAL_METHODS: frozenset[str] = frozenset(
|
||||
{
|
||||
"extract",
|
||||
"hydrate_workspace",
|
||||
"persist_workspace",
|
||||
"resolve_exposed_port",
|
||||
}
|
||||
)
|
||||
|
||||
_HIDDEN_LIFECYCLE_METHODS: frozenset[str] = frozenset({"pty_terminate_all"})
|
||||
|
||||
for _method_name in _SCRIPTABLE_METHODS | _UNSUPPORTED_OPTIONAL_METHODS | _HIDDEN_LIFECYCLE_METHODS:
|
||||
if not hasattr(BaseSandboxSession, _method_name):
|
||||
raise RuntimeError(f"Unknown BaseSandboxSession method: {_method_name}")
|
||||
|
||||
|
||||
class SandboxScriptError(Exception):
|
||||
"""Base exception for an invalid or incompletely consumed sandbox script."""
|
||||
|
||||
|
||||
class InvalidSandboxStep(SandboxScriptError):
|
||||
"""Raised when a sandbox step is invalid at factory construction time."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
reason: SandboxStepReason,
|
||||
input_index: int,
|
||||
method: str | None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
self.input_index = input_index
|
||||
self.method = method
|
||||
|
||||
|
||||
class UnexpectedSandboxCall(SandboxScriptError):
|
||||
"""Raised when a call does not match the next configured sandbox step."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
call: SandboxCall,
|
||||
call_index: int,
|
||||
expected_method: str | None,
|
||||
remaining_steps: int,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.call = call
|
||||
self.call_index = call_index
|
||||
self.actual_method = call.method
|
||||
self.expected_method = expected_method
|
||||
self.remaining_steps = remaining_steps
|
||||
|
||||
|
||||
class SandboxCallMatcherError(SandboxScriptError):
|
||||
"""Raised when a sandbox step matcher rejects its call."""
|
||||
|
||||
def __init__(self, message: str, *, call: SandboxCall, call_index: int) -> None:
|
||||
super().__init__(message)
|
||||
self.call = call
|
||||
self.call_index = call_index
|
||||
self.method = call.method
|
||||
|
||||
|
||||
class UnconsumedSandboxSteps(SandboxScriptError):
|
||||
"""Raised when configured sandbox steps remain unconsumed."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
remaining_steps: int,
|
||||
pending_methods: tuple[str, ...],
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.remaining_steps = remaining_steps
|
||||
self.pending_methods = pending_methods
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SandboxCall:
|
||||
"""A detached invocation-time sandbox call snapshot."""
|
||||
|
||||
call_index: int
|
||||
method: str
|
||||
args: tuple[Any, ...]
|
||||
kwargs: Mapping[str, Any]
|
||||
|
||||
|
||||
SandboxMatcher: TypeAlias = Callable[[SandboxCall], bool | None]
|
||||
SandboxResponder: TypeAlias = Callable[[SandboxCall], Any | Awaitable[Any]]
|
||||
|
||||
|
||||
class SandboxStepSpec(TypedDict, total=False):
|
||||
"""Dictionary form of one FIFO scripted sandbox call."""
|
||||
|
||||
method: SandboxMethod
|
||||
match: SandboxMatcher
|
||||
result: Any
|
||||
responder: SandboxResponder
|
||||
error: Exception
|
||||
|
||||
|
||||
_STEP_FIELDS = frozenset(SandboxStepSpec.__annotations__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SandboxStep:
|
||||
method: SandboxMethod
|
||||
match: SandboxMatcher | None
|
||||
outcome: Literal["result", "responder", "error"]
|
||||
value: Any
|
||||
|
||||
|
||||
def _snapshot_value(value: Any) -> Any:
|
||||
if isinstance(value, io.BytesIO):
|
||||
if value.closed:
|
||||
raise TypeError("Cannot snapshot a closed sandbox byte stream.")
|
||||
byte_snapshot = io.BytesIO(value.getvalue())
|
||||
byte_snapshot.seek(value.tell())
|
||||
return byte_snapshot
|
||||
if isinstance(value, io.StringIO):
|
||||
if value.closed:
|
||||
raise TypeError("Cannot snapshot a closed sandbox text stream.")
|
||||
text_snapshot = io.StringIO(value.getvalue())
|
||||
text_snapshot.seek(value.tell())
|
||||
return text_snapshot
|
||||
if isinstance(value, io.IOBase):
|
||||
raise TypeError("Sandbox stream snapshots support only io.BytesIO and io.StringIO.")
|
||||
if callable(value):
|
||||
return value
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_snapshot_value(item) for item in value)
|
||||
if isinstance(value, list):
|
||||
return [_snapshot_value(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {_snapshot_value(key): _snapshot_value(item) for key, item in value.items()}
|
||||
if isinstance(value, set):
|
||||
return {_snapshot_value(item) for item in value}
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def _snapshot_call(call: SandboxCall) -> SandboxCall:
|
||||
return SandboxCall(
|
||||
call_index=call.call_index,
|
||||
method=call.method,
|
||||
args=tuple(_snapshot_value(call.args)),
|
||||
kwargs=MappingProxyType(
|
||||
{name: _snapshot_value(value) for name, value in call.kwargs.items()}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _invalid_step(
|
||||
*,
|
||||
reason: SandboxStepReason,
|
||||
input_index: int,
|
||||
method: str | None,
|
||||
detail: str,
|
||||
) -> InvalidSandboxStep:
|
||||
return InvalidSandboxStep(
|
||||
f"Scripted sandbox step #{input_index + 1} {detail}.",
|
||||
reason=reason,
|
||||
input_index=input_index,
|
||||
method=method,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_step(input: object, input_index: int) -> _SandboxStep:
|
||||
if not isinstance(input, Mapping):
|
||||
raise _invalid_step(
|
||||
reason="invalid_input",
|
||||
input_index=input_index,
|
||||
method=None,
|
||||
detail="must be a mapping",
|
||||
)
|
||||
|
||||
method = input.get("method")
|
||||
if any(field not in _STEP_FIELDS for field in input):
|
||||
raise _invalid_step(
|
||||
reason="invalid_input",
|
||||
input_index=input_index,
|
||||
method=method if isinstance(method, str) else None,
|
||||
detail="contains unsupported fields",
|
||||
)
|
||||
if not isinstance(method, str) or method not in _SCRIPTABLE_METHODS:
|
||||
raise _invalid_step(
|
||||
reason="unknown_method",
|
||||
input_index=input_index,
|
||||
method=method if isinstance(method, str) else None,
|
||||
detail="has an unknown method",
|
||||
)
|
||||
|
||||
matcher = input.get("match")
|
||||
if isinstance(matcher, io.IOBase):
|
||||
raise _invalid_step(
|
||||
reason="invalid_matcher",
|
||||
input_index=input_index,
|
||||
method=method,
|
||||
detail=f"for {method} must use a non-stream callable matcher",
|
||||
)
|
||||
if matcher is not None and not callable(matcher):
|
||||
raise _invalid_step(
|
||||
reason="invalid_matcher",
|
||||
input_index=input_index,
|
||||
method=method,
|
||||
detail=f"for {method} must use a callable matcher",
|
||||
)
|
||||
|
||||
outcomes = [name for name in ("result", "responder", "error") if name in input]
|
||||
if len(outcomes) != 1:
|
||||
raise _invalid_step(
|
||||
reason="invalid_outcome",
|
||||
input_index=input_index,
|
||||
method=method,
|
||||
detail=f"for {method} must define exactly one outcome",
|
||||
)
|
||||
outcome = cast(Literal["result", "responder", "error"], outcomes[0])
|
||||
value = input[outcome]
|
||||
if outcome == "responder" and isinstance(value, io.IOBase):
|
||||
raise _invalid_step(
|
||||
reason="invalid_outcome",
|
||||
input_index=input_index,
|
||||
method=method,
|
||||
detail=f"for {method} must use a non-stream callable responder",
|
||||
)
|
||||
if outcome == "responder" and not callable(value):
|
||||
raise _invalid_step(
|
||||
reason="invalid_outcome",
|
||||
input_index=input_index,
|
||||
method=method,
|
||||
detail=f"for {method} must use a callable responder",
|
||||
)
|
||||
if outcome == "error" and not isinstance(value, Exception):
|
||||
raise _invalid_step(
|
||||
reason="invalid_outcome",
|
||||
input_index=input_index,
|
||||
method=method,
|
||||
detail=f"for {method} must use an Exception",
|
||||
)
|
||||
if outcome == "result":
|
||||
try:
|
||||
value = _snapshot_value(value)
|
||||
except TypeError as error:
|
||||
raise _invalid_step(
|
||||
reason="invalid_outcome",
|
||||
input_index=input_index,
|
||||
method=method,
|
||||
detail=f"for {method} contains a result that cannot be snapshotted",
|
||||
) from error
|
||||
return _SandboxStep(
|
||||
method=cast(SandboxMethod, method),
|
||||
match=cast(SandboxMatcher | None, matcher),
|
||||
outcome=outcome,
|
||||
value=value,
|
||||
)
|
||||
|
||||
|
||||
class _ScriptedSandboxSession(BaseSandboxSession):
|
||||
def __init__(self, steps: Sequence[_SandboxStep], *, manifest: Manifest | None) -> None:
|
||||
self.state = SandboxSessionState(
|
||||
type="scripted",
|
||||
snapshot=NoopSnapshot(id="scripted"),
|
||||
manifest=copy.deepcopy(manifest) if manifest is not None else Manifest(),
|
||||
)
|
||||
self._steps = list(steps)
|
||||
configured_methods = {step.method for step in steps}
|
||||
if configured_methods & _PTY_METHODS:
|
||||
configured_methods.update(_PTY_METHODS)
|
||||
self._configured_methods = frozenset(configured_methods)
|
||||
self._calls: list[SandboxCall] = []
|
||||
self._running = False
|
||||
|
||||
def __getattribute__(self, name: str) -> Any:
|
||||
if name in _SCRIPTABLE_METHODS:
|
||||
configured = object.__getattribute__(self, "_configured_methods")
|
||||
if name not in configured:
|
||||
raise AttributeError(name)
|
||||
if name in _UNSUPPORTED_OPTIONAL_METHODS | _HIDDEN_LIFECYCLE_METHODS:
|
||||
raise AttributeError(name)
|
||||
return super().__getattribute__(name)
|
||||
|
||||
def __dir__(self) -> list[str]:
|
||||
configured = self._configured_methods
|
||||
return sorted(
|
||||
name
|
||||
for name in super().__dir__()
|
||||
if name not in _UNSUPPORTED_OPTIONAL_METHODS | _HIDDEN_LIFECYCLE_METHODS
|
||||
and (name not in _SCRIPTABLE_METHODS or name in configured)
|
||||
)
|
||||
|
||||
@property
|
||||
def calls(self) -> tuple[SandboxCall, ...]:
|
||||
"""Return detached call-history snapshots in invocation order."""
|
||||
return tuple(_snapshot_call(call) for call in self._calls)
|
||||
|
||||
@property
|
||||
def remaining_steps(self) -> int:
|
||||
"""Return the number of configured calls that remain."""
|
||||
return len(self._steps)
|
||||
|
||||
def assert_complete(self) -> None:
|
||||
"""Raise when configured sandbox calls remain unconsumed."""
|
||||
if self._steps:
|
||||
pending_methods = tuple(step.method for step in self._steps)
|
||||
raise UnconsumedSandboxSteps(
|
||||
f"Scripted sandbox session has {len(self._steps)} unconsumed step(s).",
|
||||
remaining_steps=len(self._steps),
|
||||
pending_methods=pending_methods,
|
||||
)
|
||||
|
||||
async def _invoke(
|
||||
self, method: SandboxMethod, args: tuple[Any, ...], kwargs: dict[str, Any]
|
||||
) -> Any:
|
||||
call = SandboxCall(
|
||||
call_index=len(self._calls),
|
||||
method=method,
|
||||
args=tuple(_snapshot_value(args)),
|
||||
kwargs=MappingProxyType(
|
||||
{name: _snapshot_value(value) for name, value in kwargs.items()}
|
||||
),
|
||||
)
|
||||
call_index = call.call_index
|
||||
self._calls.append(call)
|
||||
|
||||
if not self._steps:
|
||||
raise UnexpectedSandboxCall(
|
||||
f"Unexpected sandbox {method} call #{call_index + 1}: no scripted steps remain.",
|
||||
call=_snapshot_call(call),
|
||||
call_index=call_index,
|
||||
expected_method=None,
|
||||
remaining_steps=0,
|
||||
)
|
||||
|
||||
step = self._steps[0]
|
||||
if step.method != method:
|
||||
raise UnexpectedSandboxCall(
|
||||
f"Unexpected sandbox {method} call #{call_index + 1}; expected {step.method}.",
|
||||
call=_snapshot_call(call),
|
||||
call_index=call_index,
|
||||
expected_method=step.method,
|
||||
remaining_steps=len(self._steps),
|
||||
)
|
||||
|
||||
matcher_call = _snapshot_call(call)
|
||||
if step.match is not None and step.match(matcher_call) is False:
|
||||
raise SandboxCallMatcherError(
|
||||
f"Sandbox matcher rejected {method} call #{call_index + 1}.",
|
||||
call=_snapshot_call(call),
|
||||
call_index=call_index,
|
||||
)
|
||||
self._steps.pop(0)
|
||||
if step.outcome == "error":
|
||||
raise step.value
|
||||
if step.outcome == "responder":
|
||||
result = step.value(_snapshot_call(call))
|
||||
if inspect.isawaitable(result):
|
||||
result = await result
|
||||
return result
|
||||
return step.value
|
||||
|
||||
async def start(self) -> None:
|
||||
self._running = True
|
||||
self.state.workspace_root_ready = True
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._running = False
|
||||
|
||||
async def _before_shutdown(self) -> None:
|
||||
return
|
||||
|
||||
async def running(self) -> bool:
|
||||
return self._running
|
||||
|
||||
def supports_pty(self) -> bool:
|
||||
return _PTY_METHODS.issubset(self._configured_methods)
|
||||
|
||||
async def exec(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
shell: bool | list[str] = True,
|
||||
user: str | User | None = None,
|
||||
) -> ExecResult:
|
||||
return cast(
|
||||
ExecResult,
|
||||
await self._invoke(
|
||||
"exec",
|
||||
command,
|
||||
{"timeout": timeout, "shell": shell, "user": user},
|
||||
),
|
||||
)
|
||||
|
||||
async def read(self, path: Path, *, user: str | User | None = None) -> io.IOBase:
|
||||
return cast(io.IOBase, await self._invoke("read", (path,), {"user": user}))
|
||||
|
||||
async def write(
|
||||
self,
|
||||
path: Path,
|
||||
data: io.IOBase,
|
||||
*,
|
||||
user: str | User | None = None,
|
||||
) -> None:
|
||||
await self._invoke("write", (path, data), {"user": user})
|
||||
|
||||
async def ls(
|
||||
self,
|
||||
path: Path | str,
|
||||
*,
|
||||
user: str | User | None = None,
|
||||
) -> list[FileEntry]:
|
||||
return cast(list[FileEntry], await self._invoke("ls", (path,), {"user": user}))
|
||||
|
||||
async def rm(
|
||||
self,
|
||||
path: Path | str,
|
||||
*,
|
||||
recursive: bool = False,
|
||||
user: str | User | None = None,
|
||||
) -> None:
|
||||
await self._invoke("rm", (path,), {"recursive": recursive, "user": user})
|
||||
|
||||
async def mkdir(
|
||||
self,
|
||||
path: Path | str,
|
||||
*,
|
||||
parents: bool = False,
|
||||
user: str | User | None = None,
|
||||
) -> None:
|
||||
await self._invoke("mkdir", (path,), {"parents": parents, "user": user})
|
||||
|
||||
async def apply_patch(
|
||||
self,
|
||||
operations: ApplyPatchOperation
|
||||
| dict[str, object]
|
||||
| list[ApplyPatchOperation | dict[str, object]],
|
||||
*,
|
||||
patch_format: PatchFormat | Literal["v4a"] = "v4a",
|
||||
) -> str:
|
||||
return cast(
|
||||
str,
|
||||
await self._invoke(
|
||||
"apply_patch",
|
||||
(operations,),
|
||||
{"patch_format": patch_format},
|
||||
),
|
||||
)
|
||||
|
||||
async def _exec_internal(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
) -> ExecResult:
|
||||
_ = (command, timeout)
|
||||
raise NotImplementedError
|
||||
|
||||
async def persist_workspace(self) -> io.IOBase:
|
||||
raise NotImplementedError
|
||||
|
||||
async def hydrate_workspace(self, data: io.IOBase) -> None:
|
||||
_ = data
|
||||
raise NotImplementedError
|
||||
|
||||
async def pty_exec_start(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
shell: bool | list[str] = True,
|
||||
user: str | User | None = None,
|
||||
tty: bool = False,
|
||||
yield_time_s: float | None = None,
|
||||
max_output_tokens: int | None = None,
|
||||
) -> PtyExecUpdate:
|
||||
return cast(
|
||||
PtyExecUpdate,
|
||||
await self._invoke(
|
||||
"pty_exec_start",
|
||||
command,
|
||||
{
|
||||
"timeout": timeout,
|
||||
"shell": shell,
|
||||
"user": user,
|
||||
"tty": tty,
|
||||
"yield_time_s": yield_time_s,
|
||||
"max_output_tokens": max_output_tokens,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
async def pty_write_stdin(
|
||||
self,
|
||||
*,
|
||||
session_id: int,
|
||||
chars: str,
|
||||
yield_time_s: float | None = None,
|
||||
max_output_tokens: int | None = None,
|
||||
) -> PtyExecUpdate:
|
||||
return cast(
|
||||
PtyExecUpdate,
|
||||
await self._invoke(
|
||||
"pty_write_stdin",
|
||||
(),
|
||||
{
|
||||
"session_id": session_id,
|
||||
"chars": chars,
|
||||
"yield_time_s": yield_time_s,
|
||||
"max_output_tokens": max_output_tokens,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def scripted_sandbox_session(
|
||||
steps: Iterable[SandboxStepSpec | Mapping[str, Any]] = (),
|
||||
*,
|
||||
manifest: Manifest | None = None,
|
||||
) -> _ScriptedSandboxSession:
|
||||
"""Create a deterministic provider-free sandbox session for agent workflow tests.
|
||||
|
||||
Each FIFO step defines ``method`` plus exactly one of ``result``, ``responder``, or ``error``.
|
||||
An optional ``match`` callable receives a detached ``SandboxCall``. The returned object is the
|
||||
session itself, so pass it directly to ``SandboxRunConfig(session=session)``. Only configured
|
||||
model-facing methods are visible. The two PTY methods are exposed together when either one is
|
||||
configured because they form one advertised session capability. Use a custom
|
||||
``BaseSandboxSession`` or a real provider for lifecycle, persistence, mount, or broader
|
||||
filesystem behavior.
|
||||
"""
|
||||
normalized = [_normalize_step(step, index) for index, step in enumerate(steps)]
|
||||
return _ScriptedSandboxSession(normalized, manifest=manifest)
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import field
|
||||
@@ -11,6 +12,7 @@ from pydantic import BeforeValidator, JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
_RAW_USAGE_ATTRIBUTE = "_agents_sdk_raw_usage"
|
||||
_NORMALIZED_USAGE_ATTRIBUTE = "_agents_sdk_normalized_usage"
|
||||
_RAW_USAGE_ADAPTER = TypeAdapter(dict[str, JsonValue])
|
||||
_RAW_USAGE_MISSING = object()
|
||||
|
||||
@@ -333,6 +335,10 @@ def _requests_for_response_without_usage(response: Any) -> int:
|
||||
|
||||
def _response_usage_to_usage(response_usage: Any) -> Usage:
|
||||
"""Convert Responses API usage, including adapter-supplied per-request details."""
|
||||
normalized_usage = getattr(response_usage, _NORMALIZED_USAGE_ATTRIBUTE, None)
|
||||
if isinstance(normalized_usage, Usage):
|
||||
return copy.deepcopy(normalized_usage)
|
||||
|
||||
request_usages = getattr(response_usage, "_agents_sdk_request_usages", None)
|
||||
request_count = getattr(response_usage, "_agents_sdk_request_count", 1)
|
||||
|
||||
@@ -362,6 +368,11 @@ def _response_usage_to_usage(response_usage: Any) -> Usage:
|
||||
)
|
||||
|
||||
|
||||
def _attach_normalized_usage(target: Any, usage: Usage) -> None:
|
||||
"""Attach a detached normalized usage snapshot for lossless internal conversion."""
|
||||
object.__setattr__(target, _NORMALIZED_USAGE_ATTRIBUTE, copy.deepcopy(usage))
|
||||
|
||||
|
||||
def _serialize_usage_details(details: Any, default: dict[str, int]) -> dict[str, Any]:
|
||||
"""Serialize token details while applying the given default when empty."""
|
||||
if hasattr(details, "model_dump"):
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
"""Deterministic speech and workflow components for Voice pipeline tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections.abc import AsyncIterator, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .imports import np
|
||||
from .input import AudioInput, StreamedAudioInput
|
||||
from .model import (
|
||||
StreamedTranscriptionSession,
|
||||
STTModel,
|
||||
STTModelSettings,
|
||||
TTSModel,
|
||||
TTSModelSettings,
|
||||
)
|
||||
from .workflow import VoiceWorkflowBase
|
||||
|
||||
|
||||
class VoiceScriptError(Exception):
|
||||
"""Base exception for an invalid or incompletely consumed Voice script."""
|
||||
|
||||
|
||||
class UnexpectedVoiceCall(VoiceScriptError):
|
||||
"""Raised when a Voice component is called after its script is exhausted."""
|
||||
|
||||
def __init__(self, message: str, *, operation: str) -> None:
|
||||
super().__init__(message)
|
||||
self.operation = operation
|
||||
|
||||
|
||||
class UnconsumedVoiceSteps(VoiceScriptError):
|
||||
"""Raised when a test finishes before consuming every configured Voice step."""
|
||||
|
||||
def __init__(self, message: str, *, remaining_steps: int) -> None:
|
||||
super().__init__(message)
|
||||
self.remaining_steps = remaining_steps
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class STTCall:
|
||||
"""A recorded static transcription call."""
|
||||
|
||||
input: AudioInput
|
||||
settings: STTModelSettings
|
||||
trace_include_sensitive_data: bool
|
||||
trace_include_sensitive_audio_data: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class STTSessionCall:
|
||||
"""A recorded streamed transcription-session creation call."""
|
||||
|
||||
input: StreamedAudioInput
|
||||
settings: STTModelSettings
|
||||
trace_include_sensitive_data: bool
|
||||
trace_include_sensitive_audio_data: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TTSCall:
|
||||
"""A recorded text-to-speech call."""
|
||||
|
||||
text: str
|
||||
settings: TTSModelSettings
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TTSResult:
|
||||
"""The PCM byte chunks returned by one text-to-speech call."""
|
||||
|
||||
chunks: Sequence[bytes] = field(default_factory=tuple)
|
||||
|
||||
|
||||
TranscriptionResult = str | Exception
|
||||
TTSResultItem = TTSResult | Sequence[bytes] | Exception
|
||||
WorkflowResult = str | Sequence[str] | Exception
|
||||
|
||||
_START_NOT_CONFIGURED: Any = object()
|
||||
|
||||
|
||||
def _snapshot_audio_input(input: AudioInput) -> AudioInput:
|
||||
return AudioInput(
|
||||
buffer=input.buffer.copy(),
|
||||
frame_rate=input.frame_rate,
|
||||
sample_width=input.sample_width,
|
||||
channels=input.channels,
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_stt_call(call: STTCall) -> STTCall:
|
||||
return STTCall(
|
||||
input=_snapshot_audio_input(call.input),
|
||||
settings=copy.deepcopy(call.settings),
|
||||
trace_include_sensitive_data=call.trace_include_sensitive_data,
|
||||
trace_include_sensitive_audio_data=call.trace_include_sensitive_audio_data,
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_stt_session_call(call: STTSessionCall) -> STTSessionCall:
|
||||
return STTSessionCall(
|
||||
input=call.input,
|
||||
settings=copy.deepcopy(call.settings),
|
||||
trace_include_sensitive_data=call.trace_include_sensitive_data,
|
||||
trace_include_sensitive_audio_data=call.trace_include_sensitive_audio_data,
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_tts_call(call: TTSCall) -> TTSCall:
|
||||
return TTSCall(text=call.text, settings=copy.deepcopy(call.settings))
|
||||
|
||||
|
||||
class ScriptedTranscriptionSession(StreamedTranscriptionSession):
|
||||
"""A closable stream of configured transcription turns."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
turns: str | Iterable[TranscriptionResult] = (),
|
||||
*,
|
||||
close_error: Exception | None = None,
|
||||
) -> None:
|
||||
self._turns: list[TranscriptionResult] = [turns] if isinstance(turns, str) else list(turns)
|
||||
self._close_error = close_error
|
||||
self.closed = False
|
||||
self.close_calls = 0
|
||||
|
||||
async def transcribe_turns(self) -> AsyncIterator[str]:
|
||||
while self._turns and not self.closed:
|
||||
turn = self._turns.pop(0)
|
||||
if isinstance(turn, Exception):
|
||||
raise turn
|
||||
yield turn
|
||||
|
||||
async def close(self) -> None:
|
||||
self.close_calls += 1
|
||||
if self.closed:
|
||||
return
|
||||
self.closed = True
|
||||
if self._close_error is not None:
|
||||
raise self._close_error
|
||||
|
||||
def assert_complete(self) -> None:
|
||||
"""Raise when configured transcript turns remain unconsumed."""
|
||||
if self._turns:
|
||||
raise UnconsumedVoiceSteps(
|
||||
f"{len(self._turns)} scripted transcription turn(s) were not consumed.",
|
||||
remaining_steps=len(self._turns),
|
||||
)
|
||||
|
||||
|
||||
class ScriptedSTTModel(STTModel):
|
||||
"""A deterministic speech-to-text model for static and streamed audio tests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transcriptions: str | Iterable[TranscriptionResult] = (),
|
||||
*,
|
||||
sessions: str
|
||||
| Iterable[ScriptedTranscriptionSession | Iterable[TranscriptionResult] | Exception] = (),
|
||||
model_name: str = "scripted-stt",
|
||||
) -> None:
|
||||
self._transcriptions: list[TranscriptionResult] = (
|
||||
[transcriptions] if isinstance(transcriptions, str) else list(transcriptions)
|
||||
)
|
||||
configured_sessions = [sessions] if isinstance(sessions, str) else sessions
|
||||
self._sessions: list[
|
||||
ScriptedTranscriptionSession | tuple[TranscriptionResult, ...] | Exception
|
||||
] = [
|
||||
configured
|
||||
if isinstance(configured, ScriptedTranscriptionSession | Exception)
|
||||
else (configured,)
|
||||
if isinstance(configured, str)
|
||||
else tuple(configured)
|
||||
for configured in configured_sessions
|
||||
]
|
||||
self._model_name = model_name
|
||||
self._calls: list[STTCall] = []
|
||||
self._session_calls: list[STTSessionCall] = []
|
||||
self._created_sessions: list[ScriptedTranscriptionSession] = []
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self._model_name
|
||||
|
||||
@property
|
||||
def calls(self) -> tuple[STTCall, ...]:
|
||||
"""Return detached snapshots of recorded static transcription calls."""
|
||||
return tuple(_snapshot_stt_call(call) for call in self._calls)
|
||||
|
||||
@property
|
||||
def session_calls(self) -> tuple[STTSessionCall, ...]:
|
||||
"""Return detached snapshots of recorded streamed-session calls."""
|
||||
return tuple(_snapshot_stt_session_call(call) for call in self._session_calls)
|
||||
|
||||
@property
|
||||
def created_sessions(self) -> tuple[ScriptedTranscriptionSession, ...]:
|
||||
"""Return created sessions while preserving their live object identity."""
|
||||
return tuple(self._created_sessions)
|
||||
|
||||
async def transcribe(
|
||||
self,
|
||||
input: AudioInput,
|
||||
settings: STTModelSettings,
|
||||
trace_include_sensitive_data: bool,
|
||||
trace_include_sensitive_audio_data: bool,
|
||||
) -> str:
|
||||
call = STTCall(
|
||||
input=input,
|
||||
settings=settings,
|
||||
trace_include_sensitive_data=trace_include_sensitive_data,
|
||||
trace_include_sensitive_audio_data=trace_include_sensitive_audio_data,
|
||||
)
|
||||
call = _snapshot_stt_call(call)
|
||||
self._calls.append(call)
|
||||
if not self._transcriptions:
|
||||
raise UnexpectedVoiceCall(
|
||||
"Unexpected static transcription call: no scripted transcriptions remain.",
|
||||
operation="static_transcription",
|
||||
)
|
||||
result = self._transcriptions.pop(0)
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
return result
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
input: StreamedAudioInput,
|
||||
settings: STTModelSettings,
|
||||
trace_include_sensitive_data: bool,
|
||||
trace_include_sensitive_audio_data: bool,
|
||||
) -> StreamedTranscriptionSession:
|
||||
call = STTSessionCall(
|
||||
input=input,
|
||||
settings=settings,
|
||||
trace_include_sensitive_data=trace_include_sensitive_data,
|
||||
trace_include_sensitive_audio_data=trace_include_sensitive_audio_data,
|
||||
)
|
||||
call = _snapshot_stt_session_call(call)
|
||||
self._session_calls.append(call)
|
||||
if not self._sessions:
|
||||
raise UnexpectedVoiceCall(
|
||||
"Unexpected streamed transcription session: no scripted sessions remain.",
|
||||
operation="streamed_session",
|
||||
)
|
||||
configured = self._sessions.pop(0)
|
||||
if isinstance(configured, Exception):
|
||||
raise configured
|
||||
session = (
|
||||
configured
|
||||
if isinstance(configured, ScriptedTranscriptionSession)
|
||||
else ScriptedTranscriptionSession(configured)
|
||||
)
|
||||
self._created_sessions.append(session)
|
||||
return session
|
||||
|
||||
def assert_complete(self) -> None:
|
||||
"""Raise when static transcriptions or sessions remain unconsumed."""
|
||||
remaining = len(self._transcriptions) + len(self._sessions)
|
||||
if remaining:
|
||||
raise UnconsumedVoiceSteps(
|
||||
f"{remaining} scripted STT step(s) were not consumed.",
|
||||
remaining_steps=remaining,
|
||||
)
|
||||
for session in self._created_sessions:
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
class ScriptedTTSModel(TTSModel):
|
||||
"""A deterministic text-to-speech model that yields configured PCM byte chunks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
results: Iterable[TTSResultItem] = (),
|
||||
*,
|
||||
model_name: str = "scripted-tts",
|
||||
) -> None:
|
||||
self._results = [self._coerce_result(result) for result in results]
|
||||
self._model_name = model_name
|
||||
self._calls: list[TTSCall] = []
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self._model_name
|
||||
|
||||
@property
|
||||
def calls(self) -> tuple[TTSCall, ...]:
|
||||
"""Return detached snapshots of recorded text-to-speech calls."""
|
||||
return tuple(_snapshot_tts_call(call) for call in self._calls)
|
||||
|
||||
async def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]:
|
||||
call = _snapshot_tts_call(TTSCall(text=text, settings=settings))
|
||||
self._calls.append(call)
|
||||
if not self._results:
|
||||
raise UnexpectedVoiceCall(
|
||||
"Unexpected TTS call: no scripted results remain.",
|
||||
operation="tts",
|
||||
)
|
||||
result = self._results.pop(0)
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
for chunk in result.chunks:
|
||||
yield chunk
|
||||
|
||||
def assert_complete(self) -> None:
|
||||
"""Raise when configured TTS results remain unconsumed."""
|
||||
if self._results:
|
||||
raise UnconsumedVoiceSteps(
|
||||
f"{len(self._results)} scripted TTS result(s) were not consumed.",
|
||||
remaining_steps=len(self._results),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_result(result: TTSResultItem) -> TTSResult | Exception:
|
||||
if isinstance(result, Exception):
|
||||
return result
|
||||
if isinstance(result, TTSResult):
|
||||
return TTSResult(chunks=tuple(result.chunks))
|
||||
return TTSResult(chunks=tuple(result))
|
||||
|
||||
|
||||
class ScriptedVoiceWorkflow(VoiceWorkflowBase):
|
||||
"""A deterministic Voice workflow that yields configured text fragments per turn."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
turns: str | Iterable[WorkflowResult] = (),
|
||||
*,
|
||||
start: str | Sequence[str] | Exception = _START_NOT_CONFIGURED,
|
||||
) -> None:
|
||||
configured_turns = [turns] if isinstance(turns, str) else turns
|
||||
self._turns = [
|
||||
turn if isinstance(turn, Exception) else _normalize_fragments(turn)
|
||||
for turn in configured_turns
|
||||
]
|
||||
self._start = (
|
||||
()
|
||||
if start is _START_NOT_CONFIGURED
|
||||
else start
|
||||
if isinstance(start, Exception)
|
||||
else _normalize_fragments(start)
|
||||
)
|
||||
self._start_configured = start is not _START_NOT_CONFIGURED
|
||||
self._start_pending = self._start_configured
|
||||
self._transcriptions: list[str] = []
|
||||
|
||||
@property
|
||||
def transcriptions(self) -> tuple[str, ...]:
|
||||
"""Return the recorded workflow transcriptions."""
|
||||
return tuple(self._transcriptions)
|
||||
|
||||
async def on_start(self) -> AsyncIterator[str]:
|
||||
if self._start_configured and not self._start_pending:
|
||||
raise UnexpectedVoiceCall(
|
||||
"Unexpected workflow startup call: no scripted startup step remains.",
|
||||
operation="workflow_start",
|
||||
)
|
||||
self._start_pending = False
|
||||
if isinstance(self._start, Exception):
|
||||
raise self._start
|
||||
for fragment in self._start:
|
||||
yield fragment
|
||||
|
||||
async def run(self, transcription: str) -> AsyncIterator[str]:
|
||||
self._transcriptions.append(transcription)
|
||||
if not self._turns:
|
||||
raise UnexpectedVoiceCall(
|
||||
"Unexpected workflow turn: no scripted turns remain.",
|
||||
operation="workflow_turn",
|
||||
)
|
||||
result = self._turns.pop(0)
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
for fragment in result:
|
||||
yield fragment
|
||||
|
||||
def assert_complete(self) -> None:
|
||||
"""Raise when the startup step or configured workflow turns remain unconsumed."""
|
||||
remaining = int(self._start_pending) + len(self._turns)
|
||||
if not remaining:
|
||||
return
|
||||
if self._start_pending and not self._turns:
|
||||
raise UnconsumedVoiceSteps(
|
||||
"1 scripted workflow startup step was not consumed.",
|
||||
remaining_steps=1,
|
||||
)
|
||||
if not self._start_pending:
|
||||
raise UnconsumedVoiceSteps(
|
||||
f"{len(self._turns)} scripted workflow turn(s) were not consumed.",
|
||||
remaining_steps=len(self._turns),
|
||||
)
|
||||
raise UnconsumedVoiceSteps(
|
||||
f"{remaining} scripted workflow step(s) were not consumed.",
|
||||
remaining_steps=remaining,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_fragments(fragments: str | Sequence[str]) -> tuple[str, ...]:
|
||||
return (fragments,) if isinstance(fragments, str) else tuple(fragments)
|
||||
|
||||
|
||||
def pcm16_samples(samples: Iterable[int]) -> bytes:
|
||||
"""Encode integer samples as native little-endian PCM16 bytes."""
|
||||
return np.asarray(list(samples), dtype="<i2").tobytes()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"STTCall",
|
||||
"STTSessionCall",
|
||||
"ScriptedSTTModel",
|
||||
"ScriptedTTSModel",
|
||||
"ScriptedTranscriptionSession",
|
||||
"ScriptedVoiceWorkflow",
|
||||
"TTSCall",
|
||||
"TTSResult",
|
||||
"UnconsumedVoiceSteps",
|
||||
"UnexpectedVoiceCall",
|
||||
"VoiceScriptError",
|
||||
"pcm16_samples",
|
||||
]
|
||||
@@ -4,6 +4,8 @@ Before running any tests, make sure you have `uv` installed (and ideally run `ma
|
||||
|
||||
## Running tests
|
||||
|
||||
For provider-neutral agent workflow tests, prefer `ScriptedModel` from `agents.testing` instead of adding a new mock or fake `Model`. Use `ScriptedRealtimeModel` from `agents.realtime.testing` for Realtime session tests, the scripted utilities from `agents.voice.testing` for Voice pipeline tests, and `scripted_sandbox_session()` from `agents.testing` for deterministic Sandbox session calls. Keep a specialized test double only when the test specifically requires provider-wire conversion, malformed streams, controlled suspension or concurrency, or an exact cancellation or lifecycle boundary that the scripted utilities cannot preserve; document that boundary in the test.
|
||||
|
||||
```
|
||||
make tests
|
||||
```
|
||||
|
||||
@@ -25,8 +25,8 @@ from agents import Agent, Runner, TResponseInputItem, function_tool
|
||||
from agents.extensions.memory import AdvancedSQLiteSession
|
||||
from agents.result import RunResult
|
||||
from agents.run_context import RunContextWrapper
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.usage import Usage
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Mark all tests in this file as asyncio
|
||||
@@ -52,8 +52,8 @@ async def test_tool(query: str) -> str:
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel(), tools=[test_tool])
|
||||
"""Fixture for a basic agent with a scripted model."""
|
||||
return Agent(name="test", model=ScriptedModel(), tools=[test_tool])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -74,7 +74,7 @@ def usage_data() -> Usage:
|
||||
def create_mock_run_result(usage: Usage | None = None, agent: Agent | None = None) -> RunResult:
|
||||
"""Helper function to create a mock RunResult for testing."""
|
||||
if agent is None:
|
||||
agent = Agent(name="test", model=FakeModel())
|
||||
agent = Agent(name="test", model=ScriptedModel())
|
||||
|
||||
if usage is None:
|
||||
usage = Usage(
|
||||
@@ -2057,10 +2057,10 @@ async def test_runner_integration_with_usage_tracking(agent: Agent):
|
||||
# Ignore errors in test helper
|
||||
pass
|
||||
|
||||
# Set up fake model responses
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
fake_model = agent.model
|
||||
fake_model.set_next_output([get_text_message("San Francisco")])
|
||||
# Set up scripted model responses.
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
scripted_model = agent.model
|
||||
scripted_model.enqueue([get_text_message("San Francisco")])
|
||||
|
||||
# First turn
|
||||
result1 = await Runner.run(
|
||||
@@ -2072,7 +2072,7 @@ async def test_runner_integration_with_usage_tracking(agent: Agent):
|
||||
await store_session_usage(result1, session)
|
||||
|
||||
# Second turn
|
||||
fake_model.set_next_output([get_text_message("California")])
|
||||
scripted_model.enqueue([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
await store_session_usage(result2, session)
|
||||
@@ -2085,7 +2085,7 @@ async def test_runner_integration_with_usage_tracking(agent: Agent):
|
||||
session_usage = await session.get_session_usage()
|
||||
assert session_usage is not None
|
||||
assert session_usage["total_turns"] == 2
|
||||
# FakeModel doesn't generate realistic usage data, so we just check structure exists
|
||||
# ScriptedModel doesn't generate realistic usage data, so we just check structure exists
|
||||
assert "requests" in session_usage
|
||||
assert "total_tokens" in session_usage
|
||||
|
||||
@@ -2480,9 +2480,9 @@ async def test_tool_execution_integration(agent: Agent):
|
||||
session_id = "tool_integration_test"
|
||||
session = AdvancedSQLiteSession(session_id=session_id, create_tables=True)
|
||||
|
||||
# Set up the fake model to trigger a tool call
|
||||
fake_model = cast(FakeModel, agent.model)
|
||||
fake_model.set_next_output(
|
||||
# Set up the scripted model to trigger a tool call.
|
||||
scripted_model = cast(ScriptedModel, agent.model)
|
||||
scripted_model.enqueue(
|
||||
[
|
||||
{ # type: ignore
|
||||
"type": "function_call",
|
||||
@@ -2494,7 +2494,7 @@ async def test_tool_execution_integration(agent: Agent):
|
||||
)
|
||||
|
||||
# Then set the final response
|
||||
fake_model.set_next_output([get_text_message("Tool executed successfully")])
|
||||
scripted_model.enqueue([get_text_message("Tool executed successfully")])
|
||||
|
||||
# Run the agent
|
||||
result = await Runner.run(
|
||||
@@ -2687,8 +2687,8 @@ async def test_runner_with_session_settings_override(agent: Agent):
|
||||
await session.add_items(items)
|
||||
|
||||
# Use RunConfig to override limit to 2
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("Got it")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
@@ -2700,7 +2700,7 @@ async def test_runner_with_session_settings_override(agent: Agent):
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
|
||||
@@ -19,7 +19,7 @@ pytest.importorskip("aiosqlite") # Skip tests if aiosqlite is not installed
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.extensions.memory import AsyncSQLiteSession
|
||||
from agents.memory import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
@@ -33,8 +33,8 @@ def _assert_cancel_message(exc: asyncio.CancelledError, expected: str) -> None:
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
"""Fixture for a basic agent with a scripted model."""
|
||||
return Agent(name="test", model=ScriptedModel())
|
||||
|
||||
|
||||
def _item_ids(items: Sequence[TResponseInputItem]) -> list[str]:
|
||||
@@ -267,9 +267,9 @@ async def test_async_sqlite_session_runner_integration(agent: Agent):
|
||||
db_path = Path(temp_dir) / "async_runner_integration.db"
|
||||
session = AsyncSQLiteSession("runner_integration_test", db_path)
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
agent.model.enqueue([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
@@ -277,11 +277,11 @@ async def test_async_sqlite_session_runner_integration(agent: Agent):
|
||||
)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
agent.model.enqueue([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
assert isinstance(last_input, list)
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
@@ -296,14 +296,14 @@ async def test_async_sqlite_session_session_isolation(agent: Agent):
|
||||
session1 = AsyncSQLiteSession("session_1", db_path)
|
||||
session2 = AsyncSQLiteSession("session_2", db_path)
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("I like cats.")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("I like cats.")])
|
||||
await Runner.run(agent, "I like cats.", session=session1)
|
||||
|
||||
agent.model.set_next_output([get_text_message("I like dogs.")])
|
||||
agent.model.enqueue([get_text_message("I like dogs.")])
|
||||
await Runner.run(agent, "I like dogs.", session=session2)
|
||||
|
||||
agent.model.set_next_output([get_text_message("You said you like cats.")])
|
||||
agent.model.enqueue([get_text_message("You said you like cats.")])
|
||||
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
|
||||
assert "cats" in result.final_output.lower()
|
||||
assert "dogs" not in result.final_output.lower()
|
||||
|
||||
@@ -45,7 +45,7 @@ from agents.extensions.memory import (
|
||||
DAPR_CONSISTENCY_STRONG,
|
||||
DaprSession,
|
||||
)
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Docker-backed integration tests should stay on the exclusive serial test path.
|
||||
@@ -234,8 +234,8 @@ spec:
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
"""Fixture for a basic agent with a scripted model."""
|
||||
return Agent(name="test", model=ScriptedModel())
|
||||
|
||||
|
||||
async def test_dapr_redis_integration(dapr_container, monkeypatch):
|
||||
@@ -321,8 +321,8 @@ async def test_dapr_runner_integration(agent: Agent, dapr_container, monkeypatch
|
||||
await session.clear_session()
|
||||
|
||||
# First turn
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
@@ -331,12 +331,12 @@ async def test_dapr_runner_integration(agent: Agent, dapr_container, monkeypatch
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
# Second turn - should remember context
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
agent.model.enqueue([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
# Verify history
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from agents.extensions.memory import (
|
||||
DAPR_CONSISTENCY_STRONG,
|
||||
DaprSession,
|
||||
)
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Mark all tests in this file as asyncio
|
||||
@@ -166,8 +166,8 @@ def conflict_dapr_client() -> ConflictFakeDaprClient:
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
"""Fixture for a basic agent with a scripted model."""
|
||||
return Agent(name="test", model=ScriptedModel())
|
||||
|
||||
|
||||
async def _create_test_session(
|
||||
@@ -233,8 +233,8 @@ async def test_runner_integration(agent: Agent, fake_dapr_client: FakeDaprClient
|
||||
|
||||
try:
|
||||
# First turn
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
@@ -243,12 +243,12 @@ async def test_runner_integration(agent: Agent, fake_dapr_client: FakeDaprClient
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
# Second turn
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
agent.model.enqueue([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
# Verify history was passed to the model on the second turn
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
@@ -270,23 +270,23 @@ async def test_session_isolation(fake_dapr_client: FakeDaprClient):
|
||||
)
|
||||
|
||||
try:
|
||||
agent = Agent(name="test", model=FakeModel())
|
||||
agent = Agent(name="test", model=ScriptedModel())
|
||||
|
||||
# Clean up any existing data
|
||||
await session1.clear_session()
|
||||
await session2.clear_session()
|
||||
|
||||
# Interact with session 1
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("I like cats.")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("I like cats.")])
|
||||
await Runner.run(agent, "I like cats.", session=session1)
|
||||
|
||||
# Interact with session 2
|
||||
agent.model.set_next_output([get_text_message("I like dogs.")])
|
||||
agent.model.enqueue([get_text_message("I like dogs.")])
|
||||
await Runner.run(agent, "I like dogs.", session=session2)
|
||||
|
||||
# Go back to session 1 and check its memory
|
||||
agent.model.set_next_output([get_text_message("You said you like cats.")])
|
||||
agent.model.enqueue([get_text_message("You said you like cats.")])
|
||||
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
|
||||
assert "cats" in result.final_output.lower()
|
||||
assert "dogs" not in result.final_output.lower()
|
||||
@@ -1033,7 +1033,7 @@ async def test_runner_with_session_settings_override(fake_dapr_client: FakeDaprC
|
||||
"""Test that RunConfig can override session's default settings."""
|
||||
from agents import Agent, RunConfig, Runner
|
||||
from agents.memory import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
session = DaprSession(
|
||||
@@ -1052,9 +1052,9 @@ async def test_runner_with_session_settings_override(fake_dapr_client: FakeDaprC
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("Got it")])
|
||||
model.enqueue([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
@@ -1066,7 +1066,7 @@ async def test_runner_with_session_settings_override(fake_dapr_client: FakeDaprC
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
|
||||
@@ -19,7 +19,7 @@ from agents import (
|
||||
TResponseInputItem,
|
||||
)
|
||||
from agents.extensions.memory.encrypt_session import EncryptedSession
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Mark all tests in this file as asyncio
|
||||
@@ -35,8 +35,8 @@ def _invalid_encrypted_envelope() -> TResponseInputItem:
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
"""Fixture for a basic agent with a scripted model."""
|
||||
return Agent(name="test", model=ScriptedModel())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -106,8 +106,8 @@ async def test_encrypted_session_with_runner(
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
@@ -115,11 +115,11 @@ async def test_encrypted_session_with_runner(
|
||||
)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
agent.model.enqueue([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
@@ -603,7 +603,7 @@ async def test_runner_with_session_settings_override(encryption_key: str):
|
||||
"""Test that RunConfig can override session's default settings."""
|
||||
from agents import Agent, RunConfig, Runner
|
||||
from agents.memory import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
@@ -620,9 +620,9 @@ async def test_runner_with_session_settings_override(encryption_key: str):
|
||||
items: list[TResponseInputItem] = [{"role": "user", "content": f"Turn {i}"} for i in range(10)]
|
||||
await session.add_items(items)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("Got it")])
|
||||
model.enqueue([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
@@ -634,7 +634,7 @@ async def test_runner_with_session_settings_override(encryption_key: str):
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
|
||||
@@ -22,7 +22,7 @@ import pytest
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.memory.session_settings import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
@@ -338,7 +338,7 @@ def session() -> MongoDBSession:
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
return Agent(name="test", model=FakeModel())
|
||||
return Agent(name="test", model=ScriptedModel())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1048,16 +1048,16 @@ async def test_runner_integration(agent: Agent) -> None:
|
||||
"""MongoDBSession must supply conversation history to the Runner."""
|
||||
session = _make_session("runner-test")
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(agent, "Where is the Golden Gate Bridge?", session=session)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
agent.model.enqueue([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
@@ -1069,14 +1069,14 @@ async def test_runner_session_isolation(agent: Agent) -> None:
|
||||
s1 = MongoDBSession("user-a", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
s2 = MongoDBSession("user-b", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("I like cats.")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("I like cats.")])
|
||||
await Runner.run(agent, "I like cats.", session=s1)
|
||||
|
||||
agent.model.set_next_output([get_text_message("I like dogs.")])
|
||||
agent.model.enqueue([get_text_message("I like dogs.")])
|
||||
await Runner.run(agent, "I like dogs.", session=s2)
|
||||
|
||||
agent.model.set_next_output([get_text_message("You said you like cats.")])
|
||||
agent.model.enqueue([get_text_message("You said you like cats.")])
|
||||
result = await Runner.run(agent, "What animal did I mention?", session=s1)
|
||||
assert "cats" in result.final_output.lower()
|
||||
assert "dogs" not in result.final_output.lower()
|
||||
@@ -1099,8 +1099,8 @@ async def test_runner_with_session_settings_limit(agent: Agent) -> None:
|
||||
]
|
||||
await session.add_items(history)
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("Got it")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("Got it")])
|
||||
await Runner.run(
|
||||
agent,
|
||||
"New question",
|
||||
@@ -1108,7 +1108,7 @@ async def test_runner_with_session_settings_limit(agent: Agent) -> None:
|
||||
run_config=RunConfig(session_settings=SessionSettings(limit=2)),
|
||||
)
|
||||
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
history_items = [i for i in last_input if i.get("content") != "New question"]
|
||||
assert len(history_items) == 2
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ pytest.importorskip("redis") # Skip tests if Redis is not installed
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.extensions.memory.redis_session import RedisSession
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Keep the fallback-to-real-Redis path isolated from xdist workers.
|
||||
@@ -61,8 +61,8 @@ async def _safe_rpush(client: Redis, key: str, value: str) -> None:
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
"""Fixture for a basic agent with a scripted model."""
|
||||
return Agent(name="test", model=ScriptedModel())
|
||||
|
||||
|
||||
async def _create_redis_session(
|
||||
@@ -205,8 +205,8 @@ async def test_runner_integration(agent: Agent):
|
||||
|
||||
try:
|
||||
# First turn
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
@@ -215,12 +215,12 @@ async def test_runner_integration(agent: Agent):
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
# Second turn
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
agent.model.enqueue([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
# Verify history was passed to the model on the second turn
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
@@ -234,23 +234,23 @@ async def test_session_isolation():
|
||||
session2 = await _create_redis_session("session_2")
|
||||
|
||||
try:
|
||||
agent = Agent(name="test", model=FakeModel())
|
||||
agent = Agent(name="test", model=ScriptedModel())
|
||||
|
||||
# Clean up any existing data
|
||||
await session1.clear_session()
|
||||
await session2.clear_session()
|
||||
|
||||
# Interact with session 1
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("I like cats.")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("I like cats.")])
|
||||
await Runner.run(agent, "I like cats.", session=session1)
|
||||
|
||||
# Interact with session 2
|
||||
agent.model.set_next_output([get_text_message("I like dogs.")])
|
||||
agent.model.enqueue([get_text_message("I like dogs.")])
|
||||
await Runner.run(agent, "I like dogs.", session=session2)
|
||||
|
||||
# Go back to session 1 and check its memory
|
||||
agent.model.set_next_output([get_text_message("You said you like cats.")])
|
||||
agent.model.enqueue([get_text_message("You said you like cats.")])
|
||||
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
|
||||
assert "cats" in result.final_output.lower()
|
||||
assert "dogs" not in result.final_output.lower()
|
||||
@@ -1094,7 +1094,7 @@ async def test_runner_with_session_settings_override():
|
||||
"""Test that RunConfig can override session's default settings."""
|
||||
from agents import Agent, RunConfig, Runner
|
||||
from agents.memory import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
if USE_FAKE_REDIS:
|
||||
@@ -1118,9 +1118,9 @@ async def test_runner_with_session_settings_override():
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("Got it")])
|
||||
model.enqueue([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
@@ -1132,7 +1132,7 @@ async def test_runner_with_session_settings_override():
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
|
||||
@@ -25,7 +25,7 @@ pytest.importorskip("sqlalchemy") # Skip tests if SQLAlchemy is not installed
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.extensions.memory.sqlalchemy_session import SQLAlchemySession
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Mark all tests in this file as asyncio
|
||||
@@ -72,8 +72,8 @@ def _item_ids(items: Sequence[TResponseInputItem]) -> list[str]:
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
"""Fixture for a basic agent with a scripted model."""
|
||||
return Agent(name="test", model=ScriptedModel())
|
||||
|
||||
|
||||
async def test_sqlalchemy_session_direct_ops(agent: Agent):
|
||||
@@ -159,8 +159,8 @@ async def test_runner_integration(agent: Agent):
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
# First turn
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
@@ -169,12 +169,12 @@ async def test_runner_integration(agent: Agent):
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
# Second turn
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
agent.model.enqueue([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
# Verify history was passed to the model on the second turn
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
@@ -188,16 +188,16 @@ async def test_session_isolation(agent: Agent):
|
||||
session2 = SQLAlchemySession.from_url(session_id_2, url=DB_URL, create_tables=True)
|
||||
|
||||
# Interact with session 1
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("I like cats.")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("I like cats.")])
|
||||
await Runner.run(agent, "I like cats.", session=session1)
|
||||
|
||||
# Interact with session 2
|
||||
agent.model.set_next_output([get_text_message("I like dogs.")])
|
||||
agent.model.enqueue([get_text_message("I like dogs.")])
|
||||
await Runner.run(agent, "I like dogs.", session=session2)
|
||||
|
||||
# Go back to session 1 and check its memory
|
||||
agent.model.set_next_output([get_text_message("You said you like cats.")])
|
||||
agent.model.enqueue([get_text_message("You said you like cats.")])
|
||||
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
|
||||
assert "cats" in result.final_output.lower()
|
||||
assert "dogs" not in result.final_output.lower()
|
||||
@@ -1259,8 +1259,8 @@ async def test_runner_with_session_settings_override(agent: Agent):
|
||||
await session.add_items(items)
|
||||
|
||||
# Use RunConfig to override limit to 2
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("Got it")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
@@ -1272,7 +1272,7 @@ async def test_runner_with_session_settings_override(agent: Agent):
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
|
||||
@@ -1,34 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from agents.extensions.sandbox.blaxel.mounts import (
|
||||
BlaxelCloudBucketMountConfig,
|
||||
_mount_gcs,
|
||||
_mount_s3,
|
||||
)
|
||||
from agents.sandbox import ExecResult
|
||||
from agents.testing import scripted_sandbox_session
|
||||
|
||||
_INJECTION = "x; touch /tmp/pwned"
|
||||
|
||||
|
||||
class _RecordingSession:
|
||||
"""Minimal sandbox session that records the `sh -c` commands it is asked to run."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.commands: list[str] = []
|
||||
|
||||
async def exec(self, *args: Any, **kwargs: Any) -> Any:
|
||||
if len(args) >= 3 and args[0] == "sh" and args[1] == "-c":
|
||||
self.commands.append(args[2])
|
||||
return SimpleNamespace(exit_code=0, stdout=b"", stderr=b"")
|
||||
def _successful_exec(_call: object) -> ExecResult:
|
||||
return ExecResult(exit_code=0, stdout=b"", stderr=b"")
|
||||
|
||||
|
||||
async def test_s3_mount_options_are_shell_quoted() -> None:
|
||||
session = _RecordingSession()
|
||||
session = scripted_sandbox_session(
|
||||
[{"method": "exec", "responder": _successful_exec} for _ in range(3)]
|
||||
)
|
||||
await _mount_s3(
|
||||
session, # type: ignore[arg-type]
|
||||
session,
|
||||
BlaxelCloudBucketMountConfig(
|
||||
provider="s3",
|
||||
bucket="bucket",
|
||||
@@ -36,15 +30,19 @@ async def test_s3_mount_options_are_shell_quoted() -> None:
|
||||
endpoint_url=f"http://{_INJECTION}",
|
||||
),
|
||||
)
|
||||
cmd = next(c for c in session.commands if c.startswith("s3fs"))
|
||||
commands = [call.args[2] for call in session.calls if call.args[:2] == ("sh", "-c")]
|
||||
cmd = next(command for command in commands if command.startswith("s3fs"))
|
||||
# The injected `; touch` must stay inside the -o option token, not become its own command.
|
||||
assert "touch" not in shlex.split(cmd)
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
async def test_gcs_mount_prefix_is_shell_quoted() -> None:
|
||||
session = _RecordingSession()
|
||||
session = scripted_sandbox_session(
|
||||
[{"method": "exec", "responder": _successful_exec} for _ in range(3)]
|
||||
)
|
||||
await _mount_gcs(
|
||||
session, # type: ignore[arg-type]
|
||||
session,
|
||||
BlaxelCloudBucketMountConfig(
|
||||
provider="gcs",
|
||||
bucket="bucket",
|
||||
@@ -52,5 +50,7 @@ async def test_gcs_mount_prefix_is_shell_quoted() -> None:
|
||||
prefix=_INJECTION,
|
||||
),
|
||||
)
|
||||
cmd = next(c for c in session.commands if c.startswith("gcsfuse"))
|
||||
commands = [call.args[2] for call in session.calls if call.args[:2] == ("sh", "-c")]
|
||||
cmd = next(command for command in commands if command.startswith("gcsfuse"))
|
||||
assert "touch" not in shlex.split(cmd)
|
||||
session.assert_complete()
|
||||
|
||||
@@ -12,22 +12,13 @@ from agents.extensions.sandbox._rclone import (
|
||||
)
|
||||
from agents.sandbox.errors import MountConfigError
|
||||
from agents.sandbox.types import ExecResult
|
||||
from agents.testing import scripted_sandbox_session
|
||||
|
||||
|
||||
def _result(*, exit_code: int = 0, stdout: bytes = b"") -> ExecResult:
|
||||
return ExecResult(stdout=stdout, stderr=b"", exit_code=exit_code)
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self, results: list[ExecResult]) -> None:
|
||||
self.results = results
|
||||
self.calls: list[tuple[tuple[str, ...], dict[str, object]]] = []
|
||||
|
||||
async def exec(self, *command: str, **kwargs: object) -> ExecResult:
|
||||
self.calls.append((command, kwargs))
|
||||
return self.results.pop(0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("machine", "expected"),
|
||||
[
|
||||
@@ -73,25 +64,25 @@ def test_rclone_install_command_pins_and_verifies_archive() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_rclone_preserves_preinstalled_binary() -> None:
|
||||
session = _FakeSession([_result()])
|
||||
session = scripted_sandbox_session([{"method": "exec", "result": _result()}])
|
||||
|
||||
await ensure_rclone(session) # type: ignore[arg-type]
|
||||
await ensure_rclone(session)
|
||||
|
||||
assert len(session.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_rclone_rejects_unsupported_architecture_before_install() -> None:
|
||||
session = _FakeSession(
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
_result(exit_code=1),
|
||||
_result(),
|
||||
_result(stdout=b"mips64\n"),
|
||||
{"method": "exec", "result": _result(exit_code=1)},
|
||||
{"method": "exec", "result": _result()},
|
||||
{"method": "exec", "result": _result(stdout=b"mips64\n")},
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(MountConfigError, match="architecture is unsupported") as exc_info:
|
||||
await ensure_rclone(session) # type: ignore[arg-type]
|
||||
await ensure_rclone(session)
|
||||
|
||||
assert exc_info.value.context["architecture"] == "mips64"
|
||||
assert len(session.calls) == 3
|
||||
@@ -99,19 +90,22 @@ async def test_ensure_rclone_rejects_unsupported_architecture_before_install() -
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_rclone_reports_checksum_mismatch() -> None:
|
||||
session = _FakeSession(
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
_result(exit_code=1),
|
||||
_result(),
|
||||
_result(stdout=b"x86_64\n"),
|
||||
_result(),
|
||||
_result(),
|
||||
_result(exit_code=_RCLONE_CHECKSUM_MISMATCH_EXIT),
|
||||
{"method": "exec", "result": _result(exit_code=1)},
|
||||
{"method": "exec", "result": _result()},
|
||||
{"method": "exec", "result": _result(stdout=b"x86_64\n")},
|
||||
{"method": "exec", "result": _result()},
|
||||
{"method": "exec", "result": _result()},
|
||||
{
|
||||
"method": "exec",
|
||||
"result": _result(exit_code=_RCLONE_CHECKSUM_MISMATCH_EXIT),
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(MountConfigError, match="checksum verification failed") as exc_info:
|
||||
await ensure_rclone(session) # type: ignore[arg-type]
|
||||
await ensure_rclone(session)
|
||||
|
||||
assert exc_info.value.context == {
|
||||
"package": "rclone",
|
||||
|
||||
@@ -48,8 +48,8 @@ from agents.sandbox.session.manager import Instrumentation
|
||||
from agents.sandbox.session.sinks import CallbackSink
|
||||
from agents.sandbox.snapshot import NoopSnapshot, SnapshotBase
|
||||
from agents.sandbox.types import User
|
||||
from agents.testing import ScriptedModel
|
||||
from tests._fake_workspace_paths import resolve_fake_workspace_path
|
||||
from tests.fake_model import FakeModel
|
||||
|
||||
|
||||
class _FakeNetworkPolicyRule(BaseModel):
|
||||
@@ -1045,7 +1045,7 @@ async def test_vercel_injected_session_accepts_unchanged_s3_manifest(
|
||||
manifest=_vercel_s3_manifest(package_module),
|
||||
options=vercel_module.VercelSandboxClientOptions(),
|
||||
)
|
||||
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
|
||||
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
||||
manager = SandboxRuntimeSessionManager(
|
||||
starting_agent=agent,
|
||||
sandbox_config=SandboxRunConfig(session=session),
|
||||
@@ -1074,7 +1074,7 @@ async def test_vercel_injected_session_revalidates_preexisting_s3_topology_mutat
|
||||
)
|
||||
mount = cast(S3Mount, session.state.manifest.entries["remote"])
|
||||
mount.bucket = "tampered-bucket"
|
||||
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
|
||||
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
||||
manager = SandboxRuntimeSessionManager(
|
||||
starting_agent=agent,
|
||||
sandbox_config=SandboxRunConfig(session=session),
|
||||
@@ -1105,7 +1105,7 @@ async def test_vercel_injected_session_applies_non_mount_delta_with_fixed_s3_top
|
||||
options=vercel_module.VercelSandboxClientOptions(),
|
||||
)
|
||||
sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox)
|
||||
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
|
||||
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
||||
manager = SandboxRuntimeSessionManager(
|
||||
starting_agent=agent,
|
||||
sandbox_config=SandboxRunConfig(session=session),
|
||||
@@ -1147,7 +1147,7 @@ async def test_vercel_live_manifest_update_uses_one_running_snapshot(
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(session, "running", running_once)
|
||||
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
|
||||
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
||||
|
||||
update = await SandboxRuntimeSessionManager._process_live_session_manifest(
|
||||
agent=agent,
|
||||
@@ -1176,7 +1176,7 @@ async def test_vercel_stopped_injected_session_rejects_non_mount_delta_before_st
|
||||
)
|
||||
sandbox = cast(_FakeAsyncSandbox, session._inner._sandbox)
|
||||
sandbox.status = "stopped"
|
||||
agent = SandboxAgent(name="worker", model=FakeModel(), instructions="Worker.")
|
||||
agent = SandboxAgent(name="worker", model=ScriptedModel(), instructions="Worker.")
|
||||
manager = SandboxRuntimeSessionManager(
|
||||
starting_agent=agent,
|
||||
sandbox_config=SandboxRunConfig(session=session),
|
||||
|
||||
@@ -1,404 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from openai.types.responses import (
|
||||
Response,
|
||||
ResponseApplyPatchToolCall,
|
||||
ResponseCompletedEvent,
|
||||
ResponseContentPartAddedEvent,
|
||||
ResponseContentPartDoneEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseFunctionCallArgumentsDeltaEvent,
|
||||
ResponseFunctionCallArgumentsDoneEvent,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseInProgressEvent,
|
||||
ResponseOutputItemAddedEvent,
|
||||
ResponseOutputItemDoneEvent,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningSummaryPartAddedEvent,
|
||||
ResponseReasoningSummaryPartDoneEvent,
|
||||
ResponseReasoningSummaryTextDeltaEvent,
|
||||
ResponseReasoningSummaryTextDoneEvent,
|
||||
ResponseTextDeltaEvent,
|
||||
ResponseTextDoneEvent,
|
||||
ResponseUsage,
|
||||
)
|
||||
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
|
||||
from openai.types.responses.response_reasoning_summary_part_added_event import (
|
||||
Part as AddedEventPart,
|
||||
)
|
||||
from openai.types.responses.response_reasoning_summary_part_done_event import Part as DoneEventPart
|
||||
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
|
||||
|
||||
from agents.agent_output import AgentOutputSchemaBase
|
||||
from agents.handoffs import Handoff
|
||||
from agents.items import (
|
||||
ModelResponse,
|
||||
TResponseInputItem,
|
||||
TResponseOutputItem,
|
||||
TResponseStreamEvent,
|
||||
)
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import Model, ModelTracing
|
||||
from agents.tool import Tool
|
||||
from agents.tracing import SpanError, generation_span
|
||||
from agents.usage import Usage
|
||||
|
||||
|
||||
class FakeModel(Model):
|
||||
def __init__(
|
||||
self,
|
||||
tracing_enabled: bool = False,
|
||||
initial_output: list[TResponseOutputItem] | Exception | None = None,
|
||||
):
|
||||
if initial_output is None:
|
||||
initial_output = []
|
||||
self.turn_outputs: list[list[TResponseOutputItem] | Exception] = (
|
||||
[initial_output] if initial_output else []
|
||||
)
|
||||
self.tracing_enabled = tracing_enabled
|
||||
self.last_turn_args: dict[str, Any] = {}
|
||||
self.first_turn_args: dict[str, Any] | None = None
|
||||
self.hardcoded_usage: Usage | None = None
|
||||
|
||||
def set_hardcoded_usage(self, usage: Usage):
|
||||
self.hardcoded_usage = usage
|
||||
|
||||
def set_next_output(self, output: list[TResponseOutputItem] | Exception):
|
||||
self.turn_outputs.append(output)
|
||||
|
||||
def add_multiple_turn_outputs(self, outputs: list[list[TResponseOutputItem] | Exception]):
|
||||
self.turn_outputs.extend(outputs)
|
||||
|
||||
def get_next_output(self) -> list[TResponseOutputItem] | Exception:
|
||||
if not self.turn_outputs:
|
||||
return []
|
||||
return self.turn_outputs.pop(0)
|
||||
|
||||
def _record_turn_args(
|
||||
self,
|
||||
*,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: Any | None,
|
||||
) -> None:
|
||||
turn_args = {
|
||||
"system_instructions": system_instructions,
|
||||
"input": input,
|
||||
"model_settings": model_settings,
|
||||
"tools": tools,
|
||||
"output_schema": output_schema,
|
||||
"handoffs": handoffs,
|
||||
"tracing": tracing,
|
||||
"previous_response_id": previous_response_id,
|
||||
"conversation_id": conversation_id,
|
||||
"prompt": prompt,
|
||||
}
|
||||
if self.first_turn_args is None:
|
||||
self.first_turn_args = turn_args.copy()
|
||||
self.last_turn_args = turn_args
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: Any | None,
|
||||
) -> ModelResponse:
|
||||
self._record_turn_args(
|
||||
system_instructions=system_instructions,
|
||||
input=input,
|
||||
model_settings=model_settings,
|
||||
tools=tools,
|
||||
output_schema=output_schema,
|
||||
handoffs=handoffs,
|
||||
tracing=tracing,
|
||||
previous_response_id=previous_response_id,
|
||||
conversation_id=conversation_id,
|
||||
prompt=prompt,
|
||||
)
|
||||
|
||||
with generation_span(disabled=not self.tracing_enabled) as span:
|
||||
output = self.get_next_output()
|
||||
|
||||
if isinstance(output, Exception):
|
||||
span.set_error(
|
||||
SpanError(
|
||||
message="Error",
|
||||
data={
|
||||
"name": output.__class__.__name__,
|
||||
"message": str(output),
|
||||
},
|
||||
)
|
||||
)
|
||||
raise output
|
||||
|
||||
converted_output = []
|
||||
for item in output:
|
||||
if isinstance(item, dict) and item.get("type") == "apply_patch_call":
|
||||
call_id = str(item.get("call_id") or item.get("id") or "")
|
||||
converted_output.append(
|
||||
ResponseApplyPatchToolCall(
|
||||
type="apply_patch_call",
|
||||
id=str(item.get("id") or call_id),
|
||||
call_id=call_id,
|
||||
status=item.get("status") or "completed",
|
||||
operation=item.get("operation"),
|
||||
)
|
||||
)
|
||||
else:
|
||||
converted_output.append(item)
|
||||
|
||||
return ModelResponse(
|
||||
output=converted_output,
|
||||
usage=self.hardcoded_usage or Usage(),
|
||||
response_id="resp-789",
|
||||
)
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None = None,
|
||||
conversation_id: str | None = None,
|
||||
prompt: Any | None = None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
self._record_turn_args(
|
||||
system_instructions=system_instructions,
|
||||
input=input,
|
||||
model_settings=model_settings,
|
||||
tools=tools,
|
||||
output_schema=output_schema,
|
||||
handoffs=handoffs,
|
||||
tracing=tracing,
|
||||
previous_response_id=previous_response_id,
|
||||
conversation_id=conversation_id,
|
||||
prompt=prompt,
|
||||
)
|
||||
with generation_span(disabled=not self.tracing_enabled) as span:
|
||||
output = self.get_next_output()
|
||||
if isinstance(output, Exception):
|
||||
span.set_error(
|
||||
SpanError(
|
||||
message="Error",
|
||||
data={
|
||||
"name": output.__class__.__name__,
|
||||
"message": str(output),
|
||||
},
|
||||
)
|
||||
)
|
||||
raise output
|
||||
|
||||
response = get_response_obj(output, usage=self.hardcoded_usage)
|
||||
sequence_number = 0
|
||||
|
||||
yield ResponseCreatedEvent(
|
||||
type="response.created",
|
||||
response=response,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseInProgressEvent(
|
||||
type="response.in_progress",
|
||||
response=response,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
for output_index, output_item in enumerate(output):
|
||||
yield ResponseOutputItemAddedEvent(
|
||||
type="response.output_item.added",
|
||||
item=output_item,
|
||||
output_index=output_index,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
if isinstance(output_item, ResponseReasoningItem):
|
||||
if output_item.summary:
|
||||
for summary_index, summary in enumerate(output_item.summary):
|
||||
yield ResponseReasoningSummaryPartAddedEvent(
|
||||
type="response.reasoning_summary_part.added",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
summary_index=summary_index,
|
||||
part=AddedEventPart(text=summary.text, type=summary.type),
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseReasoningSummaryTextDeltaEvent(
|
||||
type="response.reasoning_summary_text.delta",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
summary_index=summary_index,
|
||||
delta=summary.text,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseReasoningSummaryTextDoneEvent(
|
||||
type="response.reasoning_summary_text.done",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
summary_index=summary_index,
|
||||
text=summary.text,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseReasoningSummaryPartDoneEvent(
|
||||
type="response.reasoning_summary_part.done",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
summary_index=summary_index,
|
||||
part=DoneEventPart(text=summary.text, type=summary.type),
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
elif isinstance(output_item, ResponseFunctionToolCall):
|
||||
yield ResponseFunctionCallArgumentsDeltaEvent(
|
||||
type="response.function_call_arguments.delta",
|
||||
item_id=output_item.call_id,
|
||||
output_index=output_index,
|
||||
delta=output_item.arguments,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseFunctionCallArgumentsDoneEvent(
|
||||
type="response.function_call_arguments.done",
|
||||
item_id=output_item.call_id,
|
||||
output_index=output_index,
|
||||
arguments=output_item.arguments,
|
||||
name=output_item.name,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
elif isinstance(output_item, ResponseOutputMessage):
|
||||
for content_index, content_part in enumerate(output_item.content or []):
|
||||
if isinstance(content_part, ResponseOutputText):
|
||||
yield ResponseContentPartAddedEvent(
|
||||
type="response.content_part.added",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
part=content_part,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseTextDeltaEvent(
|
||||
type="response.output_text.delta",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
delta=content_part.text,
|
||||
logprobs=[],
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseTextDoneEvent(
|
||||
type="response.output_text.done",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
text=content_part.text,
|
||||
logprobs=[],
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseContentPartDoneEvent(
|
||||
type="response.content_part.done",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
part=content_part,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseOutputItemDoneEvent(
|
||||
type="response.output_item.done",
|
||||
item=output_item,
|
||||
output_index=output_index,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=response,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
|
||||
|
||||
class PromptCacheFakeModel(FakeModel):
|
||||
def _supports_default_prompt_cache_key(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def get_response_obj(
|
||||
output: list[TResponseOutputItem],
|
||||
response_id: str | None = None,
|
||||
usage: Usage | None = None,
|
||||
) -> Response:
|
||||
return Response(
|
||||
id=response_id or "resp-789",
|
||||
created_at=123,
|
||||
model="test_model",
|
||||
object="response",
|
||||
output=output,
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
top_p=None,
|
||||
parallel_tool_calls=False,
|
||||
usage=ResponseUsage(
|
||||
input_tokens=usage.input_tokens if usage else 0,
|
||||
output_tokens=usage.output_tokens if usage else 0,
|
||||
total_tokens=usage.total_tokens if usage else 0,
|
||||
input_tokens_details=InputTokensDetails.model_validate(
|
||||
{
|
||||
"cache_write_tokens": (
|
||||
getattr(usage.input_tokens_details, "cache_write_tokens", 0) if usage else 0
|
||||
),
|
||||
"cached_tokens": (
|
||||
getattr(usage.input_tokens_details, "cached_tokens", 0) if usage else 0
|
||||
),
|
||||
}
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(
|
||||
reasoning_tokens=(
|
||||
getattr(usage.output_tokens_details, "reasoning_tokens", 0) if usage else 0
|
||||
)
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -2,7 +2,8 @@ import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from inline_snapshot import snapshot
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from ..test_responses import get_text_message
|
||||
from .streaming_app import agent, app
|
||||
|
||||
@@ -14,9 +15,9 @@ async def test_streaming_context():
|
||||
leading to a tracing error because the context was closed in the wrong context. This test
|
||||
ensures that this actually works.
|
||||
"""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent.model = model
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
|
||||
transport = ASGITransport(app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as ac:
|
||||
|
||||
+3
-3
@@ -446,7 +446,7 @@ PENDING_TOOL_APPROVAL = Scenario(
|
||||
import asyncio
|
||||
|
||||
from agents import Runner, function_tool
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_function_tool_call
|
||||
|
||||
@function_tool(needs_approval=True)
|
||||
@@ -454,8 +454,8 @@ def historical_approval(account_id: str) -> str:
|
||||
return f"approved:{account_id}"
|
||||
|
||||
async def produce_pending_state():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[[get_function_tool_call(
|
||||
"historical_approval",
|
||||
'{"account_id":"account-1"}',
|
||||
|
||||
@@ -5,8 +5,8 @@ from mcp.types import Tool as MCPTool
|
||||
|
||||
from agents import Agent, RunContextWrapper, Runner
|
||||
from agents.exceptions import UserError
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_function_tool_call, get_text_message
|
||||
from ..utils.hitl import queue_function_call_and_text, resume_after_first_approval
|
||||
from .helpers import FakeMCPServer
|
||||
@@ -19,7 +19,7 @@ async def test_mcp_require_approval_pauses_and_resumes():
|
||||
server = FakeMCPServer(require_approval="always")
|
||||
server.add_tool("add", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
@@ -51,7 +51,7 @@ async def test_mcp_require_approval_tool_lists():
|
||||
server = FakeMCPServer(require_approval=require_approval)
|
||||
server.add_tool("add", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
@@ -76,7 +76,7 @@ async def test_mcp_require_approval_tool_mapping():
|
||||
server = FakeMCPServer(require_approval=require_approval)
|
||||
server.add_tool("add", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
@@ -102,7 +102,7 @@ async def test_mcp_require_approval_mapping_allows_policy_keyword_tool_names():
|
||||
server.add_tool("always", {"type": "object", "properties": {}})
|
||||
server.add_tool("never", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
@@ -167,7 +167,7 @@ async def test_mcp_require_approval_callable_can_allow_and_block_by_tool_name():
|
||||
server.add_tool("guarded", {"type": "object", "properties": {}})
|
||||
server.add_tool("safe", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
@@ -212,7 +212,7 @@ async def test_mcp_require_approval_async_callable_uses_run_context():
|
||||
server = FakeMCPServer(require_approval=require_approval)
|
||||
server.add_tool("conditional", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
|
||||
@@ -8,8 +8,8 @@ import pytest
|
||||
from agents import Agent, Runner
|
||||
from agents.mcp import MCPServerStdio
|
||||
from agents.mcp._compat import MCP_V2, result_next_cursor
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_function_tool_call, get_text_message
|
||||
|
||||
PAGINATED_SERVER_PATH = Path(__file__).parent / "servers" / "paginated.py"
|
||||
@@ -50,8 +50,8 @@ async def test_stdio_server_auto_paginates_tools_and_prompts():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_calls_tool_from_second_stdio_page():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("second_page_tool", "{}")],
|
||||
[get_text_message("done")],
|
||||
|
||||
@@ -4,8 +4,8 @@ import pytest
|
||||
from inline_snapshot import snapshot
|
||||
|
||||
from agents import Agent, RunConfig, Runner
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_function_tool, get_function_tool_call, get_text_message
|
||||
from ..testing_processor import SPAN_PROCESSOR_TESTING, fetch_normalized_spans
|
||||
from .helpers import FakeMCPServer
|
||||
@@ -13,7 +13,7 @@ from .helpers import FakeMCPServer
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_tracing():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
server = FakeMCPServer()
|
||||
server.add_tool("test_tool_1", {})
|
||||
agent = Agent(
|
||||
@@ -23,7 +23,7 @@ async def test_mcp_tracing():
|
||||
tools=[get_function_tool("non_mcp_tool", "tool_result")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[
|
||||
@@ -86,7 +86,7 @@ async def test_mcp_tracing():
|
||||
|
||||
SPAN_PROCESSOR_TESTING.clear()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[
|
||||
@@ -161,7 +161,7 @@ async def test_mcp_tracing():
|
||||
# Add more tools to the server
|
||||
server.add_tool("test_tool_3", {})
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
|
||||
@@ -223,12 +223,12 @@ async def test_mcp_tracing():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_tracing_redacts_output_when_sensitive_data_disabled():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
server = FakeMCPServer()
|
||||
server.add_tool("test_tool_1", {})
|
||||
agent = Agent(name="test", model=model, mcp_servers=[server])
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("test_tool_1", "")],
|
||||
[get_text_message("done")],
|
||||
@@ -284,7 +284,7 @@ async def test_mcp_tracing_redacts_output_when_sensitive_data_disabled():
|
||||
async def test_mcp_tracing_always_hides_url_credentials(
|
||||
trace_include_sensitive_data: bool,
|
||||
):
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
server = FakeMCPServer(
|
||||
server_name=(
|
||||
"streamable_http: https://user:s3cr3t_pw@mcp.example.test:8443/mcp"
|
||||
@@ -293,7 +293,7 @@ async def test_mcp_tracing_always_hides_url_credentials(
|
||||
)
|
||||
server.add_tool("search", {})
|
||||
agent = Agent(name="test", model=model, mcp_servers=[server])
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("search", "")],
|
||||
[get_text_message("done")],
|
||||
|
||||
@@ -5,8 +5,8 @@ from mcp.types import ListResourcesResult, ReadResourceResult
|
||||
|
||||
from agents import Agent, Runner
|
||||
from agents.mcp import MCPServer, MCPToolMetaResolver
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_text_message
|
||||
from .model_compat import ListResourceTemplatesResult
|
||||
|
||||
@@ -173,13 +173,11 @@ async def test_agent_with_prompt_instructions():
|
||||
instructions = prompt_result.messages[0].content.text
|
||||
|
||||
# Create agent with prompt-generated instructions
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="prompt_agent", instructions=instructions, model=model, mcp_servers=[server])
|
||||
|
||||
# Mock model response
|
||||
model.add_multiple_turn_outputs(
|
||||
[[get_text_message("Code analysis complete. Found security vulnerability.")]]
|
||||
)
|
||||
model.extend([[get_text_message("Code analysis complete. Found security vulnerability.")]])
|
||||
|
||||
# Run the agent
|
||||
result = await Runner.run(agent, input="Review this code: def unsafe_exec(cmd): os.system(cmd)")
|
||||
@@ -211,12 +209,12 @@ async def test_agent_with_prompt_instructions_streaming(streaming: bool):
|
||||
instructions = prompt_result.messages[0].content.text
|
||||
|
||||
# Create agent
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="streaming_prompt_agent", instructions=instructions, model=model, mcp_servers=[server]
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs([[get_text_message("Security analysis complete.")]])
|
||||
model.extend([[get_text_message("Security analysis complete.")]])
|
||||
|
||||
if streaming:
|
||||
streaming_result = Runner.run_streamed(agent, input="Review code")
|
||||
|
||||
@@ -15,8 +15,8 @@ from agents import (
|
||||
handoff,
|
||||
)
|
||||
from agents.exceptions import AgentsException
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_function_tool_call, get_text_message
|
||||
from .helpers import FakeMCPServer
|
||||
|
||||
@@ -29,14 +29,14 @@ async def test_runner_calls_mcp_tool(streaming: bool):
|
||||
server.add_tool("test_tool_1", {})
|
||||
server.add_tool("test_tool_2", {})
|
||||
server.add_tool("test_tool_3", {})
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
|
||||
@@ -63,14 +63,14 @@ async def test_runner_asserts_when_mcp_tool_not_found(streaming: bool):
|
||||
server.add_tool("test_tool_1", {})
|
||||
server.add_tool("test_tool_2", {})
|
||||
server.add_tool("test_tool_3", {})
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_doesnt_exist", "")],
|
||||
@@ -99,14 +99,14 @@ async def test_runner_works_with_multiple_mcp_servers(streaming: bool):
|
||||
server2.add_tool("test_tool_2", {})
|
||||
server2.add_tool("test_tool_3", {})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server1, server2],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
|
||||
@@ -138,14 +138,14 @@ async def test_runner_errors_when_mcp_tools_clash(streaming: bool):
|
||||
server2.add_tool("test_tool_2", {})
|
||||
server2.add_tool("test_tool_3", {})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server1, server2],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
|
||||
@@ -172,7 +172,7 @@ async def test_runner_can_call_server_prefixed_mcp_tool_names(streaming: bool):
|
||||
server2 = FakeMCPServer(server_name="calendar")
|
||||
server2.add_tool("search", {})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
@@ -180,7 +180,7 @@ async def test_runner_can_call_server_prefixed_mcp_tool_names(streaming: bool):
|
||||
mcp_config={"include_server_in_tool_names": True},
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("a_message"), get_function_tool_call("mcp_calendar__search", "")],
|
||||
[get_text_message("done")],
|
||||
@@ -220,7 +220,7 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_agent_tools(st
|
||||
on_invoke_tool=invoke_local_tool,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
@@ -238,7 +238,7 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_agent_tools(st
|
||||
assert calendar_search_tool_name != "mcp_calendar__search"
|
||||
assert calendar_search_tool_name.startswith("mcp_calendar__search_")
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
|
||||
[get_text_message("done")],
|
||||
@@ -263,11 +263,11 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(strea
|
||||
server = FakeMCPServer(server_name="calendar")
|
||||
server.add_tool("search", {})
|
||||
|
||||
target_model = FakeModel()
|
||||
target_model = ScriptedModel()
|
||||
target_agent = Agent(name="calendar_agent", model=target_model)
|
||||
target_model.add_multiple_turn_outputs([[get_text_message("handoff target")]])
|
||||
target_model.extend([[get_text_message("handoff target")]])
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
@@ -282,7 +282,7 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(strea
|
||||
assert calendar_search_tool_name != "mcp_calendar__search"
|
||||
assert calendar_search_tool_name.startswith("mcp_calendar__search_")
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
|
||||
[get_text_message("done")],
|
||||
@@ -297,7 +297,7 @@ async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(strea
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
assert server.tool_calls == ["search"]
|
||||
assert target_model.first_turn_args is None
|
||||
assert not target_model.calls
|
||||
|
||||
|
||||
class Foo(BaseModel):
|
||||
@@ -314,7 +314,7 @@ async def test_runner_calls_mcp_tool_with_args(streaming: bool):
|
||||
server.add_tool("test_tool_1", {})
|
||||
server.add_tool("test_tool_2", Foo.model_json_schema())
|
||||
server.add_tool("test_tool_3", {})
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
@@ -323,7 +323,7 @@ async def test_runner_calls_mcp_tool_with_args(streaming: bool):
|
||||
|
||||
json_args = json.dumps(Foo(bar="baz", baz=1).model_dump())
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_2", json_args)],
|
||||
@@ -362,14 +362,14 @@ async def test_runner_emits_mcp_error_tool_call_output_item(streaming: bool):
|
||||
server = CrashingFakeMCPServer()
|
||||
server.add_tool("crashing_tool", {})
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("a_message"), get_function_tool_call("crashing_tool", "{}")],
|
||||
[get_text_message("done")],
|
||||
|
||||
@@ -20,7 +20,7 @@ from agents.memory.openai_conversations_session import (
|
||||
OpenAIConversationsSession,
|
||||
start_openai_conversations_session,
|
||||
)
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
|
||||
@@ -46,8 +46,8 @@ def mock_openai_client():
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
"""Fixture for a basic agent with a scripted model."""
|
||||
return Agent(name="test", model=ScriptedModel())
|
||||
|
||||
|
||||
class TestStartOpenAIConversationsSession:
|
||||
@@ -451,8 +451,8 @@ class TestOpenAIConversationsSessionRunnerIntegration:
|
||||
with patch.object(session, "get_items", return_value=[]):
|
||||
with patch.object(session, "add_items") as mock_add_items:
|
||||
# Run the agent
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("San Francisco")])
|
||||
|
||||
result = await Runner.run(
|
||||
agent, "What city is the Golden Gate Bridge in?", session=session
|
||||
@@ -477,15 +477,15 @@ class TestOpenAIConversationsSessionRunnerIntegration:
|
||||
with patch.object(session, "get_items", return_value=conversation_history):
|
||||
with patch.object(session, "add_items"):
|
||||
# Second turn - should have access to previous conversation
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
assert isinstance(agent.model, ScriptedModel)
|
||||
agent.model.enqueue([get_text_message("California")])
|
||||
|
||||
result = await Runner.run(agent, "What state is it in?", session=session)
|
||||
|
||||
assert result.final_output == "California"
|
||||
|
||||
# Verify that the model received the conversation history
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
last_input = agent.model.calls[-1].input
|
||||
assert len(last_input) > 1 # Should include previous messages
|
||||
|
||||
# Check that previous conversation is included
|
||||
@@ -495,8 +495,8 @@ class TestOpenAIConversationsSessionRunnerIntegration:
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_persists_program_item_ids(self, mock_openai_client):
|
||||
"""Program items keep the id the Conversations create-item schema requires."""
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
Program(
|
||||
|
||||
@@ -29,7 +29,7 @@ from agents.run_internal.items import (
|
||||
TOOL_CALL_SESSION_DESCRIPTION_KEY,
|
||||
TOOL_CALL_SESSION_TITLE_KEY,
|
||||
)
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_function_tool, get_function_tool_call, get_text_message
|
||||
from tests.utils.simple_session import SimpleListSession
|
||||
|
||||
@@ -1463,7 +1463,7 @@ class TestOpenAIResponsesCompactionSession:
|
||||
should_trigger_compaction=lambda ctx: True,
|
||||
)
|
||||
|
||||
model = FakeModel(initial_output=[get_text_message("ok")])
|
||||
model = ScriptedModel(steps=[[get_text_message("ok")]])
|
||||
agent = Agent(name="assistant", model=model)
|
||||
|
||||
await Runner.run(agent, "hello", session=session)
|
||||
@@ -1486,7 +1486,7 @@ class TestOpenAIResponsesCompactionSession:
|
||||
)
|
||||
|
||||
tool = get_function_tool(name="do_thing", return_value="done")
|
||||
model = FakeModel(initial_output=[get_function_tool_call("do_thing")])
|
||||
model = ScriptedModel(steps=[[get_function_tool_call("do_thing")]])
|
||||
agent = Agent(
|
||||
name="assistant",
|
||||
model=model,
|
||||
@@ -1518,7 +1518,7 @@ class TestOpenAIResponsesCompactionSession:
|
||||
)
|
||||
|
||||
tool = get_function_tool(name="do_thing", return_value="done")
|
||||
model = FakeModel(initial_output=[get_function_tool_call("do_thing")])
|
||||
model = ScriptedModel(steps=[[get_function_tool_call("do_thing")]])
|
||||
agent = Agent(
|
||||
name="assistant",
|
||||
model=model,
|
||||
@@ -1554,8 +1554,8 @@ class TestOpenAIResponsesCompactionSession:
|
||||
)
|
||||
|
||||
tool = get_function_tool(name="do_thing", return_value="done")
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("do_thing")],
|
||||
[get_text_message("ok")],
|
||||
@@ -1596,8 +1596,8 @@ class TestOpenAIResponsesCompactionSession:
|
||||
)
|
||||
|
||||
tool = get_function_tool(name="do_thing", return_value="done")
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("do_thing")],
|
||||
[get_function_tool_call("do_thing")],
|
||||
|
||||
@@ -11,7 +11,7 @@ import pytest
|
||||
|
||||
from agents import Agent, RunConfig, Runner, SessionSettings, SQLiteSession, TResponseInputItem
|
||||
from agents.memory.sqlite_session import _await_mutation
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
|
||||
@@ -95,11 +95,11 @@ async def test_session_memory_basic_functionality_parametrized(runner_method):
|
||||
session_id = "test_session_123"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# First turn
|
||||
model.set_next_output([get_text_message("San Francisco")])
|
||||
model.enqueue([get_text_message("San Francisco")])
|
||||
result1 = await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -109,7 +109,7 @@ async def test_session_memory_basic_functionality_parametrized(runner_method):
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
# Second turn - should have conversation history
|
||||
model.set_next_output([get_text_message("California")])
|
||||
model.enqueue([get_text_message("California")])
|
||||
result2 = await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -120,7 +120,7 @@ async def test_session_memory_basic_functionality_parametrized(runner_method):
|
||||
|
||||
# Verify that the input to the second turn includes the previous conversation
|
||||
# The model should have received the full conversation history
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
assert len(last_input) > 1 # Should have more than just the current message
|
||||
|
||||
session.close()
|
||||
@@ -135,16 +135,16 @@ async def test_session_memory_with_explicit_instance_parametrized(runner_method)
|
||||
session_id = "test_session_456"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# First turn
|
||||
model.set_next_output([get_text_message("Hello")])
|
||||
model.enqueue([get_text_message("Hello")])
|
||||
result1 = await run_agent_async(runner_method, agent, "Hi there", session=session)
|
||||
assert result1.final_output == "Hello"
|
||||
|
||||
# Second turn
|
||||
model.set_next_output([get_text_message("I remember you said hi")])
|
||||
model.enqueue([get_text_message("I remember you said hi")])
|
||||
result2 = await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -160,21 +160,21 @@ async def test_session_memory_with_explicit_instance_parametrized(runner_method)
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_memory_disabled_parametrized(runner_method):
|
||||
"""Test that session memory is disabled when session=None across all runner methods."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# First turn (no session parameters = disabled)
|
||||
model.set_next_output([get_text_message("Hello")])
|
||||
model.enqueue([get_text_message("Hello")])
|
||||
result1 = await run_agent_async(runner_method, agent, "Hi there")
|
||||
assert result1.final_output == "Hello"
|
||||
|
||||
# Second turn - should NOT have conversation history
|
||||
model.set_next_output([get_text_message("I don't remember")])
|
||||
model.enqueue([get_text_message("I don't remember")])
|
||||
result2 = await run_agent_async(runner_method, agent, "Do you remember what I said?")
|
||||
assert result2.final_output == "I don't remember"
|
||||
|
||||
# Verify that the input to the second turn is just the current message
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
assert len(last_input) == 1 # Should only have the current message
|
||||
|
||||
|
||||
@@ -186,14 +186,14 @@ async def test_session_memory_different_sessions_parametrized(runner_method):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_memory.db"
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Session 1
|
||||
session_id_1 = "session_1"
|
||||
session_1 = SQLiteSession(session_id_1, db_path)
|
||||
|
||||
model.set_next_output([get_text_message("I like cats")])
|
||||
model.enqueue([get_text_message("I like cats")])
|
||||
result1 = await run_agent_async(runner_method, agent, "I like cats", session=session_1)
|
||||
assert result1.final_output == "I like cats"
|
||||
|
||||
@@ -201,12 +201,12 @@ async def test_session_memory_different_sessions_parametrized(runner_method):
|
||||
session_id_2 = "session_2"
|
||||
session_2 = SQLiteSession(session_id_2, db_path)
|
||||
|
||||
model.set_next_output([get_text_message("I like dogs")])
|
||||
model.enqueue([get_text_message("I like dogs")])
|
||||
result2 = await run_agent_async(runner_method, agent, "I like dogs", session=session_2)
|
||||
assert result2.final_output == "I like dogs"
|
||||
|
||||
# Back to Session 1 - should remember cats, not dogs
|
||||
model.set_next_output([get_text_message("Yes, you mentioned cats")])
|
||||
model.enqueue([get_text_message("Yes, you mentioned cats")])
|
||||
result3 = await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -548,7 +548,7 @@ async def test_session_memory_appends_list_input_by_default(runner_method):
|
||||
session_id = "test_validation_parametrized"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
initial_history: list[TResponseInputItem] = [
|
||||
@@ -559,10 +559,10 @@ async def test_session_memory_appends_list_input_by_default(runner_method):
|
||||
|
||||
list_input = [{"role": "user", "content": "Test message"}]
|
||||
|
||||
model.set_next_output([get_text_message("This should run")])
|
||||
model.enqueue([get_text_message("This should run")])
|
||||
await run_agent_async(runner_method, agent, list_input, session=session)
|
||||
|
||||
assert model.last_turn_args["input"] == initial_history + list_input
|
||||
assert model.calls[-1].input == initial_history + list_input
|
||||
|
||||
session.close()
|
||||
|
||||
@@ -574,7 +574,7 @@ async def test_session_callback_prepared_input(runner_method):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_memory.db"
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Session
|
||||
@@ -594,7 +594,7 @@ async def test_session_callback_prepared_input(runner_method):
|
||||
return [item for item in history if item["role"] == "user"] + new_input
|
||||
|
||||
new_turn_input = [{"role": "user", "content": "What your name?"}]
|
||||
model.set_next_output([get_text_message("I'm gpt-4o")])
|
||||
model.enqueue([get_text_message("I'm gpt-4o")])
|
||||
|
||||
# Run the agent with the callable
|
||||
await run_agent_async(
|
||||
@@ -610,8 +610,8 @@ async def test_session_callback_prepared_input(runner_method):
|
||||
new_turn_input[0], # New input
|
||||
]
|
||||
|
||||
assert len(model.last_turn_args["input"]) == 2
|
||||
assert model.last_turn_args["input"] == expected_model_input
|
||||
assert len(model.calls[-1].input) == 2
|
||||
assert model.calls[-1].input == expected_model_input
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@@ -621,7 +621,7 @@ async def test_session_callback_prepared_input(runner_method):
|
||||
async def test_session_callback_repeating_history_does_not_grow_session(runner_method):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_memory.db"
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
session = SQLiteSession("session_repeat", db_path)
|
||||
|
||||
@@ -632,7 +632,7 @@ async def test_session_callback_repeating_history_does_not_grow_session(runner_m
|
||||
|
||||
try:
|
||||
for turn in range(3):
|
||||
model.set_next_output([get_text_message(f"assistant {turn}")])
|
||||
model.enqueue([get_text_message(f"assistant {turn}")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -825,9 +825,9 @@ async def test_session_add_items_exception_propagates_in_streamed():
|
||||
|
||||
session.add_items = _failing_add_items # type: ignore[method-assign]
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("This should not be reached")])
|
||||
model.enqueue([get_text_message("This should not be reached")])
|
||||
|
||||
result = Runner.run_streamed(agent, "Hello", session=session)
|
||||
|
||||
@@ -981,9 +981,9 @@ async def test_runner_with_session_settings_override():
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("Got it")])
|
||||
model.enqueue([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
@@ -995,7 +995,7 @@ async def test_runner_with_session_settings_override():
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
|
||||
@@ -11,8 +11,8 @@ from agents.guardrail import GuardrailFunctionOutput, InputGuardrail
|
||||
from agents.memory import OpenAIResponsesCompactionSession
|
||||
from agents.memory.session import _session_accepts_wrapper, _session_method_accepts_wrapper
|
||||
from agents.run_internal.session_persistence import rewind_session_items
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import function_tool
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_function_tool_call, get_text_message
|
||||
|
||||
|
||||
@@ -136,7 +136,7 @@ class UninspectableAsyncMethod:
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_passes_same_wrapper_to_context_aware_session(streamed: bool) -> None:
|
||||
session = ContextAwareSession()
|
||||
model = FakeModel(initial_output=[get_text_message("ok")])
|
||||
model = ScriptedModel(steps=[[get_text_message("ok")]])
|
||||
agent = Agent(name="test", model=model)
|
||||
context = TenantContext(tenant_id="tenant-a")
|
||||
|
||||
@@ -161,7 +161,7 @@ async def test_runner_passes_same_wrapper_to_context_aware_session(streamed: boo
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_preserves_legacy_session_call_shapes() -> None:
|
||||
session = LegacySession()
|
||||
model = FakeModel(initial_output=[get_text_message("ok")])
|
||||
model = ScriptedModel(steps=[[get_text_message("ok")]])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
result = await Runner.run(
|
||||
@@ -179,7 +179,7 @@ async def test_runner_preserves_legacy_session_call_shapes() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_does_not_treat_legacy_kwargs_as_wrapper_opt_in() -> None:
|
||||
session = LegacyKwargsSession()
|
||||
model = FakeModel(initial_output=[get_text_message("ok")])
|
||||
model = ScriptedModel(steps=[[get_text_message("ok")]])
|
||||
|
||||
result = await Runner.run(
|
||||
Agent(name="test", model=model),
|
||||
@@ -197,7 +197,7 @@ async def test_runner_does_not_treat_legacy_kwargs_as_wrapper_opt_in() -> None:
|
||||
async def test_runner_preserves_legacy_calls_when_signature_inspection_fails() -> None:
|
||||
session = cast(Any, LegacySession())
|
||||
session.get_items = UninspectableAsyncMethod(session.get_items)
|
||||
model = FakeModel(initial_output=[get_text_message("ok")])
|
||||
model = ScriptedModel(steps=[[get_text_message("ok")]])
|
||||
|
||||
result = await Runner.run(
|
||||
Agent(name="test", model=model),
|
||||
@@ -245,7 +245,7 @@ async def test_runner_does_not_partially_enable_context_aware_session() -> None:
|
||||
pass
|
||||
|
||||
session = PartialSession()
|
||||
model = FakeModel(initial_output=[get_text_message("ok")])
|
||||
model = ScriptedModel(steps=[[get_text_message("ok")]])
|
||||
|
||||
result = await Runner.run(
|
||||
Agent(name="test", model=model),
|
||||
@@ -338,7 +338,7 @@ async def test_input_guardrail_persists_in_the_context_scope(streamed: bool) ->
|
||||
context = TenantContext(tenant_id="tenant-a")
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=FakeModel(initial_output=[get_text_message("not persisted")]),
|
||||
model=ScriptedModel(steps=[[get_text_message("not persisted")]]),
|
||||
input_guardrails=[InputGuardrail(guardrail_function=guardrail_function)],
|
||||
)
|
||||
|
||||
@@ -362,8 +362,8 @@ async def test_resumed_run_persists_in_the_context_scope(streamed: bool) -> None
|
||||
return "tool result"
|
||||
|
||||
tool = function_tool(test_tool, name_override="test_tool", needs_approval=True)
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("test_tool", "{}", call_id="call-resume")],
|
||||
[get_text_message("done")],
|
||||
@@ -415,7 +415,7 @@ async def test_compaction_session_keeps_context_aware_underlying_on_legacy_scope
|
||||
)
|
||||
|
||||
result = await Runner.run(
|
||||
Agent(name="test", model=FakeModel(initial_output=[get_text_message("done")])),
|
||||
Agent(name="test", model=ScriptedModel(steps=[[get_text_message("done")]])),
|
||||
"hello",
|
||||
context=TenantContext(tenant_id="tenant-a"),
|
||||
session=session,
|
||||
|
||||
@@ -9,7 +9,7 @@ import pytest
|
||||
from agents import Agent, RunConfig, SQLiteSession
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.memory import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.memory.test_session import run_agent_async
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
@@ -24,17 +24,17 @@ async def test_session_limit_parameter(runner_method):
|
||||
session_id = "limit_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Build up a longer conversation history
|
||||
model.set_next_output([get_text_message("Reply 1")])
|
||||
model.enqueue([get_text_message("Reply 1")])
|
||||
await run_agent_async(runner_method, agent, "Message 1", session=session)
|
||||
|
||||
model.set_next_output([get_text_message("Reply 2")])
|
||||
model.enqueue([get_text_message("Reply 2")])
|
||||
await run_agent_async(runner_method, agent, "Message 2", session=session)
|
||||
|
||||
model.set_next_output([get_text_message("Reply 3")])
|
||||
model.enqueue([get_text_message("Reply 3")])
|
||||
await run_agent_async(runner_method, agent, "Message 3", session=session)
|
||||
|
||||
# Verify we have 6 items in total (3 user + 3 assistant)
|
||||
@@ -42,7 +42,7 @@ async def test_session_limit_parameter(runner_method):
|
||||
assert len(all_items) == 6
|
||||
|
||||
# Test session_limit via RunConfig - should only get last 2 history items + new input
|
||||
model.set_next_output([get_text_message("Reply 4")])
|
||||
model.enqueue([get_text_message("Reply 4")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -52,7 +52,7 @@ async def test_session_limit_parameter(runner_method):
|
||||
)
|
||||
|
||||
# Verify model received limited history
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
# Should have: 2 history items + 1 new message = 3 total
|
||||
assert len(last_input) == 3
|
||||
# First item should be "Message 3" (not Message 1 or 2)
|
||||
@@ -102,8 +102,8 @@ async def test_session_limit_drops_unmatched_history_function_call_output(runner
|
||||
|
||||
assert await session.get_items(limit=2) == history[-2:]
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Tomorrow is sunny too.")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Tomorrow is sunny too.")])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
await run_agent_async(
|
||||
@@ -114,7 +114,7 @@ async def test_session_limit_drops_unmatched_history_function_call_output(runner
|
||||
run_config=RunConfig(session_settings=SessionSettings(limit=2)),
|
||||
)
|
||||
|
||||
assert model.last_turn_args["input"] == [
|
||||
assert model.calls[-1].input == [
|
||||
history[-1],
|
||||
{"role": "user", "content": "What about tomorrow?"},
|
||||
]
|
||||
@@ -130,18 +130,18 @@ async def test_session_limit_zero(runner_method):
|
||||
session_id = "limit_zero_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Build conversation history
|
||||
model.set_next_output([get_text_message("Reply 1")])
|
||||
model.enqueue([get_text_message("Reply 1")])
|
||||
await run_agent_async(runner_method, agent, "Message 1", session=session)
|
||||
|
||||
model.set_next_output([get_text_message("Reply 2")])
|
||||
model.enqueue([get_text_message("Reply 2")])
|
||||
await run_agent_async(runner_method, agent, "Message 2", session=session)
|
||||
|
||||
# Test with limit=0 - should get NO history, just new message
|
||||
model.set_next_output([get_text_message("Reply 3")])
|
||||
model.enqueue([get_text_message("Reply 3")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -151,7 +151,7 @@ async def test_session_limit_zero(runner_method):
|
||||
)
|
||||
|
||||
# Verify model received only the new message
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
assert len(last_input) == 1
|
||||
assert last_input[0].get("content") == "Message 3"
|
||||
|
||||
@@ -167,12 +167,12 @@ async def test_session_limit_none_gets_all_history(runner_method):
|
||||
session_id = "limit_none_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Build longer conversation
|
||||
for i in range(1, 6):
|
||||
model.set_next_output([get_text_message(f"Reply {i}")])
|
||||
model.enqueue([get_text_message(f"Reply {i}")])
|
||||
await run_agent_async(runner_method, agent, f"Message {i}", session=session)
|
||||
|
||||
# Verify 10 items in session (5 user + 5 assistant)
|
||||
@@ -180,7 +180,7 @@ async def test_session_limit_none_gets_all_history(runner_method):
|
||||
assert len(all_items) == 10
|
||||
|
||||
# Test with session_limit=None (default) - should get all history
|
||||
model.set_next_output([get_text_message("Reply 6")])
|
||||
model.enqueue([get_text_message("Reply 6")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -190,7 +190,7 @@ async def test_session_limit_none_gets_all_history(runner_method):
|
||||
)
|
||||
|
||||
# Verify model received all history + new message
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
assert len(last_input) == 11 # 10 history + 1 new
|
||||
assert last_input[0].get("content") == "Message 1"
|
||||
assert last_input[-1].get("content") == "Message 6"
|
||||
@@ -207,15 +207,15 @@ async def test_session_limit_larger_than_history(runner_method):
|
||||
session_id = "limit_large_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Build small conversation
|
||||
model.set_next_output([get_text_message("Reply 1")])
|
||||
model.enqueue([get_text_message("Reply 1")])
|
||||
await run_agent_async(runner_method, agent, "Message 1", session=session)
|
||||
|
||||
# Test with limit=100 (much larger than actual history)
|
||||
model.set_next_output([get_text_message("Reply 2")])
|
||||
model.enqueue([get_text_message("Reply 2")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
@@ -225,7 +225,7 @@ async def test_session_limit_larger_than_history(runner_method):
|
||||
)
|
||||
|
||||
# Verify model received all available history + new message
|
||||
last_input = model.last_turn_args["input"]
|
||||
last_input = model.calls[-1].input
|
||||
assert len(last_input) == 3 # 2 history + 1 new
|
||||
assert last_input[0].get("content") == "Message 1"
|
||||
# Assistant message has content as a list
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from openai.types.responses import (
|
||||
Response,
|
||||
ResponseCompletedEvent,
|
||||
ResponseOutputItemDoneEvent,
|
||||
ResponseUsage,
|
||||
)
|
||||
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
|
||||
|
||||
from agents.items import TResponseOutputItem, TResponseStreamEvent
|
||||
from agents.testing import ModelStep
|
||||
from agents.usage import Usage
|
||||
|
||||
|
||||
def get_response_obj(
|
||||
output: list[TResponseOutputItem],
|
||||
response_id: str | None = None,
|
||||
usage: Usage | None = None,
|
||||
) -> Response:
|
||||
"""Build an OpenAI response object for adapter-level tests."""
|
||||
return Response(
|
||||
id=response_id or "resp-789",
|
||||
created_at=123,
|
||||
model="test_model",
|
||||
object="response",
|
||||
output=output,
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
top_p=None,
|
||||
parallel_tool_calls=False,
|
||||
usage=ResponseUsage(
|
||||
input_tokens=usage.input_tokens if usage else 0,
|
||||
output_tokens=usage.output_tokens if usage else 0,
|
||||
total_tokens=usage.total_tokens if usage else 0,
|
||||
input_tokens_details=InputTokensDetails.model_validate(
|
||||
{
|
||||
"cache_write_tokens": (
|
||||
getattr(usage.input_tokens_details, "cache_write_tokens", 0) if usage else 0
|
||||
),
|
||||
"cached_tokens": (
|
||||
getattr(usage.input_tokens_details, "cached_tokens", 0) if usage else 0
|
||||
),
|
||||
}
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(
|
||||
reasoning_tokens=(
|
||||
getattr(usage.output_tokens_details, "reasoning_tokens", 0) if usage else 0
|
||||
)
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_exact_output_stream_step(output: list[TResponseOutputItem]) -> ModelStep:
|
||||
"""Build an exact normalized stream for tests whose subject is downstream processing."""
|
||||
stream_output = copy.deepcopy(output)
|
||||
|
||||
async def events(_call: object) -> AsyncIterator[TResponseStreamEvent]:
|
||||
for output_index, output_item in enumerate(stream_output):
|
||||
yield ResponseOutputItemDoneEvent(
|
||||
type="response.output_item.done",
|
||||
item=output_item,
|
||||
output_index=output_index,
|
||||
sequence_number=output_index,
|
||||
)
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=get_response_obj(stream_output),
|
||||
sequence_number=len(stream_output),
|
||||
)
|
||||
|
||||
return ModelStep.stream(events)
|
||||
@@ -158,9 +158,9 @@ def test_unknown_prefix_can_be_preserved_for_openai_compatible_model_ids(monkeyp
|
||||
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
fake_model = object()
|
||||
captured_result["value"] = fake_model
|
||||
return fake_model
|
||||
mapped_model = object()
|
||||
captured_result["value"] = mapped_model
|
||||
return mapped_model
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ from agents.models.openai_responses import (
|
||||
)
|
||||
from agents.retry import ModelRetryAdviceRequest
|
||||
from agents.usage import Usage
|
||||
from tests.fake_model import get_response_obj
|
||||
from tests.model_test_helpers import get_response_obj
|
||||
from tests.testing_processor import fetch_ordered_spans
|
||||
|
||||
|
||||
|
||||
@@ -4,42 +4,20 @@ import pytest
|
||||
|
||||
from agents.realtime.agent import RealtimeAgent
|
||||
from agents.realtime.config import RealtimeRunConfig, RealtimeSessionModelSettings
|
||||
from agents.realtime.model import RealtimeModel, RealtimeModelConfig
|
||||
from agents.realtime.model import RealtimeModelConfig
|
||||
from agents.realtime.runner import RealtimeRunner
|
||||
from agents.realtime.session import RealtimeSession
|
||||
from agents.realtime.testing import RealtimeConnectCall, ScriptedRealtimeModel
|
||||
from agents.tool import function_tool
|
||||
|
||||
|
||||
class MockRealtimeModel(RealtimeModel):
|
||||
def __init__(self):
|
||||
self.connect_args = None
|
||||
class RunnerRealtimeModel(ScriptedRealtimeModel):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(strict=False)
|
||||
|
||||
async def connect(self, options=None):
|
||||
self.connect_args = options
|
||||
|
||||
def add_listener(self, listener):
|
||||
pass
|
||||
|
||||
def remove_listener(self, listener):
|
||||
pass
|
||||
|
||||
async def send_event(self, event):
|
||||
pass
|
||||
|
||||
async def send_message(self, message, other_event_data=None):
|
||||
pass
|
||||
|
||||
async def send_audio(self, audio, commit=False):
|
||||
pass
|
||||
|
||||
async def send_tool_output(self, tool_call, output, start_response=True):
|
||||
pass
|
||||
|
||||
async def interrupt(self):
|
||||
pass
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
@property
|
||||
def connect_args(self) -> RealtimeConnectCall | None:
|
||||
return self.connect_calls[-1] if self.connect_calls else None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -52,12 +30,12 @@ def mock_agent():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_model():
|
||||
return MockRealtimeModel()
|
||||
return RunnerRealtimeModel()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_preserves_falsey_custom_model(mock_agent: Mock):
|
||||
class FalseyRealtimeModel(MockRealtimeModel):
|
||||
class FalseyRealtimeModel(RunnerRealtimeModel):
|
||||
def __bool__(self) -> bool:
|
||||
return False
|
||||
|
||||
@@ -70,7 +48,7 @@ async def test_run_preserves_falsey_custom_model(mock_agent: Mock):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_creates_session_with_no_settings(
|
||||
mock_agent: Mock, mock_model: MockRealtimeModel
|
||||
mock_agent: Mock, mock_model: RunnerRealtimeModel
|
||||
):
|
||||
"""Test that run() creates a session correctly if no settings are provided"""
|
||||
runner = RealtimeRunner(mock_agent, model=mock_model)
|
||||
@@ -98,7 +76,7 @@ async def test_run_creates_session_with_no_settings(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_creates_session_with_settings_only_in_init(
|
||||
mock_agent: Mock, mock_model: MockRealtimeModel
|
||||
mock_agent: Mock, mock_model: RunnerRealtimeModel
|
||||
):
|
||||
"""Test that it creates a session with the right settings if they are provided only in init"""
|
||||
config = RealtimeRunConfig(
|
||||
@@ -122,7 +100,7 @@ async def test_run_creates_session_with_settings_only_in_init(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_creates_session_with_settings_in_both_init_and_run_overrides(
|
||||
mock_agent: Mock, mock_model: MockRealtimeModel
|
||||
mock_agent: Mock, mock_model: RunnerRealtimeModel
|
||||
):
|
||||
"""Test settings provided in run() parameter are passed through"""
|
||||
init_config = RealtimeRunConfig(
|
||||
@@ -152,7 +130,7 @@ async def test_run_creates_session_with_settings_in_both_init_and_run_overrides(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_creates_session_with_settings_only_in_run(
|
||||
mock_agent: Mock, mock_model: MockRealtimeModel
|
||||
mock_agent: Mock, mock_model: RunnerRealtimeModel
|
||||
):
|
||||
"""Test settings provided only in run()"""
|
||||
runner = RealtimeRunner(mock_agent, model=mock_model)
|
||||
@@ -178,7 +156,7 @@ async def test_run_creates_session_with_settings_only_in_run(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_with_context_parameter(mock_agent: Mock, mock_model: MockRealtimeModel):
|
||||
async def test_run_with_context_parameter(mock_agent: Mock, mock_model: RunnerRealtimeModel):
|
||||
"""Test that context parameter is passed through to session"""
|
||||
runner = RealtimeRunner(mock_agent, model=mock_model)
|
||||
test_context = {"user_id": "test123"}
|
||||
@@ -194,7 +172,7 @@ async def test_run_with_context_parameter(mock_agent: Mock, mock_model: MockReal
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_with_none_values_from_agent_does_not_crash(mock_model: MockRealtimeModel):
|
||||
async def test_run_with_none_values_from_agent_does_not_crash(mock_model: RunnerRealtimeModel):
|
||||
"""Test that runner handles agents with None values without crashing"""
|
||||
agent = Mock(spec=RealtimeAgent)
|
||||
agent.get_system_prompt = AsyncMock(return_value=None)
|
||||
@@ -216,7 +194,7 @@ async def test_run_with_none_values_from_agent_does_not_crash(mock_model: MockRe
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_and_handoffs_are_correct(mock_model: MockRealtimeModel):
|
||||
async def test_tool_and_handoffs_are_correct(mock_model: RunnerRealtimeModel):
|
||||
@function_tool
|
||||
def tool_one():
|
||||
return "result_one"
|
||||
|
||||
@@ -76,6 +76,7 @@ from agents.realtime.session import (
|
||||
_PendingToolOutputSendError,
|
||||
_serialize_tool_output,
|
||||
)
|
||||
from agents.realtime.testing import RealtimeConnectCall, ScriptedRealtimeModel
|
||||
from agents.run_context import RunContextWrapper
|
||||
from agents.tool import FunctionTool, function_tool, tool_namespace
|
||||
from agents.tool_context import ToolContext
|
||||
@@ -87,39 +88,23 @@ from agents.tool_guardrails import (
|
||||
from agents.usage import Usage
|
||||
|
||||
|
||||
class _DummyModel(RealtimeModel):
|
||||
class _DummyModel(ScriptedRealtimeModel):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.events: list[Any] = []
|
||||
self.listeners: list[Any] = []
|
||||
self.connect_options: Any | None = None
|
||||
super().__init__(strict=False)
|
||||
|
||||
async def connect(self, options=None):
|
||||
self.connect_options = options
|
||||
@property
|
||||
def events(self) -> tuple[Any, ...]:
|
||||
return self.sent_events
|
||||
|
||||
async def close(self): # pragma: no cover - not used here
|
||||
pass
|
||||
|
||||
async def send_event(self, event):
|
||||
self.events.append(event)
|
||||
|
||||
def add_listener(self, listener):
|
||||
self.listeners.append(listener)
|
||||
|
||||
def remove_listener(self, listener):
|
||||
if listener in self.listeners:
|
||||
self.listeners.remove(listener)
|
||||
@property
|
||||
def connect_options(self) -> RealtimeConnectCall | None:
|
||||
return self.connect_calls[-1] if self.connect_calls else None
|
||||
|
||||
|
||||
class _FailingConnectModel(_DummyModel):
|
||||
def __init__(self, exc: BaseException) -> None:
|
||||
super().__init__()
|
||||
self.exc = exc
|
||||
self.connect_options: Any | None = None
|
||||
|
||||
async def connect(self, options=None):
|
||||
self.connect_options = options
|
||||
raise self.exc
|
||||
self._connect_error = exc
|
||||
|
||||
|
||||
def _agent_with_ambiguous_realtime_tools(name: str = "invalid_agent") -> RealtimeAgent:
|
||||
@@ -208,8 +193,11 @@ async def test_property_and_send_helpers_and_enter_alias():
|
||||
# property
|
||||
assert session.model is model
|
||||
|
||||
# enter alias calls __aenter__
|
||||
async with await session.enter():
|
||||
# The enter alias calls __aenter__, so callers close it manually.
|
||||
entered = await session.enter()
|
||||
try:
|
||||
assert entered is session
|
||||
|
||||
# send helpers
|
||||
await session.send_message("hi")
|
||||
await session.send_audio(b"abc", commit=True)
|
||||
@@ -219,6 +207,8 @@ async def test_property_and_send_helpers_and_enter_alias():
|
||||
assert any(isinstance(e, RealtimeModelSendUserInput) for e in model.events)
|
||||
assert any(isinstance(e, RealtimeModelSendAudio) and e.commit for e in model.events)
|
||||
assert any(isinstance(e, RealtimeModelSendInterrupt) for e in model.events)
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1278,7 +1268,7 @@ async def test_aenter_validates_initial_model_settings_before_listener_registrat
|
||||
with pytest.raises(UserError, match="Duplicate Realtime tool"):
|
||||
await session.__aenter__()
|
||||
|
||||
assert model.listeners == []
|
||||
assert model.listeners == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -1296,16 +1286,12 @@ async def test_aenter_removes_listener_when_connect_fails(exc: BaseException):
|
||||
await session.__aenter__()
|
||||
|
||||
assert model.connect_options is not None
|
||||
assert model.listeners == []
|
||||
assert model.listeners == ()
|
||||
|
||||
|
||||
class MockRealtimeModel(RealtimeModel):
|
||||
class RecordingRealtimeModel(ScriptedRealtimeModel):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.listeners = []
|
||||
self.connect_called = False
|
||||
self.close_called = False
|
||||
self.sent_events = []
|
||||
super().__init__(strict=False)
|
||||
# Legacy tracking for tests that haven't been updated yet
|
||||
self.sent_messages = []
|
||||
self.sent_audio = []
|
||||
@@ -1313,16 +1299,6 @@ class MockRealtimeModel(RealtimeModel):
|
||||
self.interrupts_called = 0
|
||||
self.retired_audio_response_ids = []
|
||||
|
||||
async def connect(self, options=None):
|
||||
self.connect_called = True
|
||||
|
||||
def add_listener(self, listener):
|
||||
self.listeners.append(listener)
|
||||
|
||||
def remove_listener(self, listener):
|
||||
if listener in self.listeners:
|
||||
self.listeners.remove(listener)
|
||||
|
||||
async def send_event(self, event):
|
||||
from agents.realtime.model_inputs import (
|
||||
RealtimeModelSendAudio,
|
||||
@@ -1331,7 +1307,7 @@ class MockRealtimeModel(RealtimeModel):
|
||||
RealtimeModelSendUserInput,
|
||||
)
|
||||
|
||||
self.sent_events.append(event)
|
||||
self._sent_events.append(self._snapshot_send_event(event))
|
||||
|
||||
# Update legacy tracking for compatibility
|
||||
if isinstance(event, RealtimeModelSendUserInput):
|
||||
@@ -1349,9 +1325,6 @@ class MockRealtimeModel(RealtimeModel):
|
||||
await self.send_event(event)
|
||||
return True
|
||||
|
||||
async def close(self):
|
||||
self.close_called = True
|
||||
|
||||
def _retire_response_audio(self, response_id: str) -> None:
|
||||
self.retired_audio_response_ids.append(response_id)
|
||||
|
||||
@@ -1368,7 +1341,7 @@ def mock_agent():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_model():
|
||||
return MockRealtimeModel()
|
||||
return RecordingRealtimeModel()
|
||||
|
||||
|
||||
def _set_default_timeout_fields(tool: Mock) -> Mock:
|
||||
@@ -1392,7 +1365,7 @@ def _named_function_tool(
|
||||
return tool
|
||||
|
||||
|
||||
def _sent_tool_output_strings(model: MockRealtimeModel) -> list[str]:
|
||||
def _sent_tool_output_strings(model: RecordingRealtimeModel) -> list[str]:
|
||||
return [output for _call, output, _start_response in model.sent_tool_outputs]
|
||||
|
||||
|
||||
@@ -2598,7 +2571,7 @@ class TestToolCallExecution:
|
||||
):
|
||||
"""An approved call should retry cached output only for the same invocation."""
|
||||
|
||||
class FailingToolOutputModel(MockRealtimeModel):
|
||||
class FailingToolOutputModel(RecordingRealtimeModel):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.fail_next_tool_output = True
|
||||
@@ -2696,7 +2669,7 @@ class TestToolCallExecution:
|
||||
):
|
||||
"""The async approval path should bind retries to the original invocation."""
|
||||
|
||||
class FailingToolOutputModel(MockRealtimeModel):
|
||||
class FailingToolOutputModel(RecordingRealtimeModel):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.fail_next_tool_output = True
|
||||
@@ -3012,7 +2985,7 @@ class TestToolCallExecution:
|
||||
)
|
||||
|
||||
assert session._current_agent is first_agent
|
||||
assert mock_model.sent_events == []
|
||||
assert mock_model.sent_events == ()
|
||||
assert mock_model.sent_tool_outputs == []
|
||||
assert "call_invalid" not in session._active_tool_invocations
|
||||
assert not session._context_wrapper._tool_invocations["call_invalid"].completed
|
||||
@@ -3807,7 +3780,7 @@ class TestToolCallExecution:
|
||||
):
|
||||
"""A duplicate event during rejection output sending should not emit a second output."""
|
||||
|
||||
class BlockingToolOutputModel(MockRealtimeModel):
|
||||
class BlockingToolOutputModel(RecordingRealtimeModel):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.started = asyncio.Event()
|
||||
@@ -4171,7 +4144,7 @@ class TestToolCallExecution:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_function_output_rejects_handoff_role_reuse(self):
|
||||
class FailingToolOutputModel(MockRealtimeModel):
|
||||
class FailingToolOutputModel(RecordingRealtimeModel):
|
||||
async def send_event(self, event):
|
||||
if isinstance(event, RealtimeModelSendToolOutput):
|
||||
raise RuntimeError("send failed")
|
||||
@@ -4627,14 +4600,14 @@ class TestToolCallExecution:
|
||||
assert sent_output == json.dumps({"name": "demo", "score": 7})
|
||||
|
||||
def test_serialize_tool_output_ignores_non_pydantic_model_dump_objects(self) -> None:
|
||||
class FakeModelDump:
|
||||
class ModelDumpObject:
|
||||
def model_dump(self, *_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
||||
raise AssertionError("non-pydantic objects should not use model_dump")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "fake-model-dump-object"
|
||||
|
||||
assert _serialize_tool_output(FakeModelDump()) == "fake-model-dump-object"
|
||||
assert _serialize_tool_output(ModelDumpObject()) == "fake-model-dump-object"
|
||||
|
||||
def test_serialize_tool_output_falls_back_when_pydantic_json_dump_fails(self) -> None:
|
||||
class FallbackModel(BaseModel):
|
||||
@@ -5044,7 +5017,7 @@ class TestGuardrailFunctionality:
|
||||
release_guardrail = asyncio.Event()
|
||||
operations: list[str] = []
|
||||
|
||||
class TrackingModel(MockRealtimeModel):
|
||||
class TrackingModel(RecordingRealtimeModel):
|
||||
async def send_event(self, event):
|
||||
await super().send_event(event)
|
||||
if isinstance(event, RealtimeModelSendInterrupt):
|
||||
@@ -5158,7 +5131,7 @@ class TestGuardrailFunctionality:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_audio_cleanup_error_releases_session_suppression(self, mock_agent):
|
||||
class FailingRetirementModel(MockRealtimeModel):
|
||||
class FailingRetirementModel(RecordingRealtimeModel):
|
||||
def _retire_response_audio(self, response_id: str) -> None:
|
||||
raise RuntimeError(f"failed to retire {response_id}")
|
||||
|
||||
@@ -5251,7 +5224,7 @@ class TestGuardrailFunctionality:
|
||||
feedback_send_started = asyncio.Event()
|
||||
release_feedback_send = asyncio.Event()
|
||||
|
||||
class BoundaryCheckingModel(MockRealtimeModel):
|
||||
class BoundaryCheckingModel(RecordingRealtimeModel):
|
||||
async def send_event_if(self, event, send_if):
|
||||
feedback_send_started.set()
|
||||
await release_feedback_send.wait()
|
||||
@@ -5288,7 +5261,7 @@ class TestGuardrailFunctionality:
|
||||
async def test_output_text_guardrail_skips_feedback_without_atomic_model_send(
|
||||
self, mock_agent, triggered_guardrail
|
||||
):
|
||||
class CustomModelWithoutAtomicSend(MockRealtimeModel):
|
||||
class CustomModelWithoutAtomicSend(RecordingRealtimeModel):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.feedback_send_started = False
|
||||
@@ -6128,7 +6101,7 @@ class TestUpdateAgentFunctionality:
|
||||
await session.update_agent(invalid_agent)
|
||||
|
||||
assert session._current_agent is first_agent
|
||||
assert mock_model.sent_events == []
|
||||
assert mock_model.sent_events == ()
|
||||
|
||||
|
||||
class TestTranscriptPreservation:
|
||||
|
||||
@@ -2,87 +2,23 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
import websockets.exceptions
|
||||
|
||||
from agents.realtime.events import RealtimeError
|
||||
from agents.realtime.model import RealtimeModel, RealtimeModelConfig, RealtimeModelListener
|
||||
from agents.realtime.model_events import (
|
||||
RealtimeModelErrorEvent,
|
||||
RealtimeModelEvent,
|
||||
RealtimeModelExceptionEvent,
|
||||
)
|
||||
from agents.realtime.session import RealtimeSession
|
||||
from agents.realtime.testing import ScriptedRealtimeModel
|
||||
|
||||
|
||||
class FakeRealtimeModel(RealtimeModel):
|
||||
"""Fake model for testing that forwards events to listeners."""
|
||||
|
||||
def __init__(self):
|
||||
self._listeners: list[RealtimeModelListener] = []
|
||||
self._events_to_send: list[RealtimeModelEvent] = []
|
||||
self._is_connected = False
|
||||
self._send_task: asyncio.Task[None] | None = None
|
||||
|
||||
def set_next_events(self, events: list[RealtimeModelEvent]) -> None:
|
||||
"""Set events to be sent to listeners."""
|
||||
self._events_to_send = events.copy()
|
||||
|
||||
async def connect(self, options: RealtimeModelConfig) -> None:
|
||||
"""Fake connection that starts sending events."""
|
||||
self._is_connected = True
|
||||
self._send_task = asyncio.create_task(self._send_events())
|
||||
|
||||
async def _send_events(self) -> None:
|
||||
"""Send queued events to all listeners."""
|
||||
for event in self._events_to_send:
|
||||
await asyncio.sleep(0.001) # Small delay to simulate async behavior
|
||||
for listener in self._listeners:
|
||||
await listener.on_event(event)
|
||||
|
||||
def add_listener(self, listener: RealtimeModelListener) -> None:
|
||||
"""Add a listener."""
|
||||
self._listeners.append(listener)
|
||||
|
||||
def remove_listener(self, listener: RealtimeModelListener) -> None:
|
||||
"""Remove a listener."""
|
||||
if listener in self._listeners:
|
||||
self._listeners.remove(listener)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close the fake model."""
|
||||
self._is_connected = False
|
||||
if self._send_task and not self._send_task.done():
|
||||
self._send_task.cancel()
|
||||
try:
|
||||
await self._send_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def send_message(
|
||||
self, message: Any, other_event_data: dict[str, Any] | None = None
|
||||
) -> None:
|
||||
"""Fake send message."""
|
||||
pass
|
||||
|
||||
async def send_audio(self, audio: bytes, *, commit: bool = False) -> None:
|
||||
"""Fake send audio."""
|
||||
pass
|
||||
|
||||
async def send_event(self, event: Any) -> None:
|
||||
"""Fake send event."""
|
||||
pass
|
||||
|
||||
async def send_tool_output(self, tool_call: Any, output: str, start_response: bool) -> None:
|
||||
"""Fake send tool output."""
|
||||
pass
|
||||
|
||||
async def interrupt(self) -> None:
|
||||
"""Fake interrupt."""
|
||||
pass
|
||||
def model_with_events(*events: RealtimeModelEvent) -> ScriptedRealtimeModel:
|
||||
return ScriptedRealtimeModel(connect_events=events, strict=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -95,19 +31,11 @@ def fake_agent():
|
||||
return agent
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_model():
|
||||
"""Create a fake model for testing."""
|
||||
return FakeRealtimeModel()
|
||||
|
||||
|
||||
class TestSessionExceptions:
|
||||
"""Test exception handling in RealtimeSession."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_to_end_exception_propagation_and_cleanup(
|
||||
self, fake_model: FakeRealtimeModel, fake_agent
|
||||
):
|
||||
async def test_end_to_end_exception_propagation_and_cleanup(self, fake_agent):
|
||||
"""Test that exceptions are stored, trigger cleanup, and are raised in __aiter__."""
|
||||
# Create test exception
|
||||
test_exception = ValueError("Test error")
|
||||
@@ -116,10 +44,8 @@ class TestSessionExceptions:
|
||||
)
|
||||
|
||||
# Set up session
|
||||
session = RealtimeSession(fake_model, fake_agent, None)
|
||||
|
||||
# Set events to send
|
||||
fake_model.set_next_events([exception_event])
|
||||
model = model_with_events(exception_event)
|
||||
session = RealtimeSession(model, fake_agent, None)
|
||||
|
||||
# Start session
|
||||
async with session:
|
||||
@@ -131,13 +57,11 @@ class TestSessionExceptions:
|
||||
# Verify cleanup occurred
|
||||
assert session._closed is True
|
||||
assert session._stored_exception == test_exception
|
||||
assert fake_model._is_connected is False
|
||||
assert len(fake_model._listeners) == 0
|
||||
assert model.connected is False
|
||||
assert model.listeners == ()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_connection_closure_type_distinction(
|
||||
self, fake_model: FakeRealtimeModel, fake_agent
|
||||
):
|
||||
async def test_websocket_connection_closure_type_distinction(self, fake_agent):
|
||||
"""Test different WebSocket closure types generate appropriate events."""
|
||||
# Test ConnectionClosed (should create exception event)
|
||||
error_closure = websockets.exceptions.ConnectionClosed(None, None)
|
||||
@@ -145,8 +69,7 @@ class TestSessionExceptions:
|
||||
exception=error_closure, context="WebSocket connection closed unexpectedly"
|
||||
)
|
||||
|
||||
session = RealtimeSession(fake_model, fake_agent, None)
|
||||
fake_model.set_next_events([error_event])
|
||||
session = RealtimeSession(model_with_events(error_event), fake_agent, None)
|
||||
|
||||
with pytest.raises(websockets.exceptions.ConnectionClosed):
|
||||
async with session:
|
||||
@@ -158,7 +81,7 @@ class TestSessionExceptions:
|
||||
assert isinstance(session._stored_exception, websockets.exceptions.ConnectionClosed)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_json_parsing_error_handling(self, fake_model: FakeRealtimeModel, fake_agent):
|
||||
async def test_json_parsing_error_handling(self, fake_agent):
|
||||
"""Test JSON parsing errors are properly handled and contextualized."""
|
||||
# Create JSON decode error
|
||||
json_error = json.JSONDecodeError("Invalid JSON", "bad json", 0)
|
||||
@@ -166,8 +89,7 @@ class TestSessionExceptions:
|
||||
exception=json_error, context="Failed to parse WebSocket message as JSON"
|
||||
)
|
||||
|
||||
session = RealtimeSession(fake_model, fake_agent, None)
|
||||
fake_model.set_next_events([json_exception_event])
|
||||
session = RealtimeSession(model_with_events(json_exception_event), fake_agent, None)
|
||||
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
async with session:
|
||||
@@ -179,7 +101,7 @@ class TestSessionExceptions:
|
||||
assert session._closed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exception_context_preservation(self, fake_model: FakeRealtimeModel, fake_agent):
|
||||
async def test_exception_context_preservation(self, fake_agent):
|
||||
"""Test that exception context information is preserved through the handling process."""
|
||||
test_contexts = [
|
||||
("Failed to send audio", RuntimeError("Audio encoding failed")),
|
||||
@@ -190,8 +112,8 @@ class TestSessionExceptions:
|
||||
for context, exception in test_contexts:
|
||||
exception_event = RealtimeModelExceptionEvent(exception=exception, context=context)
|
||||
|
||||
session = RealtimeSession(fake_model, fake_agent, None)
|
||||
fake_model.set_next_events([exception_event])
|
||||
model = model_with_events(exception_event)
|
||||
session = RealtimeSession(model, fake_agent, None)
|
||||
|
||||
with pytest.raises(type(exception)):
|
||||
async with session:
|
||||
@@ -202,14 +124,8 @@ class TestSessionExceptions:
|
||||
assert session._stored_exception == exception
|
||||
assert session._closed is True
|
||||
|
||||
# Reset for next iteration
|
||||
fake_model._is_connected = False
|
||||
fake_model._listeners.clear()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_exception_handling_behavior(
|
||||
self, fake_model: FakeRealtimeModel, fake_agent
|
||||
):
|
||||
async def test_multiple_exception_handling_behavior(self, fake_agent):
|
||||
"""Test behavior when multiple exceptions occur before consumption."""
|
||||
# Create multiple exceptions
|
||||
first_exception = ValueError("First error")
|
||||
@@ -222,13 +138,11 @@ class TestSessionExceptions:
|
||||
exception=second_exception, context="Second context"
|
||||
)
|
||||
|
||||
session = RealtimeSession(fake_model, fake_agent, None)
|
||||
fake_model.set_next_events([first_event, second_event])
|
||||
session = RealtimeSession(model_with_events(first_event, second_event), fake_agent, None)
|
||||
|
||||
# Start session and let events process
|
||||
# Start the session after both events are configured for connection.
|
||||
async with session:
|
||||
# Give time for events to be processed
|
||||
await asyncio.sleep(0.05)
|
||||
pass
|
||||
|
||||
# The first exception should be stored (second should overwrite, but that's
|
||||
# the current behavior). In practice, once an exception occurs, cleanup
|
||||
@@ -237,9 +151,7 @@ class TestSessionExceptions:
|
||||
assert session._closed is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exception_during_guardrail_processing(
|
||||
self, fake_model: FakeRealtimeModel, fake_agent
|
||||
):
|
||||
async def test_exception_during_guardrail_processing(self, fake_agent):
|
||||
"""Test that exceptions don't interfere with guardrail task cleanup."""
|
||||
# Create exception event
|
||||
test_exception = RuntimeError("Processing error")
|
||||
@@ -247,7 +159,8 @@ class TestSessionExceptions:
|
||||
exception=test_exception, context="Processing failed"
|
||||
)
|
||||
|
||||
session = RealtimeSession(fake_model, fake_agent, None)
|
||||
model = model_with_events(exception_event)
|
||||
session = RealtimeSession(model, fake_agent, None)
|
||||
|
||||
async def running_task() -> None:
|
||||
await asyncio.Event().wait()
|
||||
@@ -261,8 +174,6 @@ class TestSessionExceptions:
|
||||
await completed
|
||||
session._guardrail_tasks = {pending, completed}
|
||||
|
||||
fake_model.set_next_events([exception_event])
|
||||
|
||||
with pytest.raises(RuntimeError, match="Processing error"):
|
||||
async with session:
|
||||
async for _event in session:
|
||||
@@ -275,9 +186,7 @@ class TestSessionExceptions:
|
||||
assert len(session._guardrail_tasks) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normal_events_still_work_before_exception(
|
||||
self, fake_model: FakeRealtimeModel, fake_agent
|
||||
):
|
||||
async def test_normal_events_still_work_before_exception(self, fake_agent):
|
||||
"""Test that normal events are processed before an exception occurs."""
|
||||
# Create normal event followed by exception
|
||||
normal_event = RealtimeModelErrorEvent(error={"message": "Normal error"})
|
||||
@@ -285,15 +194,25 @@ class TestSessionExceptions:
|
||||
exception=ValueError("Fatal error"), context="Fatal context"
|
||||
)
|
||||
|
||||
session = RealtimeSession(fake_model, fake_agent, None)
|
||||
fake_model.set_next_events([normal_event, exception_event])
|
||||
model = ScriptedRealtimeModel(strict=False)
|
||||
session = RealtimeSession(model, fake_agent, None)
|
||||
|
||||
events_received = []
|
||||
|
||||
with pytest.raises(ValueError, match="Fatal error"):
|
||||
async with session:
|
||||
async for event in session:
|
||||
events_received.append(event)
|
||||
|
||||
async def emit_events() -> None:
|
||||
await model.emit(normal_event)
|
||||
await asyncio.sleep(0)
|
||||
await model.emit(exception_event)
|
||||
|
||||
emitter = asyncio.create_task(emit_events())
|
||||
try:
|
||||
async for event in session:
|
||||
events_received.append(event)
|
||||
finally:
|
||||
await emitter
|
||||
|
||||
# Should have received events before exception
|
||||
assert len(events_received) >= 1
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,6 +17,7 @@ from agents.run_internal.run_steps import ToolRunCustom
|
||||
from agents.run_internal.tool_actions import CustomToolAction
|
||||
from agents.sandbox.capabilities.tools import SandboxApplyPatchTool
|
||||
from agents.sandbox.types import User
|
||||
from agents.testing import scripted_sandbox_session
|
||||
from tests.sandbox._apply_patch_test_session import (
|
||||
ApplyPatchSession,
|
||||
UserRecordingApplyPatchSession,
|
||||
@@ -26,7 +27,7 @@ from tests.utils.hitl import make_context_wrapper
|
||||
|
||||
class TestSandboxApplyPatchTool:
|
||||
def test_exposes_custom_apply_patch_tool(self) -> None:
|
||||
tool = SandboxApplyPatchTool(session=ApplyPatchSession())
|
||||
tool = SandboxApplyPatchTool(session=scripted_sandbox_session())
|
||||
|
||||
assert isinstance(tool, CustomTool)
|
||||
assert tool.name == "apply_patch"
|
||||
@@ -36,7 +37,7 @@ class TestSandboxApplyPatchTool:
|
||||
assert tool.tool_config["format"]["syntax"] == "lark"
|
||||
|
||||
def test_converter_uses_sandbox_custom_apply_patch_tool_config(self) -> None:
|
||||
tool = SandboxApplyPatchTool(session=ApplyPatchSession())
|
||||
tool = SandboxApplyPatchTool(session=scripted_sandbox_session())
|
||||
|
||||
converted = Converter.convert_tools([tool], handoffs=[])
|
||||
|
||||
@@ -55,7 +56,9 @@ class TestSandboxApplyPatchTool:
|
||||
) -> bool:
|
||||
return operation.type != "create_file"
|
||||
|
||||
tool = SandboxApplyPatchTool(session=ApplyPatchSession(), needs_approval=needs_approval)
|
||||
tool = SandboxApplyPatchTool(
|
||||
session=scripted_sandbox_session(), needs_approval=needs_approval
|
||||
)
|
||||
|
||||
assert cast(object, tool.needs_approval) is needs_approval
|
||||
assert cast(object, tool.operation_needs_approval) is needs_approval
|
||||
@@ -67,7 +70,7 @@ class TestSandboxApplyPatchTool:
|
||||
) -> bool:
|
||||
return operation.type == "delete_file"
|
||||
|
||||
tool = SandboxApplyPatchTool(session=ApplyPatchSession())
|
||||
tool = SandboxApplyPatchTool(session=scripted_sandbox_session())
|
||||
tool.needs_approval = needs_approval
|
||||
|
||||
result = await _execute_custom_tool_call(
|
||||
@@ -147,7 +150,7 @@ class TestSandboxApplyPatchTool:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_patch_input_surfaces_tool_error_after_approval_precheck(self) -> None:
|
||||
tool = SandboxApplyPatchTool(session=ApplyPatchSession(), needs_approval=True)
|
||||
tool = SandboxApplyPatchTool(session=scripted_sandbox_session(), needs_approval=True)
|
||||
|
||||
result = await _execute_custom_tool_call(
|
||||
tool,
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
@@ -17,243 +15,60 @@ from agents.sandbox.capabilities.tools import (
|
||||
)
|
||||
from agents.sandbox.capabilities.tools.shell_tool import _resolve_shell
|
||||
from agents.sandbox.errors import ExecTimeoutError, ExecTransportError, PtySessionNotFoundError
|
||||
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
|
||||
from agents.sandbox.session.pty_types import PtyExecUpdate
|
||||
from agents.sandbox.snapshot import NoopSnapshot
|
||||
from agents.sandbox.types import ExecResult, User
|
||||
from agents.testing import scripted_sandbox_session
|
||||
from agents.tool import FunctionTool
|
||||
from agents.tool_context import ToolContext
|
||||
from tests.utils.factories import TestSessionState
|
||||
|
||||
|
||||
class _ShellSession(BaseSandboxSession):
|
||||
def __init__(self, manifest: Manifest) -> None:
|
||||
self.state = TestSessionState(
|
||||
manifest=manifest,
|
||||
snapshot=NoopSnapshot(id=str(uuid.uuid4())),
|
||||
)
|
||||
self.exec_calls: list[tuple[str, float | None, bool | list[str]]] = []
|
||||
self.exec_users: list[str | None] = []
|
||||
|
||||
async def start(self) -> None:
|
||||
return None
|
||||
|
||||
async def stop(self) -> None:
|
||||
return None
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
return None
|
||||
|
||||
async def running(self) -> bool:
|
||||
return True
|
||||
|
||||
async def read(self, path: Path, *, user: object = None) -> io.BytesIO:
|
||||
_ = (path, user)
|
||||
raise AssertionError("read() should not be called")
|
||||
|
||||
async def write(self, path: Path, data: io.IOBase, *, user: object = None) -> None:
|
||||
_ = (path, data, user)
|
||||
raise AssertionError("write() should not be called")
|
||||
|
||||
async def _exec_internal(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
) -> ExecResult:
|
||||
_ = command
|
||||
_ = timeout
|
||||
raise AssertionError("_exec_internal() should not be called directly")
|
||||
|
||||
async def exec(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
user: str | User | None = None,
|
||||
shell: bool | list[str] = False,
|
||||
) -> ExecResult:
|
||||
self.exec_users.append(user.name if isinstance(user, User) else user)
|
||||
rendered_command = " ".join(str(part) for part in command)
|
||||
self.exec_calls.append((rendered_command, timeout, shell))
|
||||
return ExecResult(
|
||||
stdout=f"stdout: {rendered_command}".encode(),
|
||||
stderr=f"stderr: {rendered_command}".encode(),
|
||||
exit_code=7,
|
||||
)
|
||||
|
||||
async def persist_workspace(self) -> io.IOBase:
|
||||
return io.BytesIO()
|
||||
|
||||
async def hydrate_workspace(self, data: io.IOBase) -> None:
|
||||
_ = data
|
||||
def _default_exec_result(call: Any) -> ExecResult:
|
||||
rendered_command = " ".join(str(part) for part in call.args)
|
||||
return ExecResult(
|
||||
stdout=f"stdout: {rendered_command}".encode(),
|
||||
stderr=f"stderr: {rendered_command}".encode(),
|
||||
exit_code=7,
|
||||
)
|
||||
|
||||
|
||||
class _TimeoutShellSession(_ShellSession):
|
||||
async def exec(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
user: str | User | None = None,
|
||||
shell: bool | list[str] = False,
|
||||
) -> ExecResult:
|
||||
_ = (command, user, shell)
|
||||
raise ExecTimeoutError(command=("sleep 30",), timeout_s=timeout)
|
||||
def _shell_session(
|
||||
*,
|
||||
manifest: Manifest | None = None,
|
||||
result: ExecResult | None = None,
|
||||
error: Exception | None = None,
|
||||
) -> Any:
|
||||
outcome: dict[str, object]
|
||||
if error is not None:
|
||||
outcome = {"error": error}
|
||||
elif result is not None:
|
||||
outcome = {"result": result}
|
||||
else:
|
||||
outcome = {"responder": _default_exec_result}
|
||||
step: dict[str, object] = {"method": "exec"}
|
||||
step.update(outcome)
|
||||
return scripted_sandbox_session(
|
||||
cast(Any, [step]),
|
||||
manifest=manifest or Manifest(root="/workspace"),
|
||||
)
|
||||
|
||||
|
||||
class _OutputShellSession(_ShellSession):
|
||||
def __init__(
|
||||
self,
|
||||
manifest: Manifest,
|
||||
*,
|
||||
stdout: bytes,
|
||||
stderr: bytes,
|
||||
exit_code: int = 7,
|
||||
) -> None:
|
||||
super().__init__(manifest)
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
self.exit_code = exit_code
|
||||
|
||||
async def exec(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
user: str | User | None = None,
|
||||
shell: bool | list[str] = False,
|
||||
) -> ExecResult:
|
||||
self.exec_users.append(user.name if isinstance(user, User) else user)
|
||||
rendered_command = " ".join(str(part) for part in command)
|
||||
self.exec_calls.append((rendered_command, timeout, shell))
|
||||
return ExecResult(stdout=self.stdout, stderr=self.stderr, exit_code=self.exit_code)
|
||||
def _pty_session(
|
||||
steps: list[dict[str, object]],
|
||||
*,
|
||||
manifest: Manifest | None = None,
|
||||
) -> Any:
|
||||
return scripted_sandbox_session(
|
||||
cast(Any, steps),
|
||||
manifest=manifest or Manifest(root="/workspace"),
|
||||
)
|
||||
|
||||
|
||||
class _PtyShellSession(_ShellSession):
|
||||
def __init__(self, manifest: Manifest) -> None:
|
||||
super().__init__(manifest)
|
||||
self._next_session_id = 1337
|
||||
self._live_sessions: set[int] = set()
|
||||
self.last_exec_yield_time_s: float | None = None
|
||||
self.last_exec_user: str | None = None
|
||||
self.last_write_yield_time_s: float | None = None
|
||||
|
||||
def supports_pty(self) -> bool:
|
||||
return True
|
||||
|
||||
async def pty_exec_start(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
shell: bool | list[str] = True,
|
||||
user: str | User | None = None,
|
||||
tty: bool = False,
|
||||
yield_time_s: float | None = None,
|
||||
max_output_tokens: int | None = None,
|
||||
) -> PtyExecUpdate:
|
||||
_ = (command, timeout, shell, tty, max_output_tokens)
|
||||
self.last_exec_user = user.name if isinstance(user, User) else user
|
||||
self.last_exec_yield_time_s = yield_time_s
|
||||
session_id = self._next_session_id
|
||||
self._next_session_id += 1
|
||||
self._live_sessions.add(session_id)
|
||||
return PtyExecUpdate(
|
||||
process_id=session_id,
|
||||
output=b"",
|
||||
exit_code=None,
|
||||
original_token_count=None,
|
||||
)
|
||||
|
||||
async def pty_write_stdin(
|
||||
self,
|
||||
*,
|
||||
session_id: int,
|
||||
chars: str,
|
||||
yield_time_s: float | None = None,
|
||||
max_output_tokens: int | None = None,
|
||||
) -> PtyExecUpdate:
|
||||
_ = max_output_tokens
|
||||
self.last_write_yield_time_s = yield_time_s
|
||||
if session_id not in self._live_sessions:
|
||||
raise PtySessionNotFoundError(session_id=session_id)
|
||||
|
||||
self._live_sessions.discard(session_id)
|
||||
return PtyExecUpdate(
|
||||
process_id=None,
|
||||
output=chars.encode("utf-8", errors="replace"),
|
||||
exit_code=0,
|
||||
original_token_count=None,
|
||||
)
|
||||
|
||||
|
||||
class _PtyNoStdinShellSession(_PtyShellSession):
|
||||
async def pty_write_stdin(
|
||||
self,
|
||||
*,
|
||||
session_id: int,
|
||||
chars: str,
|
||||
yield_time_s: float | None = None,
|
||||
max_output_tokens: int | None = None,
|
||||
) -> PtyExecUpdate:
|
||||
_ = (chars, yield_time_s, max_output_tokens)
|
||||
if session_id not in self._live_sessions:
|
||||
raise PtySessionNotFoundError(session_id=session_id)
|
||||
raise RuntimeError("stdin is not available for this process")
|
||||
|
||||
|
||||
class _PtyUnexpectedStdinErrorShellSession(_PtyShellSession):
|
||||
async def pty_write_stdin(
|
||||
self,
|
||||
*,
|
||||
session_id: int,
|
||||
chars: str,
|
||||
yield_time_s: float | None = None,
|
||||
max_output_tokens: int | None = None,
|
||||
) -> PtyExecUpdate:
|
||||
_ = (session_id, chars, yield_time_s, max_output_tokens)
|
||||
raise RuntimeError("unexpected stdin failure")
|
||||
|
||||
|
||||
class _PtyTransportFailingShellSession(_OutputShellSession):
|
||||
def __init__(
|
||||
self,
|
||||
manifest: Manifest,
|
||||
*,
|
||||
stdout: bytes = b"",
|
||||
stderr: bytes = b"",
|
||||
exit_code: int = 0,
|
||||
transport_context: dict[str, object] | None = None,
|
||||
) -> None:
|
||||
super().__init__(manifest, stdout=stdout, stderr=stderr, exit_code=exit_code)
|
||||
self.transport_context = transport_context or {}
|
||||
self.exec_call_count = 0
|
||||
|
||||
def supports_pty(self) -> bool:
|
||||
return True
|
||||
|
||||
async def exec(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
user: str | User | None = None,
|
||||
shell: bool | list[str] = False,
|
||||
) -> ExecResult:
|
||||
self.exec_call_count += 1
|
||||
return await super().exec(*command, timeout=timeout, user=user, shell=shell)
|
||||
|
||||
async def pty_exec_start(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
shell: bool | list[str] = True,
|
||||
user: str | User | None = None,
|
||||
tty: bool = False,
|
||||
yield_time_s: float | None = None,
|
||||
max_output_tokens: int | None = None,
|
||||
) -> PtyExecUpdate:
|
||||
_ = (timeout, shell, user, tty, yield_time_s, max_output_tokens)
|
||||
raise ExecTransportError(
|
||||
command=command,
|
||||
context=self.transport_context,
|
||||
cause=RuntimeError("connection closed while reading HTTP status line"),
|
||||
)
|
||||
def _transport_error(context: dict[str, object]) -> ExecTransportError:
|
||||
return ExecTransportError(
|
||||
command=("pwd",),
|
||||
context=context,
|
||||
cause=RuntimeError("connection closed while reading HTTP status line"),
|
||||
)
|
||||
|
||||
|
||||
def _patch_shell_tool_clock(
|
||||
@@ -286,7 +101,7 @@ class TestShellCapability:
|
||||
|
||||
def test_tools_exposes_exec_command_function_tool_after_bind(self) -> None:
|
||||
capability = Shell()
|
||||
capability.bind(_ShellSession(Manifest(root="/workspace")))
|
||||
capability.bind(_shell_session())
|
||||
|
||||
tools = capability.tools()
|
||||
|
||||
@@ -297,7 +112,7 @@ class TestShellCapability:
|
||||
|
||||
def test_tools_exposes_write_stdin_for_pty_sessions(self) -> None:
|
||||
capability = Shell()
|
||||
capability.bind(_PtyShellSession(Manifest(root="/workspace")))
|
||||
capability.bind(_pty_session([{"method": "pty_write_stdin", "result": None}]))
|
||||
|
||||
tools = capability.tools()
|
||||
|
||||
@@ -307,6 +122,17 @@ class TestShellCapability:
|
||||
assert tools[0].name == "exec_command"
|
||||
assert tools[1].name == "write_stdin"
|
||||
|
||||
def test_tools_keep_both_pty_session_methods_callable(self) -> None:
|
||||
capability = Shell()
|
||||
session = _pty_session([{"method": "pty_exec_start", "result": None}])
|
||||
capability.bind(session)
|
||||
|
||||
tools = capability.tools()
|
||||
|
||||
assert len(tools) == 2
|
||||
assert hasattr(session, "pty_exec_start")
|
||||
assert hasattr(session, "pty_write_stdin")
|
||||
|
||||
def test_configure_tools_can_customize_shell_approvals_after_clone(self) -> None:
|
||||
async def exec_command_needs_approval(
|
||||
_ctx: Any, params: dict[str, Any], _call_id: str
|
||||
@@ -324,7 +150,7 @@ class TestShellCapability:
|
||||
toolset.write_stdin.needs_approval = write_stdin_needs_approval
|
||||
|
||||
capability = Shell(configure_tools=configure_tools).clone()
|
||||
capability.bind(_PtyShellSession(Manifest(root="/workspace")))
|
||||
capability.bind(_pty_session([{"method": "pty_write_stdin", "result": None}]))
|
||||
|
||||
tools = capability.tools()
|
||||
exec_command_tool = cast(ExecCommandTool, tools[0])
|
||||
@@ -341,7 +167,7 @@ class TestShellCapability:
|
||||
saw_missing_write_stdin = toolset.write_stdin is None
|
||||
|
||||
capability = Shell(configure_tools=configure_tools)
|
||||
capability.bind(_ShellSession(Manifest(root="/workspace")))
|
||||
capability.bind(_shell_session())
|
||||
|
||||
tools = capability.tools()
|
||||
|
||||
@@ -361,7 +187,7 @@ class TestShellCapability:
|
||||
toolset.exec_command = replacement_exec_command
|
||||
|
||||
capability = Shell(configure_tools=configure_tools)
|
||||
capability.bind(_ShellSession(Manifest(root="/workspace")))
|
||||
capability.bind(_shell_session())
|
||||
|
||||
tools = capability.tools()
|
||||
exec_command_tool = cast(ExecCommandTool, tools[0])
|
||||
@@ -392,7 +218,7 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
capability = Shell()
|
||||
session = _ShellSession(Manifest(root="/workspace"))
|
||||
session = _shell_session()
|
||||
capability.bind(session)
|
||||
tool = cast(FunctionTool, capability.tools()[0])
|
||||
|
||||
@@ -412,7 +238,9 @@ class TestShellCapability:
|
||||
ExecCommandArgs(cmd="pwd", yield_time_ms=1500).model_dump_json(),
|
||||
)
|
||||
|
||||
assert session.exec_calls == [("pwd", 1.5, True)]
|
||||
assert session.calls[0].args == ("pwd",)
|
||||
assert session.calls[0].kwargs["timeout"] == 1.5
|
||||
assert session.calls[0].kwargs["shell"] is True
|
||||
assert (
|
||||
output == "Chunk ID: 123456\n"
|
||||
"Wall time: 0.2500 seconds\n"
|
||||
@@ -425,7 +253,14 @@ class TestShellCapability:
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_command_tool_runs_as_bound_user(self) -> None:
|
||||
capability = Shell()
|
||||
session = _ShellSession(Manifest(root="/workspace"))
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{
|
||||
"method": "exec",
|
||||
"result": ExecResult(stdout=b"", stderr=b"", exit_code=0),
|
||||
}
|
||||
]
|
||||
)
|
||||
capability.bind(session)
|
||||
capability.bind_run_as(User(name="sandbox-user"))
|
||||
tool = cast(FunctionTool, capability.tools()[0])
|
||||
@@ -435,7 +270,8 @@ class TestShellCapability:
|
||||
ExecCommandArgs(cmd="pwd").model_dump_json(),
|
||||
)
|
||||
|
||||
assert session.exec_users == ["sandbox-user"]
|
||||
assert session.calls[0].kwargs["user"] == User(name="sandbox-user")
|
||||
session.assert_complete()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_command_tool_includes_original_token_count_when_truncating(
|
||||
@@ -443,7 +279,7 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
capability = Shell()
|
||||
session = _ShellSession(Manifest(root="/workspace"))
|
||||
session = _shell_session()
|
||||
capability.bind(session)
|
||||
tool = cast(FunctionTool, capability.tools()[0])
|
||||
|
||||
@@ -478,7 +314,7 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
capability = Shell()
|
||||
session = _ShellSession(Manifest(root="/workspace"))
|
||||
session = _shell_session()
|
||||
capability.bind(session)
|
||||
tool = cast(FunctionTool, capability.tools()[0])
|
||||
_patch_shell_tool_clock(
|
||||
@@ -498,9 +334,9 @@ class TestShellCapability:
|
||||
).model_dump_json(),
|
||||
)
|
||||
|
||||
assert session.exec_calls == [
|
||||
("cd /workspace/src/project && pwd", 10.0, ["/bin/bash", "-c"])
|
||||
]
|
||||
assert session.calls[0].args == ("cd /workspace/src/project && pwd",)
|
||||
assert session.calls[0].kwargs["timeout"] == 10.0
|
||||
assert session.calls[0].kwargs["shell"] == ["/bin/bash", "-c"]
|
||||
assert (
|
||||
output == "Chunk ID: 876543\n"
|
||||
"Wall time: 0.1250 seconds\n"
|
||||
@@ -516,8 +352,8 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
capability = Shell()
|
||||
session = _ShellSession(
|
||||
Manifest(
|
||||
session = _shell_session(
|
||||
manifest=Manifest(
|
||||
root="/workspace",
|
||||
extra_path_grants=(
|
||||
SandboxPathGrant(
|
||||
@@ -547,7 +383,9 @@ class TestShellCapability:
|
||||
).model_dump_json(),
|
||||
)
|
||||
|
||||
assert session.exec_calls == [("cd /mnt/shared-data && pwd", 10.0, ["/bin/bash", "-c"])]
|
||||
assert session.calls[0].args == ("cd /mnt/shared-data && pwd",)
|
||||
assert session.calls[0].kwargs["timeout"] == 10.0
|
||||
assert session.calls[0].kwargs["shell"] == ["/bin/bash", "-c"]
|
||||
assert (
|
||||
output == "Chunk ID: 111111\n"
|
||||
"Wall time: 0.2500 seconds\n"
|
||||
@@ -563,7 +401,19 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
capability = Shell()
|
||||
session = _PtyShellSession(Manifest(root="/workspace"))
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_exec_start",
|
||||
"result": PtyExecUpdate(
|
||||
process_id=1337,
|
||||
output=b"",
|
||||
exit_code=None,
|
||||
original_token_count=None,
|
||||
),
|
||||
}
|
||||
]
|
||||
)
|
||||
capability.bind(session)
|
||||
tool = cast(FunctionTool, capability.tools()[0])
|
||||
_patch_shell_tool_clock(
|
||||
@@ -578,7 +428,7 @@ class TestShellCapability:
|
||||
ExecCommandArgs(cmd="pwd", yield_time_ms=0, tty=True).model_dump_json(),
|
||||
)
|
||||
|
||||
assert session.last_exec_yield_time_s == 0.0
|
||||
assert session.calls[0].kwargs["yield_time_s"] == 0.0
|
||||
assert (
|
||||
output == "Chunk ID: abcdef\n"
|
||||
"Wall time: 0.0500 seconds\n"
|
||||
@@ -590,7 +440,19 @@ class TestShellCapability:
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_command_tool_starts_pty_as_bound_user(self) -> None:
|
||||
capability = Shell()
|
||||
session = _PtyShellSession(Manifest(root="/workspace"))
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_exec_start",
|
||||
"result": PtyExecUpdate(
|
||||
process_id=1337,
|
||||
output=b"",
|
||||
exit_code=None,
|
||||
original_token_count=None,
|
||||
),
|
||||
}
|
||||
]
|
||||
)
|
||||
capability.bind(session)
|
||||
capability.bind_run_as(User(name="sandbox-user"))
|
||||
tool = cast(FunctionTool, capability.tools()[0])
|
||||
@@ -600,7 +462,7 @@ class TestShellCapability:
|
||||
ExecCommandArgs(cmd="pwd", yield_time_ms=0, tty=True).model_dump_json(),
|
||||
)
|
||||
|
||||
assert session.last_exec_user == "sandbox-user"
|
||||
assert session.calls[0].kwargs["user"] == User(name="sandbox-user")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_command_tool_formats_timeout_without_exit_code(
|
||||
@@ -608,7 +470,7 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
capability = Shell()
|
||||
session = _TimeoutShellSession(Manifest(root="/workspace"))
|
||||
session = _shell_session(error=ExecTimeoutError(command=("sleep 30",), timeout_s=0.005))
|
||||
capability.bind(session)
|
||||
tool = cast(FunctionTool, capability.tools()[0])
|
||||
_patch_shell_tool_clock(
|
||||
@@ -635,13 +497,19 @@ class TestShellCapability:
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
tool = ExecCommandTool(
|
||||
session=_PtyTransportFailingShellSession(
|
||||
Manifest(root="/workspace"),
|
||||
stdout=b"fallback ok",
|
||||
transport_context={"stage": "open_pipe", "retry_safe": True},
|
||||
)
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_exec_start",
|
||||
"error": _transport_error({"stage": "open_pipe", "retry_safe": True}),
|
||||
},
|
||||
{
|
||||
"method": "exec",
|
||||
"result": ExecResult(stdout=b"fallback ok", stderr=b"", exit_code=0),
|
||||
},
|
||||
]
|
||||
)
|
||||
tool = ExecCommandTool(session=session)
|
||||
_patch_shell_tool_clock(
|
||||
monkeypatch,
|
||||
chunk_id="44444444444444444444444444444444",
|
||||
@@ -661,12 +529,17 @@ class TestShellCapability:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exec_command_tool_does_not_fall_back_for_tty_sessions(self) -> None:
|
||||
tool = ExecCommandTool(
|
||||
session=_PtyTransportFailingShellSession(
|
||||
Manifest(root="/workspace"),
|
||||
transport_context={"stage": "open_pipe", "retry_safe": True, "tty": True},
|
||||
)
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_exec_start",
|
||||
"error": _transport_error(
|
||||
{"stage": "open_pipe", "retry_safe": True, "tty": True}
|
||||
),
|
||||
}
|
||||
]
|
||||
)
|
||||
tool = ExecCommandTool(session=session)
|
||||
|
||||
with pytest.raises(ExecTransportError):
|
||||
await tool.on_invoke_tool(
|
||||
@@ -678,12 +551,15 @@ class TestShellCapability:
|
||||
async def test_exec_command_tool_does_not_fall_back_for_non_retry_safe_transport_errors(
|
||||
self,
|
||||
) -> None:
|
||||
tool = ExecCommandTool(
|
||||
session=_PtyTransportFailingShellSession(
|
||||
Manifest(root="/workspace"),
|
||||
transport_context={"stage": "open_pipe"},
|
||||
)
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_exec_start",
|
||||
"error": _transport_error({"stage": "open_pipe"}),
|
||||
}
|
||||
]
|
||||
)
|
||||
tool = ExecCommandTool(session=session)
|
||||
|
||||
with pytest.raises(ExecTransportError):
|
||||
await tool.on_invoke_tool(
|
||||
@@ -697,10 +573,8 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
tool = ExecCommandTool(
|
||||
session=_OutputShellSession(
|
||||
Manifest(root="/workspace"),
|
||||
stdout=b"stdout only\n",
|
||||
stderr=b"",
|
||||
session=_shell_session(
|
||||
result=ExecResult(stdout=b"stdout only\n", stderr=b"", exit_code=7)
|
||||
)
|
||||
)
|
||||
_patch_shell_tool_clock(
|
||||
@@ -729,10 +603,8 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
tool = ExecCommandTool(
|
||||
session=_OutputShellSession(
|
||||
Manifest(root="/workspace"),
|
||||
stdout=b"",
|
||||
stderr=b"stderr only\n",
|
||||
session=_shell_session(
|
||||
result=ExecResult(stdout=b"", stderr=b"stderr only\n", exit_code=7)
|
||||
)
|
||||
)
|
||||
_patch_shell_tool_clock(
|
||||
@@ -761,10 +633,12 @@ class TestShellCapability:
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
tool = ExecCommandTool(
|
||||
session=_OutputShellSession(
|
||||
Manifest(root="/workspace"),
|
||||
stdout=b"stdout line\n",
|
||||
stderr=b"stderr line\n",
|
||||
session=_shell_session(
|
||||
result=ExecResult(
|
||||
stdout=b"stdout line\n",
|
||||
stderr=b"stderr line\n",
|
||||
exit_code=7,
|
||||
)
|
||||
)
|
||||
)
|
||||
_patch_shell_tool_clock(
|
||||
@@ -793,8 +667,19 @@ class TestShellCapability:
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
session = _PtyShellSession(Manifest(root="/workspace"))
|
||||
session._live_sessions.add(1337)
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_write_stdin",
|
||||
"result": PtyExecUpdate(
|
||||
process_id=None,
|
||||
output=b"hello",
|
||||
exit_code=0,
|
||||
original_token_count=None,
|
||||
),
|
||||
}
|
||||
]
|
||||
)
|
||||
tool = WriteStdinTool(session=session)
|
||||
_patch_shell_tool_clock(
|
||||
monkeypatch,
|
||||
@@ -818,7 +703,7 @@ class TestShellCapability:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_stdin_tool_rejects_non_pty_sessions(self) -> None:
|
||||
tool = WriteStdinTool(session=_ShellSession(Manifest(root="/workspace")))
|
||||
tool = WriteStdinTool(session=_shell_session())
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError, match="write_stdin is not available for non-PTY sandboxes"
|
||||
@@ -833,7 +718,15 @@ class TestShellCapability:
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
tool = WriteStdinTool(session=_PtyShellSession(Manifest(root="/workspace")))
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_write_stdin",
|
||||
"error": PtySessionNotFoundError(session_id=9999),
|
||||
}
|
||||
]
|
||||
)
|
||||
tool = WriteStdinTool(session=session)
|
||||
_patch_shell_tool_clock(
|
||||
monkeypatch,
|
||||
chunk_id="66666666666666666666666666666666",
|
||||
@@ -859,8 +752,14 @@ class TestShellCapability:
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
session = _PtyNoStdinShellSession(Manifest(root="/workspace"))
|
||||
session._live_sessions.add(1337)
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_write_stdin",
|
||||
"error": RuntimeError("stdin is not available for this process"),
|
||||
}
|
||||
]
|
||||
)
|
||||
tool = WriteStdinTool(session=session)
|
||||
_patch_shell_tool_clock(
|
||||
monkeypatch,
|
||||
@@ -885,9 +784,15 @@ class TestShellCapability:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_stdin_tool_reraises_unexpected_runtime_error(self) -> None:
|
||||
tool = WriteStdinTool(
|
||||
session=_PtyUnexpectedStdinErrorShellSession(Manifest(root="/workspace"))
|
||||
session = _pty_session(
|
||||
[
|
||||
{
|
||||
"method": "pty_write_stdin",
|
||||
"error": RuntimeError("unexpected stdin failure"),
|
||||
}
|
||||
]
|
||||
)
|
||||
tool = WriteStdinTool(session=session)
|
||||
|
||||
with pytest.raises(RuntimeError, match="unexpected stdin failure"):
|
||||
await tool.on_invoke_tool(
|
||||
|
||||
@@ -21,6 +21,7 @@ from agents.sandbox.session.sandbox_session import SandboxSession
|
||||
from agents.sandbox.snapshot import NoopSnapshot
|
||||
from agents.sandbox.types import ExecResult, FileMode, Group, Permissions, User
|
||||
from agents.sandbox.workspace_paths import coerce_posix_path, sandbox_path_str
|
||||
from agents.testing import scripted_sandbox_session
|
||||
from agents.tool import FunctionTool
|
||||
from agents.tool_context import ToolContext
|
||||
from agents.tracing import trace
|
||||
@@ -644,7 +645,11 @@ class TestSkillsLazyLoading:
|
||||
skill_dir.mkdir(parents=True)
|
||||
(skill_dir / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
|
||||
capability = Skills(lazy_from=LocalDirLazySkillSource(source=LocalDir(src=src_root)))
|
||||
capability.bind(_SkillsSession(_source_granted_manifest(workspace_root, source=src_root)))
|
||||
capability.bind(
|
||||
scripted_sandbox_session(
|
||||
manifest=_source_granted_manifest(workspace_root, source=src_root)
|
||||
)
|
||||
)
|
||||
|
||||
tools = capability.tools()
|
||||
|
||||
@@ -785,8 +790,10 @@ class TestSkillsLazyLoading:
|
||||
lazy_from=LocalDirLazySkillSource(source=LocalDir(src=tmp_path / "missing-skills"))
|
||||
)
|
||||
capability.bind(
|
||||
_SkillsSession(
|
||||
_source_granted_manifest(workspace_root, source=tmp_path / "missing-skills")
|
||||
scripted_sandbox_session(
|
||||
manifest=_source_granted_manifest(
|
||||
workspace_root, source=tmp_path / "missing-skills"
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -811,7 +818,11 @@ class TestSkillsLazyLoading:
|
||||
encoding="utf-8",
|
||||
)
|
||||
capability = Skills(lazy_from=LocalDirLazySkillSource(source=LocalDir(src=src_root)))
|
||||
capability.bind(_SkillsSession(_source_granted_manifest(workspace_root, source=src_root)))
|
||||
capability.bind(
|
||||
scripted_sandbox_session(
|
||||
manifest=_source_granted_manifest(workspace_root, source=src_root)
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(SkillsConfigError):
|
||||
await capability.load_skill("shared-skill")
|
||||
@@ -840,7 +851,11 @@ class TestSkillsLazyLoading:
|
||||
second_instructions = await capability.instructions(
|
||||
_source_granted_manifest(workspace_root, source=src_root)
|
||||
)
|
||||
capability.bind(_SkillsSession(_source_granted_manifest(workspace_root, source=src_root)))
|
||||
capability.bind(
|
||||
scripted_sandbox_session(
|
||||
manifest=_source_granted_manifest(workspace_root, source=src_root)
|
||||
)
|
||||
)
|
||||
third_instructions = await capability.instructions(
|
||||
_source_granted_manifest(workspace_root, source=src_root)
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
@@ -11,12 +10,10 @@ import pytest
|
||||
from agents.sandbox import Manifest
|
||||
from agents.sandbox.capabilities.tools import ViewImageTool
|
||||
from agents.sandbox.errors import WorkspaceReadNotFoundError
|
||||
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
|
||||
from agents.sandbox.snapshot import NoopSnapshot
|
||||
from agents.sandbox.types import ExecResult, User
|
||||
from agents.sandbox.types import User
|
||||
from agents.testing import scripted_sandbox_session
|
||||
from agents.tool import ToolOutputImage
|
||||
from agents.tool_context import ToolContext
|
||||
from tests.utils.factories import TestSessionState
|
||||
|
||||
_MAX_IMAGE_BYTES = 10 * 1024 * 1024
|
||||
_PNG_BASE64 = (
|
||||
@@ -25,76 +22,9 @@ _PNG_BASE64 = (
|
||||
_PNG_BYTES = base64.b64decode(_PNG_BASE64)
|
||||
|
||||
|
||||
class _ImageSession(BaseSandboxSession):
|
||||
def __init__(self, manifest: Manifest) -> None:
|
||||
self.state = TestSessionState(
|
||||
manifest=manifest,
|
||||
snapshot=NoopSnapshot(id=str(uuid.uuid4())),
|
||||
)
|
||||
self.files: dict[Path, bytes] = {}
|
||||
self.read_users: list[str | None] = []
|
||||
|
||||
async def start(self) -> None:
|
||||
return None
|
||||
|
||||
async def stop(self) -> None:
|
||||
return None
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
return None
|
||||
|
||||
async def running(self) -> bool:
|
||||
return True
|
||||
|
||||
async def read(self, path: Path, *, user: str | User | None = None) -> io.BytesIO:
|
||||
self.read_users.append(user.name if isinstance(user, User) else user)
|
||||
normalized = self.normalize_path(path)
|
||||
if normalized not in self.files:
|
||||
raise FileNotFoundError(normalized)
|
||||
return io.BytesIO(self.files[normalized])
|
||||
|
||||
async def write(
|
||||
self,
|
||||
path: Path,
|
||||
data: io.IOBase,
|
||||
*,
|
||||
user: str | User | None = None,
|
||||
) -> None:
|
||||
_ = user
|
||||
normalized = self.normalize_path(path)
|
||||
payload = data.read()
|
||||
if isinstance(payload, str):
|
||||
self.files[normalized] = payload.encode("utf-8")
|
||||
else:
|
||||
self.files[normalized] = bytes(payload)
|
||||
|
||||
async def _exec_internal(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
) -> ExecResult:
|
||||
_ = (command, timeout)
|
||||
raise AssertionError("_exec_internal() should not be called")
|
||||
|
||||
async def persist_workspace(self) -> io.IOBase:
|
||||
return io.BytesIO()
|
||||
|
||||
async def hydrate_workspace(self, data: io.IOBase) -> None:
|
||||
_ = data
|
||||
|
||||
|
||||
class _ProviderNotFoundImageSession(_ImageSession):
|
||||
async def read(self, path: Path, *, user: str | User | None = None) -> io.BytesIO:
|
||||
self.read_users.append(user.name if isinstance(user, User) else user)
|
||||
normalized = self.normalize_path(path)
|
||||
if normalized in self.files:
|
||||
return io.BytesIO(self.files[normalized])
|
||||
raise WorkspaceReadNotFoundError(path=normalized)
|
||||
|
||||
|
||||
class TestViewImageTool:
|
||||
def test_view_image_accepts_needs_approval_setting(self) -> None:
|
||||
session = _ImageSession(Manifest(root="/workspace"))
|
||||
session = scripted_sandbox_session()
|
||||
|
||||
async def needs_approval(_ctx: object, params: dict[str, object], _call_id: str) -> bool:
|
||||
return str(params["path"]).startswith("sensitive/")
|
||||
@@ -105,8 +35,7 @@ class TestViewImageTool:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_view_image_returns_tool_output_image_for_png(self) -> None:
|
||||
session = _ImageSession(Manifest(root="/workspace"))
|
||||
session.files[Path("/workspace/images/dot.png")] = _PNG_BYTES
|
||||
session = scripted_sandbox_session([{"method": "read", "result": io.BytesIO(_PNG_BYTES)}])
|
||||
tool = ViewImageTool(session=session)
|
||||
|
||||
output = await tool.on_invoke_tool(
|
||||
@@ -117,11 +46,11 @@ class TestViewImageTool:
|
||||
assert isinstance(output, ToolOutputImage)
|
||||
assert output.image_url == f"data:image/png;base64,{_PNG_BASE64}"
|
||||
assert output.detail is None
|
||||
session.assert_complete()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_view_image_reads_as_bound_user(self) -> None:
|
||||
session = _ImageSession(Manifest(root="/workspace"))
|
||||
session.files[Path("/workspace/images/dot.png")] = _PNG_BYTES
|
||||
session = scripted_sandbox_session([{"method": "read", "result": io.BytesIO(_PNG_BYTES)}])
|
||||
tool = ViewImageTool(session=session, user=User(name="sandbox-user"))
|
||||
|
||||
output = await tool.on_invoke_tool(
|
||||
@@ -130,12 +59,12 @@ class TestViewImageTool:
|
||||
)
|
||||
|
||||
assert isinstance(output, ToolOutputImage)
|
||||
assert session.read_users == ["sandbox-user"]
|
||||
assert session.calls[0].kwargs["user"] == User(name="sandbox-user")
|
||||
session.assert_complete()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_view_image_rejects_non_image_files(self) -> None:
|
||||
session = _ImageSession(Manifest(root="/workspace"))
|
||||
session.files[Path("/workspace/notes.txt")] = b"hello\n"
|
||||
session = scripted_sandbox_session([{"method": "read", "result": io.BytesIO(b"hello\n")}])
|
||||
tool = ViewImageTool(session=session)
|
||||
|
||||
output = await tool.on_invoke_tool(
|
||||
@@ -144,12 +73,17 @@ class TestViewImageTool:
|
||||
)
|
||||
|
||||
assert output == "image path `notes.txt` is not a supported image file"
|
||||
session.assert_complete()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_view_image_rejects_images_larger_than_10mb(self) -> None:
|
||||
session = _ImageSession(Manifest(root="/workspace"))
|
||||
session.files[Path("/workspace/images/huge.png")] = b"\x89PNG\r\n\x1a\n" + (
|
||||
b"0" * (_MAX_IMAGE_BYTES + 1)
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{
|
||||
"method": "read",
|
||||
"result": io.BytesIO(b"\x89PNG\r\n\x1a\n" + (b"0" * (_MAX_IMAGE_BYTES + 1))),
|
||||
}
|
||||
]
|
||||
)
|
||||
tool = ViewImageTool(session=session)
|
||||
|
||||
@@ -162,14 +96,24 @@ class TestViewImageTool:
|
||||
"image path `images/huge.png` exceeded the allowed size of 10MB; "
|
||||
"resize or compress the image and try again"
|
||||
)
|
||||
session.assert_complete()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_view_image_rejection_text_does_not_expose_provider_path(self) -> None:
|
||||
provider_root = Path("/provider/private/root")
|
||||
session = _ProviderNotFoundImageSession(Manifest(root=str(provider_root)))
|
||||
session.files[provider_root / "notes.txt"] = b"hello\n"
|
||||
session.files[provider_root / "images/huge.png"] = b"\x89PNG\r\n\x1a\n" + (
|
||||
b"0" * (_MAX_IMAGE_BYTES + 1)
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{
|
||||
"method": "read",
|
||||
"error": WorkspaceReadNotFoundError(path=provider_root / "images/missing.png"),
|
||||
},
|
||||
{"method": "read", "result": io.BytesIO(b"hello\n")},
|
||||
{
|
||||
"method": "read",
|
||||
"result": io.BytesIO(b"\x89PNG\r\n\x1a\n" + (b"0" * (_MAX_IMAGE_BYTES + 1))),
|
||||
},
|
||||
],
|
||||
manifest=Manifest(root=str(provider_root)),
|
||||
)
|
||||
tool = ViewImageTool(session=session)
|
||||
|
||||
|
||||
@@ -5,19 +5,19 @@ from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
from agents.items import TResponseOutputItem
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_final_output_message, get_function_tool_call
|
||||
|
||||
__test__ = False
|
||||
|
||||
|
||||
class TestModel(FakeModel):
|
||||
class TestModel(ScriptedModel):
|
||||
"""Reusable queued model for sandbox integration tests."""
|
||||
|
||||
__test__ = False
|
||||
|
||||
def queue_turn(self, *items: TResponseOutputItem) -> None:
|
||||
self.set_next_output(list(items))
|
||||
self.enqueue(list(items))
|
||||
|
||||
def queue_function_call(
|
||||
self,
|
||||
|
||||
+144
-123
@@ -75,7 +75,7 @@ from agents.sandbox.memory.storage import (
|
||||
)
|
||||
from agents.sandbox.runtime import _stream_memory_input_override
|
||||
from agents.sandbox.sandboxes.unix_local import UnixLocalSandboxClient
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_final_output_message, get_text_message
|
||||
from tests.utils.hitl import make_shell_call
|
||||
|
||||
@@ -148,8 +148,8 @@ def _memory_config(
|
||||
extra_prompt: str | None = None,
|
||||
layout: MemoryLayoutConfig | None = None,
|
||||
read: MemoryReadConfig | None = None,
|
||||
phase_one_model: FakeModel | None = None,
|
||||
phase_two_model: FakeModel | None = None,
|
||||
phase_one_model: ScriptedModel | None = None,
|
||||
phase_two_model: ScriptedModel | None = None,
|
||||
) -> Memory:
|
||||
return Memory(
|
||||
layout=layout or MemoryLayoutConfig(),
|
||||
@@ -157,14 +157,16 @@ def _memory_config(
|
||||
generate=MemoryGenerateConfig(
|
||||
max_raw_memories_for_consolidation=max_raw_memories_for_consolidation,
|
||||
extra_prompt=extra_prompt,
|
||||
phase_one_model=phase_one_model or FakeModel(initial_output=[_phase_one_message()]),
|
||||
phase_one_model=phase_one_model or ScriptedModel(steps=[[_phase_one_message()]]),
|
||||
phase_two_model=phase_two_model
|
||||
or FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "memory entry"),
|
||||
_patch_update_call(
|
||||
"memory-summary", "memories/memory_summary.md", "summary entry"
|
||||
),
|
||||
or ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "memory entry"),
|
||||
_patch_update_call(
|
||||
"memory-summary", "memories/memory_summary.md", "summary entry"
|
||||
),
|
||||
]
|
||||
]
|
||||
),
|
||||
),
|
||||
@@ -175,13 +177,12 @@ def _run_config_for_session(session: Any) -> RunConfig:
|
||||
return RunConfig(sandbox=SandboxRunConfig(session=session))
|
||||
|
||||
|
||||
def _extract_user_text(fake_model: FakeModel) -> str:
|
||||
assert fake_model.first_turn_args is not None
|
||||
return _extract_user_text_from_turn_args(fake_model.first_turn_args)
|
||||
def _extract_user_text(scripted_model: ScriptedModel) -> str:
|
||||
assert bool(scripted_model.calls)
|
||||
return _extract_user_text_from_model_input(scripted_model.calls[0].input)
|
||||
|
||||
|
||||
def _extract_user_text_from_turn_args(turn_args: dict[str, Any]) -> str:
|
||||
input_items = turn_args["input"]
|
||||
def _extract_user_text_from_model_input(input_items: str | list[Any]) -> str:
|
||||
assert isinstance(input_items, list)
|
||||
first_item = cast(dict[str, Any], input_items[0])
|
||||
content = first_item["content"]
|
||||
@@ -649,23 +650,25 @@ async def test_runner_memory_generation_sanitizes_and_truncates_phase_one_prompt
|
||||
monkeypatch.setattr(phase_one_module, "_PHASE_ONE_ROLLOUT_TOKEN_LIMIT", 1000)
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
phase_one_model = FakeModel(initial_output=[_phase_one_message()])
|
||||
phase_one_model = ScriptedModel(steps=[[_phase_one_message()]])
|
||||
memory = _memory_config(phase_one_model=phase_one_model)
|
||||
agent = SandboxAgent(
|
||||
name="worker",
|
||||
model=FakeModel(
|
||||
initial_output=[
|
||||
ResponseReasoningItem(id="rs_1", summary=[], type="reasoning"),
|
||||
cast(
|
||||
TResponseOutputItem,
|
||||
{
|
||||
"id": "compaction_1",
|
||||
"type": "compaction",
|
||||
"summary": "compacted-so-far",
|
||||
"encrypted_content": "encrypted",
|
||||
},
|
||||
),
|
||||
get_text_message("done"),
|
||||
model=ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
ResponseReasoningItem(id="rs_1", summary=[], type="reasoning"),
|
||||
cast(
|
||||
TResponseOutputItem,
|
||||
{
|
||||
"id": "compaction_1",
|
||||
"type": "compaction",
|
||||
"summary": "compacted-so-far",
|
||||
"encrypted_content": "encrypted",
|
||||
},
|
||||
),
|
||||
get_text_message("done"),
|
||||
]
|
||||
]
|
||||
),
|
||||
instructions="Worker.",
|
||||
@@ -694,7 +697,7 @@ async def test_runner_memory_generation_sanitizes_and_truncates_phase_one_prompt
|
||||
)
|
||||
|
||||
assert result.final_output == "done"
|
||||
assert phase_one_model.first_turn_args is None
|
||||
assert not phase_one_model.calls
|
||||
|
||||
await session.aclose()
|
||||
closed = True
|
||||
@@ -721,7 +724,7 @@ async def test_sandbox_agent_without_memory_capability_skips_memory_generation()
|
||||
session = await client.create(manifest=Manifest())
|
||||
agent = SandboxAgent(
|
||||
name="worker",
|
||||
model=FakeModel(initial_output=[get_final_output_message("done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
||||
instructions="Worker.",
|
||||
)
|
||||
|
||||
@@ -990,14 +993,16 @@ async def test_memory_capability_live_update_instructions() -> None:
|
||||
async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
phase_one_model = FakeModel(initial_output=[_phase_one_message()])
|
||||
phase_two_model = FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "memory entry"),
|
||||
_patch_update_call("memory-summary", "memories/memory_summary.md", "summary entry"),
|
||||
phase_one_model = ScriptedModel(steps=[[_phase_one_message()]])
|
||||
phase_two_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "memory entry"),
|
||||
_patch_update_call("memory-summary", "memories/memory_summary.md", "summary entry"),
|
||||
]
|
||||
]
|
||||
)
|
||||
phase_two_model.set_next_output([get_final_output_message("consolidated")])
|
||||
phase_two_model.enqueue([get_final_output_message("consolidated")])
|
||||
memory = _memory_config(
|
||||
extra_prompt="Track durable user preferences.",
|
||||
phase_one_model=phase_one_model,
|
||||
@@ -1005,7 +1010,7 @@ async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
|
||||
)
|
||||
agent = SandboxAgent(
|
||||
name="worker",
|
||||
model=FakeModel(initial_output=[get_final_output_message("done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
||||
instructions="Worker.",
|
||||
capabilities=[memory],
|
||||
)
|
||||
@@ -1023,7 +1028,7 @@ async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
|
||||
|
||||
assert result.final_output == "done"
|
||||
assert len(rollouts) == 1
|
||||
assert phase_one_model.first_turn_args is None
|
||||
assert not phase_one_model.calls
|
||||
|
||||
await session.aclose()
|
||||
closed = True
|
||||
@@ -1048,16 +1053,12 @@ async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
|
||||
assert "rollout_path: sessions/" in rollout_summaries[0].read_text()
|
||||
assert "terminal_state: completed" in rollout_summaries[0].read_text()
|
||||
assert '"terminal_state":"completed"' in _extract_user_text(phase_one_model)
|
||||
assert phase_one_model.first_turn_args is not None
|
||||
assert (
|
||||
"DEVELOPER-SPECIFIC EXTRA GUIDANCE"
|
||||
in phase_one_model.first_turn_args["system_instructions"]
|
||||
)
|
||||
assert (
|
||||
"Track durable user preferences."
|
||||
in phase_one_model.first_turn_args["system_instructions"]
|
||||
)
|
||||
assert phase_two_model.first_turn_args is not None
|
||||
assert bool(phase_one_model.calls)
|
||||
system_instructions = phase_one_model.calls[0].system_instructions
|
||||
assert system_instructions is not None
|
||||
assert "DEVELOPER-SPECIFIC EXTRA GUIDANCE" in system_instructions
|
||||
assert "Track durable user preferences." in system_instructions
|
||||
assert bool(phase_two_model.calls)
|
||||
assert "DEVELOPER-SPECIFIC EXTRA GUIDANCE" in _extract_user_text(phase_two_model)
|
||||
assert "Track durable user preferences." in _extract_user_text(phase_two_model)
|
||||
finally:
|
||||
@@ -1068,24 +1069,28 @@ async def test_sandbox_memory_writes_rollouts_and_memory_files() -> None:
|
||||
async def test_sandbox_memory_uses_custom_layout() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
phase_two_model = FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("memory-md", "agent_memory/MEMORY.md", "memory entry"),
|
||||
_patch_update_call("memory-summary", "agent_memory/memory_summary.md", "summary entry"),
|
||||
phase_two_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("memory-md", "agent_memory/MEMORY.md", "memory entry"),
|
||||
_patch_update_call(
|
||||
"memory-summary", "agent_memory/memory_summary.md", "summary entry"
|
||||
),
|
||||
]
|
||||
]
|
||||
)
|
||||
phase_two_model.set_next_output([get_final_output_message("consolidated")])
|
||||
phase_two_model.enqueue([get_final_output_message("consolidated")])
|
||||
memory = Memory(
|
||||
layout=MemoryLayoutConfig(memories_dir="agent_memory", sessions_dir="agent_sessions"),
|
||||
read=None,
|
||||
generate=MemoryGenerateConfig(
|
||||
phase_one_model=FakeModel(initial_output=[_phase_one_message()]),
|
||||
phase_one_model=ScriptedModel(steps=[[_phase_one_message()]]),
|
||||
phase_two_model=phase_two_model,
|
||||
),
|
||||
)
|
||||
agent = SandboxAgent(
|
||||
name="worker",
|
||||
model=FakeModel(initial_output=[get_final_output_message("done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
||||
instructions="Worker.",
|
||||
capabilities=[memory],
|
||||
)
|
||||
@@ -1114,47 +1119,51 @@ async def test_sandbox_memory_uses_custom_layout() -> None:
|
||||
async def test_sandbox_memory_supports_multiple_generating_layouts_in_one_session() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
phase_two_model_a = FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("a-memory", "agent_a_memory/MEMORY.md", "agent a entry"),
|
||||
_patch_update_call(
|
||||
"a-summary",
|
||||
"agent_a_memory/memory_summary.md",
|
||||
"agent a summary",
|
||||
),
|
||||
phase_two_model_a = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("a-memory", "agent_a_memory/MEMORY.md", "agent a entry"),
|
||||
_patch_update_call(
|
||||
"a-summary",
|
||||
"agent_a_memory/memory_summary.md",
|
||||
"agent a summary",
|
||||
),
|
||||
]
|
||||
]
|
||||
)
|
||||
phase_two_model_a.set_next_output([get_final_output_message("agent a consolidated")])
|
||||
phase_two_model_b = FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("b-memory", "agent_b_memory/MEMORY.md", "agent b entry"),
|
||||
_patch_update_call(
|
||||
"b-summary",
|
||||
"agent_b_memory/memory_summary.md",
|
||||
"agent b summary",
|
||||
),
|
||||
phase_two_model_a.enqueue([get_final_output_message("agent a consolidated")])
|
||||
phase_two_model_b = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("b-memory", "agent_b_memory/MEMORY.md", "agent b entry"),
|
||||
_patch_update_call(
|
||||
"b-summary",
|
||||
"agent_b_memory/memory_summary.md",
|
||||
"agent b summary",
|
||||
),
|
||||
]
|
||||
]
|
||||
)
|
||||
phase_two_model_b.set_next_output([get_final_output_message("agent b consolidated")])
|
||||
phase_two_model_b.enqueue([get_final_output_message("agent b consolidated")])
|
||||
memory_a = _memory_config(
|
||||
layout=MemoryLayoutConfig(memories_dir="agent_a_memory", sessions_dir="agent_a_sessions"),
|
||||
phase_one_model=FakeModel(initial_output=[_phase_one_message(raw_memory="agent a raw\n")]),
|
||||
phase_one_model=ScriptedModel(steps=[[_phase_one_message(raw_memory="agent a raw\n")]]),
|
||||
phase_two_model=phase_two_model_a,
|
||||
)
|
||||
memory_b = _memory_config(
|
||||
layout=MemoryLayoutConfig(memories_dir="agent_b_memory", sessions_dir="agent_b_sessions"),
|
||||
phase_one_model=FakeModel(initial_output=[_phase_one_message(raw_memory="agent b raw\n")]),
|
||||
phase_one_model=ScriptedModel(steps=[[_phase_one_message(raw_memory="agent b raw\n")]]),
|
||||
phase_two_model=phase_two_model_b,
|
||||
)
|
||||
agent_a = SandboxAgent(
|
||||
name="agent-a",
|
||||
model=FakeModel(initial_output=[get_final_output_message("a done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("a done")]]),
|
||||
instructions="Agent A.",
|
||||
capabilities=[memory_a],
|
||||
)
|
||||
agent_b = SandboxAgent(
|
||||
name="agent-b",
|
||||
model=FakeModel(initial_output=[get_final_output_message("b done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("b done")]]),
|
||||
instructions="Agent B.",
|
||||
capabilities=[memory_b],
|
||||
)
|
||||
@@ -1183,7 +1192,7 @@ async def test_sandbox_memory_rejects_different_generate_configs_for_same_layout
|
||||
session = await client.create(manifest=Manifest())
|
||||
memory = _memory_config()
|
||||
different_memory = _memory_config(
|
||||
phase_one_model=FakeModel(initial_output=[_phase_one_message(raw_memory="different\n")])
|
||||
phase_one_model=ScriptedModel(steps=[[_phase_one_message(raw_memory="different\n")]])
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -1266,27 +1275,31 @@ async def test_sandbox_memory_rejects_shared_sessions_dir_for_different_memories
|
||||
async def test_sandbox_memory_groups_segments_by_sdk_session_until_close() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
phase_one_model = FakeModel(initial_output=[_phase_one_message(raw_memory="joined raw\n")])
|
||||
phase_two_model = FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "joined entry"),
|
||||
_patch_update_call("memory-summary", "memories/memory_summary.md", "joined summary"),
|
||||
phase_one_model = ScriptedModel(steps=[[_phase_one_message(raw_memory="joined raw\n")]])
|
||||
phase_two_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "joined entry"),
|
||||
_patch_update_call(
|
||||
"memory-summary", "memories/memory_summary.md", "joined summary"
|
||||
),
|
||||
]
|
||||
]
|
||||
)
|
||||
phase_two_model.set_next_output([get_final_output_message("joined")])
|
||||
phase_two_model.enqueue([get_final_output_message("joined")])
|
||||
memory = _memory_config(
|
||||
phase_one_model=phase_one_model,
|
||||
phase_two_model=phase_two_model,
|
||||
)
|
||||
first_agent = SandboxAgent(
|
||||
name="first-worker",
|
||||
model=FakeModel(initial_output=[get_final_output_message("first done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("first done")]]),
|
||||
instructions="Worker.",
|
||||
capabilities=[memory],
|
||||
)
|
||||
second_agent = SandboxAgent(
|
||||
name="second-worker",
|
||||
model=FakeModel(initial_output=[get_final_output_message("second done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("second done")]]),
|
||||
instructions="Worker.",
|
||||
capabilities=[memory],
|
||||
)
|
||||
@@ -1324,7 +1337,7 @@ async def test_sandbox_memory_groups_segments_by_sdk_session_until_close() -> No
|
||||
]
|
||||
assert segments[0]["input"] == [{"content": "first", "role": "user"}]
|
||||
assert segments[1]["input"] == [{"content": "second", "role": "user"}]
|
||||
assert phase_one_model.first_turn_args is None
|
||||
assert not phase_one_model.calls
|
||||
|
||||
await session.aclose()
|
||||
closed = True
|
||||
@@ -1345,8 +1358,8 @@ async def test_sandbox_memory_groups_segments_by_sdk_session_until_close() -> No
|
||||
async def test_sandbox_memory_fallback_does_not_mutate_run_config() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
agent_model = FakeModel()
|
||||
agent_model.add_multiple_turn_outputs(
|
||||
agent_model = ScriptedModel()
|
||||
agent_model.extend(
|
||||
[
|
||||
[get_final_output_message("first done")],
|
||||
[get_final_output_message("second done")],
|
||||
@@ -1387,7 +1400,7 @@ async def test_sandbox_memory_uses_conversation_id_when_sdk_session_is_absent()
|
||||
session = await client.create(manifest=Manifest())
|
||||
agent = SandboxAgent(
|
||||
name="worker",
|
||||
model=FakeModel(initial_output=[get_final_output_message("done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
||||
instructions="Worker.",
|
||||
capabilities=[_memory_config()],
|
||||
)
|
||||
@@ -1413,8 +1426,8 @@ async def test_sandbox_memory_uses_conversation_id_when_sdk_session_is_absent()
|
||||
async def test_sandbox_memory_uses_group_id_when_sdk_session_is_absent() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
agent_model = FakeModel()
|
||||
agent_model.add_multiple_turn_outputs(
|
||||
agent_model = ScriptedModel()
|
||||
agent_model.extend(
|
||||
[
|
||||
[get_final_output_message("first done")],
|
||||
[get_final_output_message("second done")],
|
||||
@@ -1450,8 +1463,8 @@ async def test_sandbox_memory_uses_group_id_when_sdk_session_is_absent() -> None
|
||||
async def test_sandbox_memory_uses_per_run_conversation_when_no_conversation_id() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
agent_model = FakeModel()
|
||||
agent_model.add_multiple_turn_outputs(
|
||||
agent_model = ScriptedModel()
|
||||
agent_model.extend(
|
||||
[
|
||||
[get_final_output_message("first done")],
|
||||
[get_final_output_message("second done")],
|
||||
@@ -1483,27 +1496,29 @@ async def test_sandbox_memory_uses_per_run_conversation_when_no_conversation_id(
|
||||
async def test_sandbox_memory_caps_phase_two_selection_and_surfaces_removed_rollouts() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
phase_one_model = FakeModel()
|
||||
phase_one_model.add_multiple_turn_outputs(
|
||||
phase_one_model = ScriptedModel()
|
||||
phase_one_model.extend(
|
||||
[
|
||||
[_phase_one_message(slug="first", raw_memory="first raw\n")],
|
||||
[_phase_one_message(slug="second", raw_memory="second raw\n")],
|
||||
]
|
||||
)
|
||||
phase_two_model = FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "first entry"),
|
||||
_patch_update_call("memory-summary", "memories/memory_summary.md", "first summary"),
|
||||
phase_two_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "first entry"),
|
||||
_patch_update_call("memory-summary", "memories/memory_summary.md", "first summary"),
|
||||
]
|
||||
]
|
||||
)
|
||||
phase_two_model.set_next_output([get_final_output_message("consolidated")])
|
||||
phase_two_model.enqueue([get_final_output_message("consolidated")])
|
||||
memory = _memory_config(
|
||||
max_raw_memories_for_consolidation=1,
|
||||
phase_one_model=phase_one_model,
|
||||
phase_two_model=phase_two_model,
|
||||
)
|
||||
agent_model = FakeModel()
|
||||
agent_model.add_multiple_turn_outputs(
|
||||
agent_model = ScriptedModel()
|
||||
agent_model.extend(
|
||||
[
|
||||
[get_final_output_message("first done")],
|
||||
[get_final_output_message("second done")],
|
||||
@@ -1551,8 +1566,8 @@ async def test_sandbox_memory_caps_phase_two_selection_and_surfaces_removed_roll
|
||||
assert "second raw" in merged_raw_memories
|
||||
assert "first raw" not in merged_raw_memories
|
||||
|
||||
assert phase_two_model.first_turn_args is not None
|
||||
prompt = _extract_user_text_from_turn_args(phase_two_model.first_turn_args)
|
||||
assert bool(phase_two_model.calls)
|
||||
prompt = _extract_user_text_from_model_input(phase_two_model.calls[0].input)
|
||||
assert "newly added since the last successful Phase 2 run: 1" in prompt
|
||||
assert f"rollout_id={selected_rollout_ids[0]}" in prompt
|
||||
finally:
|
||||
@@ -1563,21 +1578,25 @@ async def test_sandbox_memory_caps_phase_two_selection_and_surfaces_removed_roll
|
||||
async def test_sandbox_memory_runs_phase_one_and_phase_two_on_session_close() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
phase_one_model = FakeModel(initial_output=[_phase_one_message()])
|
||||
phase_two_model = FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "shutdown entry"),
|
||||
_patch_update_call("memory-summary", "memories/memory_summary.md", "shutdown summary"),
|
||||
phase_one_model = ScriptedModel(steps=[[_phase_one_message()]])
|
||||
phase_two_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "shutdown entry"),
|
||||
_patch_update_call(
|
||||
"memory-summary", "memories/memory_summary.md", "shutdown summary"
|
||||
),
|
||||
]
|
||||
]
|
||||
)
|
||||
phase_two_model.set_next_output([get_final_output_message("shutdown")])
|
||||
phase_two_model.enqueue([get_final_output_message("shutdown")])
|
||||
memory = _memory_config(
|
||||
phase_one_model=phase_one_model,
|
||||
phase_two_model=phase_two_model,
|
||||
)
|
||||
agent = SandboxAgent(
|
||||
name="worker",
|
||||
model=FakeModel(initial_output=[get_final_output_message("done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
||||
instructions="Worker.",
|
||||
capabilities=[memory],
|
||||
)
|
||||
@@ -1752,7 +1771,7 @@ async def test_sandbox_memory_enqueue_failure_follows_both_data_policies(
|
||||
client = _DeleteTrackingUnixLocalSandboxClient()
|
||||
agent = SandboxAgent(
|
||||
name="worker",
|
||||
model=FakeModel(initial_output=[get_final_output_message("done")]),
|
||||
model=ScriptedModel(steps=[[get_final_output_message("done")]]),
|
||||
instructions="Worker.",
|
||||
capabilities=[_memory_config()],
|
||||
)
|
||||
@@ -1796,23 +1815,25 @@ async def test_sandbox_memory_enqueue_failure_follows_both_data_policies(
|
||||
async def test_sandbox_memory_marks_interrupted_runs_in_phase_one_prompt() -> None:
|
||||
client = UnixLocalSandboxClient()
|
||||
session = await client.create(manifest=Manifest())
|
||||
phase_one_model = FakeModel(initial_output=[_phase_one_message()])
|
||||
phase_two_model = FakeModel(
|
||||
initial_output=[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "interrupted entry"),
|
||||
_patch_update_call(
|
||||
"memory-summary", "memories/memory_summary.md", "interrupted summary"
|
||||
),
|
||||
phase_one_model = ScriptedModel(steps=[[_phase_one_message()]])
|
||||
phase_two_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
_patch_update_call("memory-md", "memories/MEMORY.md", "interrupted entry"),
|
||||
_patch_update_call(
|
||||
"memory-summary", "memories/memory_summary.md", "interrupted summary"
|
||||
),
|
||||
]
|
||||
]
|
||||
)
|
||||
phase_two_model.set_next_output([get_final_output_message("done")])
|
||||
phase_two_model.enqueue([get_final_output_message("done")])
|
||||
memory = _memory_config(
|
||||
phase_one_model=phase_one_model,
|
||||
phase_two_model=phase_two_model,
|
||||
)
|
||||
agent = SandboxAgent(
|
||||
name="worker",
|
||||
model=FakeModel(initial_output=[make_shell_call("approval-call")]),
|
||||
model=ScriptedModel(steps=[[make_shell_call("approval-call")]]),
|
||||
instructions="Worker.",
|
||||
tools=[ShellTool(executor=lambda _request: "ok", needs_approval=True)],
|
||||
capabilities=[memory],
|
||||
|
||||
+191
-191
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable, Coroutine
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
@@ -16,8 +15,8 @@ from agents.sandbox.capabilities import Capability, Compaction, Memory
|
||||
from agents.sandbox.entries import BaseEntry, File
|
||||
from agents.sandbox.manifest import Manifest
|
||||
from agents.sandbox.sandbox_agent import SandboxAgent
|
||||
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
|
||||
from agents.sandbox.types import User
|
||||
from agents.testing import scripted_sandbox_session
|
||||
|
||||
|
||||
def test_sandbox_agent_normalizes_first_party_dictionary_configuration() -> None:
|
||||
@@ -84,8 +83,8 @@ class _Capability:
|
||||
return self.fragment
|
||||
|
||||
|
||||
def _session_with_manifest(manifest: Manifest | None) -> object:
|
||||
return SimpleNamespace(state=SimpleNamespace(manifest=manifest))
|
||||
def _session_with_manifest(manifest: Manifest | None):
|
||||
return scripted_sandbox_session(manifest=manifest)
|
||||
|
||||
|
||||
def test_prepare_sandbox_agent_passes_session_manifest_to_capability_instructions():
|
||||
@@ -97,7 +96,7 @@ def test_prepare_sandbox_agent_passes_session_manifest_to_capability_instruction
|
||||
base_instructions="base instructions",
|
||||
instructions="additional instructions",
|
||||
),
|
||||
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
|
||||
session=_session_with_manifest(manifest),
|
||||
capabilities=cast(list[Capability], [capability]),
|
||||
)
|
||||
instructions = cast(
|
||||
@@ -134,7 +133,7 @@ def test_prepare_sandbox_agent_wraps_capabilities_without_agent_instructions():
|
||||
name="sandbox",
|
||||
base_instructions="base instructions",
|
||||
),
|
||||
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
|
||||
session=_session_with_manifest(manifest),
|
||||
capabilities=cast(list[Capability], [capability]),
|
||||
)
|
||||
instructions = cast(
|
||||
@@ -170,7 +169,7 @@ def test_prepare_sandbox_agent_passes_default_model_to_capability_sampling_param
|
||||
name="sandbox",
|
||||
instructions="base instructions",
|
||||
),
|
||||
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
|
||||
session=_session_with_manifest(manifest),
|
||||
capabilities=cast(list[Capability], [capability]),
|
||||
)
|
||||
|
||||
@@ -185,7 +184,7 @@ def test_prepare_sandbox_agent_prepares_default_compaction_policy() -> None:
|
||||
name="sandbox",
|
||||
instructions="base instructions",
|
||||
),
|
||||
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
|
||||
session=_session_with_manifest(manifest),
|
||||
capabilities=[Compaction()],
|
||||
)
|
||||
|
||||
@@ -203,7 +202,7 @@ def test_prepare_sandbox_agent_uses_default_sandbox_instructions_when_base_missi
|
||||
name="sandbox",
|
||||
instructions="additional instructions",
|
||||
),
|
||||
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
|
||||
session=_session_with_manifest(manifest),
|
||||
capabilities=cast(list[Capability], [capability]),
|
||||
)
|
||||
instructions = cast(
|
||||
@@ -259,7 +258,7 @@ def test_prepare_sandbox_agent_validates_required_capabilities() -> None:
|
||||
instructions="base instructions",
|
||||
capabilities=[Memory()],
|
||||
),
|
||||
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
|
||||
session=_session_with_manifest(manifest),
|
||||
capabilities=[Memory()],
|
||||
)
|
||||
|
||||
@@ -270,7 +269,7 @@ def test_prepare_sandbox_agent_validates_required_capabilities() -> None:
|
||||
instructions="base instructions",
|
||||
capabilities=[Memory(read=MemoryReadConfig(live_update=False), generate=None)],
|
||||
),
|
||||
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
|
||||
session=_session_with_manifest(manifest),
|
||||
capabilities=[Memory(read=MemoryReadConfig(live_update=False), generate=None)],
|
||||
)
|
||||
|
||||
@@ -280,7 +279,7 @@ def test_prepare_sandbox_agent_validates_required_capabilities() -> None:
|
||||
instructions="base instructions",
|
||||
capabilities=[Memory()],
|
||||
),
|
||||
session=cast(BaseSandboxSession, _session_with_manifest(manifest)),
|
||||
session=_session_with_manifest(manifest),
|
||||
capabilities=cast(
|
||||
list[Capability],
|
||||
[
|
||||
|
||||
+69
-63
@@ -47,8 +47,8 @@ from agents.agent_tool_state import (
|
||||
from agents.run_context import _ApprovalRecord
|
||||
from agents.run_state import _build_agent_map
|
||||
from agents.stream_events import AgentUpdatedStreamEvent, RawResponsesStreamEvent
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool_context import ToolContext
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.mcp.helpers import FakeMCPServer
|
||||
from tests.mcp.model_compat import create_mcp_error
|
||||
from tests.test_responses import get_function_tool_call, get_text_message
|
||||
@@ -1790,14 +1790,14 @@ async def test_agent_as_tool_resume_survives_cancellation_after_nested_output_co
|
||||
nested_model_waiting = asyncio.Event()
|
||||
keep_nested_model_waiting = asyncio.Event()
|
||||
|
||||
class BlockingSecondModel(FakeModel):
|
||||
class BlockingSecondModel(ScriptedModel):
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.calls = 0
|
||||
self.response_calls = 0
|
||||
|
||||
async def get_response(self, *args: Any, **kwargs: Any) -> ModelResponse:
|
||||
self.calls += 1
|
||||
if self.calls == 2:
|
||||
self.response_calls += 1
|
||||
if self.response_calls == 2:
|
||||
nested_model_waiting.set()
|
||||
await keep_nested_model_waiting.wait()
|
||||
return await super().get_response(*args, **kwargs)
|
||||
@@ -1808,24 +1808,26 @@ async def test_agent_as_tool_resume_survives_cancellation_after_nested_output_co
|
||||
return "inner value"
|
||||
|
||||
inner_model = BlockingSecondModel(
|
||||
initial_output=[get_function_tool_call("sensitive", "{}", call_id="inner_call")]
|
||||
steps=[[get_function_tool_call("sensitive", "{}", call_id="inner_call")]]
|
||||
)
|
||||
inner_model.set_next_output([get_text_message("inner done")])
|
||||
inner_model.enqueue([get_text_message("inner done")])
|
||||
inner_agent = Agent(name="inner", model=inner_model, tools=[sensitive])
|
||||
nested_tool = inner_agent.as_tool(
|
||||
tool_name="delegate",
|
||||
tool_description="Delegate",
|
||||
)
|
||||
outer_model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call(
|
||||
"delegate",
|
||||
'{"input":"hi"}',
|
||||
call_id="outer_call",
|
||||
)
|
||||
outer_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call(
|
||||
"delegate",
|
||||
'{"input":"hi"}',
|
||||
call_id="outer_call",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
outer_model.set_next_output([get_text_message("outer done")])
|
||||
outer_model.enqueue([get_text_message("outer done")])
|
||||
outer_agent = Agent(name="outer", model=outer_model, tools=[nested_tool])
|
||||
|
||||
async def run_outer(input_value: Any) -> RunResult | RunResultStreaming:
|
||||
@@ -1843,7 +1845,7 @@ async def test_agent_as_tool_resume_survives_cancellation_after_nested_output_co
|
||||
resume_task = asyncio.create_task(run_outer(state))
|
||||
await nested_model_waiting.wait()
|
||||
assert tool_attempts == ["ran"]
|
||||
assert inner_model.calls == 2
|
||||
assert inner_model.response_calls == 2
|
||||
|
||||
resume_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
@@ -1853,7 +1855,7 @@ async def test_agent_as_tool_resume_survives_cancellation_after_nested_output_co
|
||||
|
||||
assert result.final_output == "outer done"
|
||||
assert tool_attempts == ["ran"]
|
||||
assert inner_model.calls == 3
|
||||
assert inner_model.response_calls == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -2515,28 +2517,30 @@ async def test_agent_as_tool_streaming_works_with_custom_extractor(
|
||||
async def test_agent_as_tool_streaming_settles_multi_segment_text_output() -> None:
|
||||
agent = Agent(
|
||||
name="streamer",
|
||||
model=FakeModel(
|
||||
initial_output=[
|
||||
ResponseOutputMessage(
|
||||
id="msg_multi_segment",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text="first ",
|
||||
type="output_text",
|
||||
logprobs=[],
|
||||
),
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text="second",
|
||||
type="output_text",
|
||||
logprobs=[],
|
||||
),
|
||||
],
|
||||
)
|
||||
model=ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
ResponseOutputMessage(
|
||||
id="msg_multi_segment",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text="first ",
|
||||
type="output_text",
|
||||
logprobs=[],
|
||||
),
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text="second",
|
||||
type="output_text",
|
||||
logprobs=[],
|
||||
),
|
||||
],
|
||||
)
|
||||
]
|
||||
]
|
||||
),
|
||||
)
|
||||
@@ -2578,28 +2582,30 @@ async def test_agent_as_tool_streaming_settles_multi_segment_structured_output()
|
||||
|
||||
agent = Agent(
|
||||
name="streamer",
|
||||
model=FakeModel(
|
||||
initial_output=[
|
||||
ResponseOutputMessage(
|
||||
id="msg_multi_segment_structured",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text='{"answer":"str',
|
||||
type="output_text",
|
||||
logprobs=[],
|
||||
),
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text='uctured"}',
|
||||
type="output_text",
|
||||
logprobs=[],
|
||||
),
|
||||
],
|
||||
)
|
||||
model=ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
ResponseOutputMessage(
|
||||
id="msg_multi_segment_structured",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text='{"answer":"str',
|
||||
type="output_text",
|
||||
logprobs=[],
|
||||
),
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text='uctured"}',
|
||||
type="output_text",
|
||||
logprobs=[],
|
||||
),
|
||||
],
|
||||
)
|
||||
]
|
||||
]
|
||||
),
|
||||
output_type=StructuredOutput,
|
||||
@@ -2686,10 +2692,10 @@ async def test_agent_as_tool_streaming_settles_final_text_after_nested_mcp_failu
|
||||
|
||||
agent = Agent(
|
||||
name="streamer",
|
||||
model=FakeModel(),
|
||||
model=ScriptedModel(),
|
||||
mcp_servers=[nested_server],
|
||||
)
|
||||
cast(FakeModel, agent.model).add_multiple_turn_outputs(
|
||||
cast(ScriptedModel, agent.model).extend(
|
||||
[
|
||||
[get_function_tool_call(tool_name, "{}")],
|
||||
[
|
||||
|
||||
+30
-30
@@ -11,10 +11,10 @@ from agents.agent import Agent
|
||||
from agents.lifecycle import AgentHooks
|
||||
from agents.run import Runner
|
||||
from agents.run_context import AgentHookContext, RunContextWrapper, TContext
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import Tool
|
||||
from agents.tool_context import ToolContext
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_final_output_message,
|
||||
get_function_tool,
|
||||
@@ -82,14 +82,14 @@ class FalsyAgentHooks(AgentHooksForTests):
|
||||
@pytest.mark.asyncio
|
||||
async def test_falsy_agent_hooks_are_invoked() -> None:
|
||||
hooks = FalsyAgentHooks()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
tools=[get_function_tool("some_function", "result")],
|
||||
hooks=hooks,
|
||||
)
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
[get_text_message("done")],
|
||||
@@ -109,7 +109,7 @@ async def test_falsy_agent_hooks_are_invoked() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_agent_hooks():
|
||||
hooks = AgentHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -128,12 +128,12 @@ async def test_non_streamed_agent_hooks():
|
||||
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_text_message("user_message")])
|
||||
model.enqueue([get_text_message("user_message")])
|
||||
output = await Runner.run(agent_3, input="user_message")
|
||||
assert hooks.events == {"on_start": 1, "on_end": 1}, f"{output}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
[get_text_message("done")],
|
||||
@@ -144,7 +144,7 @@ async def test_non_streamed_agent_hooks():
|
||||
assert len(set(hooks.tool_context_ids)) == 1
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -165,7 +165,7 @@ async def test_non_streamed_agent_hooks():
|
||||
}, f"got unexpected event count: {hooks.events}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
@@ -196,7 +196,7 @@ async def test_non_streamed_agent_hooks():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_agent_hooks():
|
||||
hooks = AgentHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(name="test_2", model=model)
|
||||
agent_3 = Agent(
|
||||
@@ -209,14 +209,14 @@ async def test_streamed_agent_hooks():
|
||||
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_text_message("user_message")])
|
||||
model.enqueue([get_text_message("user_message")])
|
||||
output = Runner.run_streamed(agent_3, input="user_message")
|
||||
async for _ in output.stream_events():
|
||||
pass
|
||||
assert hooks.events == {"on_start": 1, "on_end": 1}, f"{output}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -239,7 +239,7 @@ async def test_streamed_agent_hooks():
|
||||
}, f"got unexpected event count: {hooks.events}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
@@ -276,7 +276,7 @@ class Foo(TypedDict):
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_non_streamed_agent_hooks():
|
||||
hooks = AgentHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(name="test_2", model=model)
|
||||
agent_3 = Agent(
|
||||
@@ -290,12 +290,12 @@ async def test_structured_output_non_streamed_agent_hooks():
|
||||
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
output = await Runner.run(agent_3, input="user_message")
|
||||
assert hooks.events == {"on_start": 1, "on_end": 1}, f"{output}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -316,7 +316,7 @@ async def test_structured_output_non_streamed_agent_hooks():
|
||||
}, f"got unexpected event count: {hooks.events}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
@@ -347,7 +347,7 @@ async def test_structured_output_non_streamed_agent_hooks():
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_streamed_agent_hooks():
|
||||
hooks = AgentHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(name="test_2", model=model)
|
||||
agent_3 = Agent(
|
||||
@@ -361,14 +361,14 @@ async def test_structured_output_streamed_agent_hooks():
|
||||
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
output = Runner.run_streamed(agent_3, input="user_message")
|
||||
async for _ in output.stream_events():
|
||||
pass
|
||||
assert hooks.events == {"on_start": 1, "on_end": 1}, f"{output}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -388,7 +388,7 @@ async def test_structured_output_streamed_agent_hooks():
|
||||
}, f"got unexpected event count: {hooks.events}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
@@ -425,7 +425,7 @@ class EmptyAgentHooks(AgentHooks):
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_agent_hooks_dont_crash():
|
||||
hooks = EmptyAgentHooks()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(name="test_2", model=model)
|
||||
agent_3 = Agent(
|
||||
@@ -438,12 +438,12 @@ async def test_base_agent_hooks_dont_crash():
|
||||
)
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
output = Runner.run_streamed(agent_3, input="user_message")
|
||||
async for _ in output.stream_events():
|
||||
pass
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -455,7 +455,7 @@ async def test_base_agent_hooks_dont_crash():
|
||||
)
|
||||
await Runner.run(agent_3, input="user_message")
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -490,10 +490,10 @@ class AgentHooksWithTurnInput(AgentHooks):
|
||||
async def test_agent_hooks_receives_turn_input_string():
|
||||
"""Test that on_start receives turn_input when input is a string."""
|
||||
hooks = AgentHooksWithTurnInput()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model, hooks=hooks)
|
||||
|
||||
model.set_next_output([get_text_message("response")])
|
||||
model.enqueue([get_text_message("response")])
|
||||
await Runner.run(agent, input="hello world")
|
||||
|
||||
assert len(hooks.captured_turn_inputs) == 1
|
||||
@@ -507,7 +507,7 @@ async def test_agent_hooks_receives_turn_input_string():
|
||||
async def test_agent_hooks_receives_turn_input_list():
|
||||
"""Test that on_start receives turn_input when input is a list."""
|
||||
hooks = AgentHooksWithTurnInput()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model, hooks=hooks)
|
||||
|
||||
input_items: list[Any] = [
|
||||
@@ -515,7 +515,7 @@ async def test_agent_hooks_receives_turn_input_list():
|
||||
{"role": "user", "content": "second message"},
|
||||
]
|
||||
|
||||
model.set_next_output([get_text_message("response")])
|
||||
model.enqueue([get_text_message("response")])
|
||||
await Runner.run(agent, input=input_items)
|
||||
|
||||
assert len(hooks.captured_turn_inputs) == 1
|
||||
@@ -529,10 +529,10 @@ async def test_agent_hooks_receives_turn_input_list():
|
||||
async def test_agent_hooks_receives_turn_input_streamed():
|
||||
"""Test that on_start receives turn_input in streamed mode."""
|
||||
hooks = AgentHooksWithTurnInput()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model, hooks=hooks)
|
||||
|
||||
model.set_next_output([get_text_message("response")])
|
||||
model.enqueue([get_text_message("response")])
|
||||
result = Runner.run_streamed(agent, input="streamed input")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
|
||||
@@ -8,9 +8,9 @@ from agents.items import ItemHelpers, ModelResponse, TResponseInputItem
|
||||
from agents.lifecycle import AgentHooks
|
||||
from agents.run import Runner
|
||||
from agents.run_context import AgentHookContext, RunContextWrapper, TContext
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import Tool
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_function_tool,
|
||||
get_text_message,
|
||||
@@ -74,12 +74,12 @@ class AgentHooksForTests(AgentHooks):
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_agent_hooks_with_llm():
|
||||
hooks = AgentHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[], hooks=hooks
|
||||
)
|
||||
# Simulate a single LLM call producing an output:
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
await Runner.run(agent, input="hello")
|
||||
# Expect one on_start, one on_llm_start, one on_llm_end, and one on_end
|
||||
assert hooks.events == {"on_start": 1, "on_llm_start": 1, "on_llm_end": 1, "on_end": 1}
|
||||
@@ -88,12 +88,12 @@ async def test_async_agent_hooks_with_llm():
|
||||
# test_sync_agent_hook_with_llm()
|
||||
def test_sync_agent_hook_with_llm():
|
||||
hooks = AgentHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[], hooks=hooks
|
||||
)
|
||||
# Simulate a single LLM call producing an output:
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
Runner.run_sync(agent, input="hello")
|
||||
# Expect one on_start, one on_llm_start, one on_llm_end, and one on_end
|
||||
assert hooks.events == {"on_start": 1, "on_llm_start": 1, "on_llm_end": 1, "on_end": 1}
|
||||
@@ -103,12 +103,12 @@ def test_sync_agent_hook_with_llm():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_agent_hooks_with_llm():
|
||||
hooks = AgentHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[], hooks=hooks
|
||||
)
|
||||
# Simulate a single LLM call producing an output:
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
stream = Runner.run_streamed(agent, input="hello")
|
||||
|
||||
async for event in stream.stream_events():
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
|
||||
from agents import Agent, Runner
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
|
||||
def _make_message(text: str) -> ResponseOutputMessage:
|
||||
@@ -22,8 +22,8 @@ def _make_message(text: str) -> ResponseOutputMessage:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_is_released_after_run() -> None:
|
||||
fake_model = FakeModel(initial_output=[_make_message("Paris")])
|
||||
agent = Agent(name="leak-test-agent", instructions="Answer questions.", model=fake_model)
|
||||
scripted_model = ScriptedModel(steps=[[_make_message("Paris")]])
|
||||
agent = Agent(name="leak-test-agent", instructions="Answer questions.", model=scripted_model)
|
||||
agent_ref = weakref.ref(agent)
|
||||
|
||||
# Running the agent should not leave behind strong references once the result goes out of scope.
|
||||
|
||||
@@ -10,13 +10,14 @@ from agents import Agent, Prompt, RunConfig, RunContextWrapper, Runner
|
||||
from agents.models.interface import Model, ModelProvider
|
||||
from agents.models.openai_responses import OpenAIResponsesModel
|
||||
from agents.prompts import GenerateDynamicPromptData
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.model_test_helpers import get_response_obj
|
||||
|
||||
from .fake_model import FakeModel, get_response_obj
|
||||
from .test_responses import get_text_message
|
||||
|
||||
|
||||
class PromptCaptureFakeModel(FakeModel):
|
||||
"""Subclass of FakeModel that records the prompt passed to the model."""
|
||||
class PromptCaptureScriptedModel(ScriptedModel):
|
||||
"""Subclass of ScriptedModel that records the prompt passed to the model."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -91,11 +92,11 @@ async def test_dynamic_prompt_is_resolved_correctly():
|
||||
async def test_prompt_is_passed_to_model():
|
||||
static_prompt: Prompt = {"id": "model_prompt"}
|
||||
|
||||
model = PromptCaptureFakeModel()
|
||||
model = PromptCaptureScriptedModel()
|
||||
agent = Agent(name="test", model=model, prompt=static_prompt)
|
||||
|
||||
# Ensure the model returns a simple message so the run completes in one turn.
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
|
||||
await Runner.run(agent, input="hello")
|
||||
|
||||
@@ -171,7 +172,7 @@ async def test_run_cancels_sibling_instructions_when_prompt_resolution_fails() -
|
||||
|
||||
agent = Agent(
|
||||
name="prompt-agent",
|
||||
model=FakeModel(),
|
||||
model=ScriptedModel(),
|
||||
instructions=slow_instructions,
|
||||
prompt=failing_prompt,
|
||||
)
|
||||
@@ -206,7 +207,7 @@ async def test_run_streamed_cancels_sibling_instructions_when_prompt_resolution_
|
||||
|
||||
agent = Agent(
|
||||
name="prompt-agent",
|
||||
model=FakeModel(),
|
||||
model=ScriptedModel(),
|
||||
instructions=slow_instructions,
|
||||
prompt=failing_prompt,
|
||||
)
|
||||
|
||||
+323
-336
File diff suppressed because it is too large
Load Diff
+162
-213
File diff suppressed because it is too large
Load Diff
+63
-63
@@ -9,9 +9,9 @@ from openai.types.responses.response_usage import InputTokensDetails
|
||||
|
||||
from agents import Agent, RunConfig, Runner, RunState, custom_span, function_tool, trace
|
||||
from agents.sandbox.runtime import SandboxRuntime
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.usage import Usage
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_function_tool_call, get_text_message
|
||||
from .testing_processor import (
|
||||
assert_no_traces,
|
||||
@@ -22,7 +22,7 @@ from .testing_processor import (
|
||||
)
|
||||
|
||||
|
||||
def _make_approval_agent(model: FakeModel) -> Agent[None]:
|
||||
def _make_approval_agent(model: ScriptedModel) -> Agent[None]:
|
||||
@function_tool(name_override="approval_tool", needs_approval=True)
|
||||
def approval_tool() -> str:
|
||||
return "ok"
|
||||
@@ -43,8 +43,8 @@ def _usage_metadata(requests: int, input_tokens: int, output_tokens: int) -> dic
|
||||
async def test_single_run_is_single_trace():
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
initial_output=[get_text_message("first_test")],
|
||||
model=ScriptedModel(
|
||||
steps=[[get_text_message("first_test")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ async def test_agent_span_uses_resolved_tool_name_collision_view(
|
||||
surface: str,
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("done")])
|
||||
model = ScriptedModel(steps=[[get_text_message("done")]])
|
||||
expected_tools: list[str]
|
||||
expected_handoffs: list[str]
|
||||
|
||||
@@ -136,14 +136,14 @@ async def test_task_and_turn_spans_export_aggregate_usage():
|
||||
def foo_tool() -> str:
|
||||
return "foo result"
|
||||
|
||||
model = FakeModel(tracing_enabled=True)
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel(emit_traces=True)
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("foo_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
model.set_hardcoded_usage(
|
||||
model.set_default_usage(
|
||||
Usage(
|
||||
requests=1,
|
||||
input_tokens=10,
|
||||
@@ -259,8 +259,8 @@ async def test_task_and_turn_spans_can_be_disabled():
|
||||
def foo_tool() -> str:
|
||||
return "foo result"
|
||||
|
||||
model = FakeModel(tracing_enabled=True)
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel(emit_traces=True)
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("foo_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
@@ -296,9 +296,9 @@ async def test_task_and_turn_spans_can_be_disabled():
|
||||
async def test_task_and_turn_spans_can_be_explicitly_enabled():
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
tracing_enabled=True,
|
||||
initial_output=[get_text_message("done")],
|
||||
model=ScriptedModel(
|
||||
emit_traces=True,
|
||||
steps=[[get_text_message("done")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -317,9 +317,9 @@ async def test_task_and_turn_spans_can_be_explicitly_enabled():
|
||||
async def test_task_span_resets_current_span_if_run_setup_fails(monkeypatch: pytest.MonkeyPatch):
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
tracing_enabled=True,
|
||||
initial_output=[get_text_message("first_test")],
|
||||
model=ScriptedModel(
|
||||
emit_traces=True,
|
||||
steps=[[get_text_message("first_test")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -347,8 +347,8 @@ async def test_task_span_resets_current_span_if_run_setup_fails(monkeypatch: pyt
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_runs_are_multiple_traces():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("first_test")],
|
||||
[get_text_message("second_test")],
|
||||
@@ -398,8 +398,8 @@ async def test_multiple_runs_are_multiple_traces():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_run_reuses_original_trace_without_duplicate_trace_start():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
@@ -425,14 +425,14 @@ async def test_resumed_run_reuses_original_trace_without_duplicate_trace_start()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_run_task_span_usage_is_run_local_delta():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
model.set_hardcoded_usage(Usage(requests=1, input_tokens=10, output_tokens=3, total_tokens=13))
|
||||
model.set_default_usage(Usage(requests=1, input_tokens=10, output_tokens=3, total_tokens=13))
|
||||
agent = _make_approval_agent(model)
|
||||
|
||||
first = await Runner.run(agent, input="first_test")
|
||||
@@ -461,8 +461,8 @@ async def test_resumed_run_task_span_usage_is_run_local_delta():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_run_from_serialized_state_reuses_original_trace():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
@@ -490,8 +490,8 @@ async def test_resumed_run_from_serialized_state_reuses_original_trace():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_run_from_serialized_state_preserves_explicit_trace_key():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
@@ -530,8 +530,8 @@ async def test_resumed_run_from_serialized_state_preserves_explicit_trace_key():
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_run_with_workflow_override_starts_new_trace() -> None:
|
||||
trace_id = f"trace_{uuid4().hex}"
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
@@ -570,8 +570,8 @@ async def test_resumed_run_with_workflow_override_starts_new_trace() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapped_trace_is_single_trace():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("first_test")],
|
||||
[get_text_message("second_test")],
|
||||
@@ -631,8 +631,8 @@ async def test_parent_disabled_trace_disabled_agent_trace():
|
||||
with trace(workflow_name="test_workflow", disabled=True):
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
initial_output=[get_text_message("first_test")],
|
||||
model=ScriptedModel(
|
||||
steps=[[get_text_message("first_test")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -645,8 +645,8 @@ async def test_parent_disabled_trace_disabled_agent_trace():
|
||||
async def test_manual_disabling_works():
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
initial_output=[get_text_message("first_test")],
|
||||
model=ScriptedModel(
|
||||
steps=[[get_text_message("first_test")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -659,8 +659,8 @@ async def test_manual_disabling_works():
|
||||
async def test_trace_config_works():
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
initial_output=[get_text_message("first_test")],
|
||||
model=ScriptedModel(
|
||||
steps=[[get_text_message("first_test")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -696,8 +696,8 @@ async def test_trace_config_works():
|
||||
async def test_not_starting_streaming_creates_trace():
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
initial_output=[get_text_message("first_test")],
|
||||
model=ScriptedModel(
|
||||
steps=[[get_text_message("first_test")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -737,8 +737,8 @@ async def test_not_starting_streaming_creates_trace():
|
||||
async def test_streaming_single_run_is_single_trace():
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
initial_output=[get_text_message("first_test")],
|
||||
model=ScriptedModel(
|
||||
steps=[[get_text_message("first_test")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -768,8 +768,8 @@ async def test_streaming_single_run_is_single_trace():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_streamed_runs_are_multiple_traces():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("first_test")],
|
||||
[get_text_message("second_test")],
|
||||
@@ -824,8 +824,8 @@ async def test_multiple_streamed_runs_are_multiple_traces():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_streaming_run_reuses_original_trace_without_duplicate_trace_start():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
@@ -855,14 +855,14 @@ async def test_resumed_streaming_run_reuses_original_trace_without_duplicate_tra
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_streaming_run_task_span_usage_is_run_local_delta():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="call-1")],
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
model.set_hardcoded_usage(Usage(requests=1, input_tokens=11, output_tokens=4, total_tokens=15))
|
||||
model.set_default_usage(Usage(requests=1, input_tokens=11, output_tokens=4, total_tokens=15))
|
||||
agent = _make_approval_agent(model)
|
||||
|
||||
first = Runner.run_streamed(agent, input="first_test")
|
||||
@@ -895,8 +895,8 @@ async def test_resumed_streaming_run_task_span_usage_is_run_local_delta():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapped_streaming_trace_is_single_trace():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("first_test")],
|
||||
[get_text_message("second_test")],
|
||||
@@ -963,9 +963,9 @@ async def test_wrapped_streaming_trace_is_single_trace():
|
||||
async def test_wrapped_streaming_run_creates_root_task_span():
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
tracing_enabled=True,
|
||||
initial_output=[get_text_message("first_test")],
|
||||
model=ScriptedModel(
|
||||
emit_traces=True,
|
||||
steps=[[get_text_message("first_test")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -997,7 +997,7 @@ async def test_wrapped_run_task_span_uses_run_workflow_name():
|
||||
def _make_agent() -> Agent[None]:
|
||||
return Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(initial_output=[get_text_message("first_test")]),
|
||||
model=ScriptedModel(steps=[[get_text_message("first_test")]]),
|
||||
)
|
||||
|
||||
run_config = RunConfig(workflow_name="inner_workflow")
|
||||
@@ -1021,9 +1021,9 @@ async def test_wrapped_run_task_span_uses_run_workflow_name():
|
||||
async def test_wrapped_streaming_run_can_disable_task_and_turn_spans():
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
model=FakeModel(
|
||||
tracing_enabled=True,
|
||||
initial_output=[get_text_message("done")],
|
||||
model=ScriptedModel(
|
||||
emit_traces=True,
|
||||
steps=[[get_text_message("done")]],
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1050,8 +1050,8 @@ async def test_wrapped_streaming_run_can_disable_task_and_turn_spans():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapped_mixed_trace_is_single_trace():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("first_test")],
|
||||
[get_text_message("second_test")],
|
||||
@@ -1114,8 +1114,8 @@ async def test_wrapped_mixed_trace_is_single_trace():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parent_disabled_trace_disables_streaming_agent_trace():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("first_test")],
|
||||
[get_text_message("second_test")],
|
||||
@@ -1136,8 +1136,8 @@ async def test_parent_disabled_trace_disables_streaming_agent_trace():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manual_streaming_disabling_works():
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("first_test")],
|
||||
[get_text_message("second_test")],
|
||||
|
||||
@@ -6,8 +6,8 @@ import pytest
|
||||
|
||||
from agents import Agent, RunConfig, Runner, TResponseInputItem, UserError
|
||||
from agents.run import CallModelData, ModelInputData
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_text_input_item, get_text_message
|
||||
from .testing_processor import fetch_span_errors
|
||||
|
||||
@@ -16,11 +16,11 @@ SENSITIVE_ERROR_MESSAGE = "sensitive-filter-error"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_sync_non_streamed() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Prepare model output
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
def filter_fn(data: CallModelData[Any]) -> ModelInputData:
|
||||
mi = data.model_data
|
||||
@@ -33,19 +33,19 @@ async def test_call_model_input_filter_sync_non_streamed() -> None:
|
||||
run_config=RunConfig(call_model_input_filter=filter_fn),
|
||||
)
|
||||
|
||||
assert model.last_turn_args["system_instructions"] == "filtered-sync"
|
||||
assert isinstance(model.last_turn_args["input"], list)
|
||||
assert len(model.last_turn_args["input"]) == 2
|
||||
assert model.last_turn_args["input"][-1]["content"] == "added-sync"
|
||||
assert model.calls[-1].system_instructions == "filtered-sync"
|
||||
assert isinstance(model.calls[-1].input, list)
|
||||
assert len(model.calls[-1].input) == 2
|
||||
assert model.calls[-1].input[-1]["content"] == "added-sync"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_async_streamed() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Prepare model output
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
async def filter_fn(data: CallModelData[Any]) -> ModelInputData:
|
||||
mi = data.model_data
|
||||
@@ -60,15 +60,15 @@ async def test_call_model_input_filter_async_streamed() -> None:
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
|
||||
assert model.last_turn_args["system_instructions"] == "filtered-async"
|
||||
assert isinstance(model.last_turn_args["input"], list)
|
||||
assert len(model.last_turn_args["input"]) == 2
|
||||
assert model.last_turn_args["input"][-1]["content"] == "added-async"
|
||||
assert model.calls[-1].system_instructions == "filtered-async"
|
||||
assert isinstance(model.calls[-1].input, list)
|
||||
assert len(model.calls[-1].input) == 2
|
||||
assert model.calls[-1].input[-1]["content"] == "added-async"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_invalid_return_type_raises() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
def invalid_filter(_data: CallModelData[Any]):
|
||||
@@ -99,7 +99,7 @@ async def test_call_model_input_filter_error_respects_sensitive_data_setting(
|
||||
|
||||
with pytest.raises(ValueError, match=SENSITIVE_ERROR_MESSAGE):
|
||||
await Runner.run(
|
||||
Agent(name="test", model=FakeModel(tracing_enabled=False)),
|
||||
Agent(name="test", model=ScriptedModel(emit_traces=False)),
|
||||
input="start",
|
||||
run_config=RunConfig(
|
||||
call_model_input_filter=filter_fn,
|
||||
@@ -117,9 +117,9 @@ async def test_call_model_input_filter_error_respects_sensitive_data_setting(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_prefers_latest_duplicate_outputs_non_streamed() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
duplicate_old = cast(
|
||||
TResponseInputItem,
|
||||
@@ -152,7 +152,7 @@ async def test_call_model_input_filter_prefers_latest_duplicate_outputs_non_stre
|
||||
|
||||
outputs = [
|
||||
item
|
||||
for item in model.last_turn_args["input"]
|
||||
for item in model.calls[-1].input
|
||||
if item.get("type") == "function_call_output" and item.get("call_id") == "dup-call"
|
||||
]
|
||||
assert len(outputs) == 1
|
||||
@@ -161,9 +161,9 @@ async def test_call_model_input_filter_prefers_latest_duplicate_outputs_non_stre
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_prefers_latest_duplicate_outputs_streamed() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
duplicate_old = cast(
|
||||
TResponseInputItem,
|
||||
@@ -198,7 +198,7 @@ async def test_call_model_input_filter_prefers_latest_duplicate_outputs_streamed
|
||||
|
||||
outputs = [
|
||||
item
|
||||
for item in model.last_turn_args["input"]
|
||||
for item in model.calls[-1].input
|
||||
if item.get("type") == "function_call_output" and item.get("call_id") == "dup-call-stream"
|
||||
]
|
||||
assert len(outputs) == 1
|
||||
@@ -294,9 +294,9 @@ def _sent_item_types(sent_input: Any) -> list[str | None]:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_keeps_duplicate_item_order_non_streamed() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
def filter_fn(data: CallModelData[Any]) -> ModelInputData:
|
||||
return ModelInputData(
|
||||
@@ -312,7 +312,7 @@ async def test_call_model_input_filter_keeps_duplicate_item_order_non_streamed()
|
||||
|
||||
# Collapsing the repeated call must not move it behind its output; the Responses API
|
||||
# rejects a function_call_output whose function_call has not been sent yet.
|
||||
assert _sent_item_types(model.last_turn_args["input"]) == [
|
||||
assert _sent_item_types(model.calls[-1].input) == [
|
||||
"function_call",
|
||||
"function_call_output",
|
||||
]
|
||||
@@ -320,9 +320,9 @@ async def test_call_model_input_filter_keeps_duplicate_item_order_non_streamed()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_keeps_duplicate_item_order_streamed() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
async def filter_fn(data: CallModelData[Any]) -> ModelInputData:
|
||||
return ModelInputData(
|
||||
@@ -338,7 +338,7 @@ async def test_call_model_input_filter_keeps_duplicate_item_order_streamed() ->
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
|
||||
assert _sent_item_types(model.last_turn_args["input"]) == [
|
||||
assert _sent_item_types(model.calls[-1].input) == [
|
||||
"function_call",
|
||||
"function_call_output",
|
||||
]
|
||||
@@ -346,9 +346,9 @@ async def test_call_model_input_filter_keeps_duplicate_item_order_streamed() ->
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_keeps_duplicate_output_order_non_streamed() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
def filter_fn(data: CallModelData[Any]) -> ModelInputData:
|
||||
return ModelInputData(
|
||||
@@ -362,18 +362,18 @@ async def test_call_model_input_filter_keeps_duplicate_output_order_non_streamed
|
||||
run_config=RunConfig(call_model_input_filter=filter_fn),
|
||||
)
|
||||
|
||||
assert _sent_item_types(model.last_turn_args["input"]) == [
|
||||
assert _sent_item_types(model.calls[-1].input) == [
|
||||
"function_call",
|
||||
"function_call_output",
|
||||
]
|
||||
assert model.last_turn_args["input"][-1]["output"] == "new"
|
||||
assert model.calls[-1].input[-1]["output"] == "new"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_keeps_duplicate_output_order_streamed() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
async def filter_fn(data: CallModelData[Any]) -> ModelInputData:
|
||||
return ModelInputData(
|
||||
@@ -389,18 +389,18 @@ async def test_call_model_input_filter_keeps_duplicate_output_order_streamed() -
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
|
||||
assert _sent_item_types(model.last_turn_args["input"]) == [
|
||||
assert _sent_item_types(model.calls[-1].input) == [
|
||||
"function_call",
|
||||
"function_call_output",
|
||||
]
|
||||
assert model.last_turn_args["input"][-1]["output"] == "new"
|
||||
assert model.calls[-1].input[-1]["output"] == "new"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_keeps_reasoning_before_required_follower() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model.enqueue([get_text_message("ok")])
|
||||
|
||||
def filter_fn(data: CallModelData[Any]) -> ModelInputData:
|
||||
return ModelInputData(
|
||||
@@ -414,12 +414,10 @@ async def test_call_model_input_filter_keeps_reasoning_before_required_follower(
|
||||
run_config=RunConfig(call_model_input_filter=filter_fn),
|
||||
)
|
||||
|
||||
assert _sent_item_types(model.last_turn_args["input"]) == [
|
||||
assert _sent_item_types(model.calls[-1].input) == [
|
||||
"reasoning",
|
||||
"function_call",
|
||||
]
|
||||
reasoning_items = [
|
||||
item for item in model.last_turn_args["input"] if item.get("type") == "reasoning"
|
||||
]
|
||||
reasoning_items = [item for item in model.calls[-1].input if item.get("type") == "reasoning"]
|
||||
assert len(reasoning_items) == 1
|
||||
assert reasoning_items[0]["summary"] == [{"type": "summary_text", "text": "new"}]
|
||||
|
||||
@@ -9,15 +9,15 @@ from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
from agents.agent import Agent
|
||||
from agents.exceptions import UserError
|
||||
from agents.run import CallModelData, ModelInputData, RunConfig, Runner
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_sync_non_streamed_unit() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
model.set_next_output(
|
||||
model.enqueue(
|
||||
[
|
||||
ResponseOutputMessage(
|
||||
id="1",
|
||||
@@ -44,18 +44,18 @@ async def test_call_model_input_filter_sync_non_streamed_unit() -> None:
|
||||
run_config=RunConfig(call_model_input_filter=filter_fn),
|
||||
)
|
||||
|
||||
assert model.last_turn_args["system_instructions"] == "filtered-sync"
|
||||
assert isinstance(model.last_turn_args["input"], list)
|
||||
assert len(model.last_turn_args["input"]) == 2
|
||||
assert model.last_turn_args["input"][-1]["content"] == "added-sync"
|
||||
assert model.calls[-1].system_instructions == "filtered-sync"
|
||||
assert isinstance(model.calls[-1].input, list)
|
||||
assert len(model.calls[-1].input) == 2
|
||||
assert model.calls[-1].input[-1]["content"] == "added-sync"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_async_streamed_unit() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
model.set_next_output(
|
||||
model.enqueue(
|
||||
[
|
||||
ResponseOutputMessage(
|
||||
id="1",
|
||||
@@ -84,15 +84,15 @@ async def test_call_model_input_filter_async_streamed_unit() -> None:
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
|
||||
assert model.last_turn_args["system_instructions"] == "filtered-async"
|
||||
assert isinstance(model.last_turn_args["input"], list)
|
||||
assert len(model.last_turn_args["input"]) == 2
|
||||
assert model.last_turn_args["input"][-1]["content"] == "added-async"
|
||||
assert model.calls[-1].system_instructions == "filtered-async"
|
||||
assert isinstance(model.calls[-1].input, list)
|
||||
assert len(model.calls[-1].input) == 2
|
||||
assert model.calls[-1].input[-1]["content"] == "added-async"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_model_input_filter_invalid_return_type_raises_unit() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
def invalid_filter(_data: CallModelData[Any]):
|
||||
|
||||
@@ -8,13 +8,13 @@ from openai.types.responses import ResponseCompletedEvent
|
||||
from agents import Agent, Runner
|
||||
from agents.guardrail import input_guardrail
|
||||
from agents.stream_events import RawResponsesStreamEvent
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_function_tool, get_function_tool_call, get_text_message
|
||||
|
||||
|
||||
class SlowCompleteFakeModel(FakeModel):
|
||||
"""A FakeModel that delays before emitting the completed event in streaming."""
|
||||
class SlowCompleteScriptedModel(ScriptedModel):
|
||||
"""A ScriptedModel that delays before emitting the completed event in streaming."""
|
||||
|
||||
def __init__(self, delay_seconds: float):
|
||||
super().__init__()
|
||||
@@ -29,7 +29,7 @@ class SlowCompleteFakeModel(FakeModel):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_simple_streaming_with_cancel():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Joker", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
|
||||
@@ -46,14 +46,14 @@ async def test_simple_streaming_with_cancel():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_events_streaming_with_cancel():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="Joker",
|
||||
model=model,
|
||||
tools=[get_function_tool("foo", "tool_result")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[
|
||||
@@ -79,7 +79,7 @@ async def test_multiple_events_streaming_with_cancel():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_prevents_further_events():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Joker", model=model)
|
||||
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
|
||||
events = []
|
||||
@@ -95,7 +95,7 @@ async def test_cancel_prevents_further_events():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_is_idempotent():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Joker", model=model)
|
||||
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
|
||||
events = []
|
||||
@@ -110,7 +110,7 @@ async def test_cancel_is_idempotent():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_before_streaming():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Joker", model=model)
|
||||
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
|
||||
result.cancel() # Cancel before streaming
|
||||
@@ -120,7 +120,7 @@ async def test_cancel_before_streaming():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_cleans_up_resources():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Joker", model=model)
|
||||
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
|
||||
# Start streaming, then cancel
|
||||
@@ -138,7 +138,7 @@ async def test_cancel_cleans_up_resources():
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_immediate_mode_explicit():
|
||||
"""Test explicit immediate mode behaves same as default."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Joker", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
|
||||
@@ -154,8 +154,8 @@ async def test_cancel_immediate_mode_explicit():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_events_respects_asyncio_timeout_cancellation():
|
||||
model = SlowCompleteFakeModel(delay_seconds=0.5)
|
||||
model.set_next_output([get_text_message("Final response")])
|
||||
model = SlowCompleteScriptedModel(delay_seconds=0.5)
|
||||
model.enqueue([get_text_message("Final response")])
|
||||
agent = Agent(name="TimeoutTester", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
|
||||
@@ -183,7 +183,7 @@ async def test_stream_events_respects_asyncio_timeout_cancellation():
|
||||
async def test_cancel_immediate_unblocks_waiting_stream_consumer():
|
||||
block_event = asyncio.Event()
|
||||
|
||||
class BlockingFakeModel(FakeModel):
|
||||
class BlockingScriptedModel(ScriptedModel):
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions,
|
||||
@@ -213,7 +213,7 @@ async def test_cancel_immediate_unblocks_waiting_stream_consumer():
|
||||
):
|
||||
yield event
|
||||
|
||||
model = BlockingFakeModel()
|
||||
model = BlockingScriptedModel()
|
||||
agent = Agent(name="Joker", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Please tell me 5 jokes.")
|
||||
@@ -236,8 +236,8 @@ async def test_cancel_immediate_unblocks_waiting_stream_consumer():
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_loop_exception_property_is_none_on_success():
|
||||
"""run_loop_exception is None when the stream completes without error."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
agent = Agent(name="A", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="hi")
|
||||
@@ -251,7 +251,7 @@ async def test_run_loop_exception_property_is_none_on_success():
|
||||
async def test_run_loop_exception_surfaced_after_stream():
|
||||
"""run_loop_exception is set when the run loop raises before yielding events."""
|
||||
|
||||
class BoomModel(FakeModel):
|
||||
class BoomModel(ScriptedModel):
|
||||
async def get_response(self, *args, **kwargs):
|
||||
raise RuntimeError("run loop boom")
|
||||
|
||||
@@ -278,7 +278,7 @@ async def test_falsy_run_loop_exception_is_surfaced_after_stream() -> None:
|
||||
def __bool__(self) -> bool:
|
||||
return False
|
||||
|
||||
class BoomModel(FakeModel):
|
||||
class BoomModel(ScriptedModel):
|
||||
async def stream_response(self, *args, **kwargs):
|
||||
raise FalsyRuntimeError("falsy run loop boom")
|
||||
yield
|
||||
@@ -300,8 +300,8 @@ async def test_falsy_input_guardrail_exception_is_surfaced_after_stream() -> Non
|
||||
async def raising_guardrail(context, agent, input):
|
||||
raise FalsyRuntimeError("falsy guardrail boom")
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
result = Runner.run_streamed(
|
||||
Agent(name="A", model=model, input_guardrails=[raising_guardrail]),
|
||||
input="hi",
|
||||
|
||||
@@ -46,9 +46,9 @@ from agents import (
|
||||
from agents.items import ToolCallOutputItem
|
||||
from agents.run_internal import run_loop
|
||||
from agents.run_internal.run_loop import ComputerAction, ToolRunComputerAction
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import ComputerToolSafetyCheckData
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_text_message
|
||||
from .testing_processor import SPAN_PROCESSOR_TESTING
|
||||
|
||||
@@ -639,8 +639,8 @@ async def test_runner_trace_lists_ga_computer_tool_name() -> None:
|
||||
pending_safety_checks=[],
|
||||
status="completed",
|
||||
)
|
||||
model = FakeModel(tracing_enabled=True)
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel(emit_traces=True)
|
||||
model.extend(
|
||||
[
|
||||
[tool_call],
|
||||
[get_text_message("done")],
|
||||
|
||||
@@ -26,7 +26,7 @@ from agents import (
|
||||
)
|
||||
from agents.computer import Button, Computer, Environment
|
||||
from agents.models.openai_responses import Converter
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
|
||||
class FakeComputer(Computer):
|
||||
@@ -158,7 +158,7 @@ async def test_runner_disposes_computer_after_run() -> None:
|
||||
dispose = AsyncMock()
|
||||
|
||||
tool = ComputerTool(computer=ComputerProvider[FakeComputer](create=create, dispose=dispose))
|
||||
model = FakeModel(initial_output=[_make_message("done")])
|
||||
model = ScriptedModel(steps=[[_make_message("done")]])
|
||||
agent = Agent(name="ComputerAgent", model=model, tools=[tool])
|
||||
|
||||
result = await Runner.run(agent, "hello")
|
||||
@@ -167,7 +167,7 @@ async def test_runner_disposes_computer_after_run() -> None:
|
||||
create.assert_awaited_once()
|
||||
dispose.assert_awaited_once()
|
||||
dispose.assert_awaited_with(run_context=result.context_wrapper, computer=created)
|
||||
resolved_tool = cast(ComputerTool[Any], model.last_turn_args["tools"][0])
|
||||
resolved_tool = cast(ComputerTool[Any], model.calls[-1].tools[0])
|
||||
assert resolved_tool is not tool
|
||||
assert resolved_tool.computer is created
|
||||
|
||||
@@ -194,27 +194,29 @@ async def test_runner_preserves_concrete_computer_tool_identity_for_hooks() -> N
|
||||
self.ended.append(tool)
|
||||
|
||||
tool = ComputerTool(computer=FakeComputer("concrete"))
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
ResponseComputerToolCall(
|
||||
id="computer-call",
|
||||
type="computer_call",
|
||||
action=ActionScreenshot(type="screenshot"),
|
||||
call_id="computer-call",
|
||||
pending_safety_checks=[],
|
||||
status="completed",
|
||||
)
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
ResponseComputerToolCall(
|
||||
id="computer-call",
|
||||
type="computer_call",
|
||||
action=ActionScreenshot(type="screenshot"),
|
||||
call_id="computer-call",
|
||||
pending_safety_checks=[],
|
||||
status="completed",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
model.set_next_output([_make_message("done")])
|
||||
model.enqueue([_make_message("done")])
|
||||
agent = Agent(name="ComputerAgent", model=model, tools=[tool])
|
||||
hooks = IdentityHooks()
|
||||
|
||||
result = await Runner.run(agent, "hello", hooks=hooks)
|
||||
|
||||
assert result.final_output == "done"
|
||||
assert model.first_turn_args is not None
|
||||
assert model.first_turn_args["tools"][0] is tool
|
||||
assert bool(model.calls)
|
||||
assert model.calls[0].tools[0] is tool
|
||||
assert hooks.started == [tool]
|
||||
assert hooks.ended == [tool]
|
||||
|
||||
@@ -239,10 +241,10 @@ async def test_concurrent_runs_keep_computer_provider_instances_isolated() -> No
|
||||
release_model = [asyncio.Event(), asyncio.Event()]
|
||||
serialized_widths: list[int] = []
|
||||
|
||||
class GatedSerializationModel(FakeModel):
|
||||
class GatedSerializationModel(ScriptedModel):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(initial_output=[_make_message("done")])
|
||||
self.set_next_output([_make_message("done")])
|
||||
super().__init__(steps=[[_make_message("done")]])
|
||||
self.enqueue([_make_message("done")])
|
||||
self.call_count = 0
|
||||
|
||||
async def get_response(
|
||||
@@ -331,7 +333,7 @@ async def test_streamed_run_disposes_computer_after_completion() -> None:
|
||||
dispose = AsyncMock()
|
||||
|
||||
tool = ComputerTool(computer=ComputerProvider[FakeComputer](create=create, dispose=dispose))
|
||||
model = FakeModel(initial_output=[_make_message("done")])
|
||||
model = ScriptedModel(steps=[[_make_message("done")]])
|
||||
agent = Agent(name="ComputerAgent", model=model, tools=[tool])
|
||||
|
||||
streamed_result = Runner.run_streamed(agent, "hello")
|
||||
@@ -342,6 +344,6 @@ async def test_streamed_run_disposes_computer_after_completion() -> None:
|
||||
create.assert_awaited_once()
|
||||
dispose.assert_awaited_once()
|
||||
dispose.assert_awaited_with(run_context=streamed_result.context_wrapper, computer=created)
|
||||
resolved_tool = cast(ComputerTool[Any], model.last_turn_args["tools"][0])
|
||||
resolved_tool = cast(ComputerTool[Any], model.calls[-1].tools[0])
|
||||
assert resolved_tool is not tool
|
||||
assert resolved_tool.computer is created
|
||||
|
||||
@@ -65,13 +65,13 @@ from agents.run_internal.tool_execution import (
|
||||
resolve_approval_rejection_message,
|
||||
)
|
||||
from agents.run_state import _deserialize_items
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool_context import ToolContext
|
||||
from agents.tracing.processor_interface import TracingProcessor
|
||||
from agents.tracing.provider import SynchronousMultiTracingProcessor
|
||||
from agents.tracing.spans import Span
|
||||
from agents.tracing.traces import Trace
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_function_tool_call, get_text_message
|
||||
from .utils.simple_session import SimpleListSession
|
||||
|
||||
@@ -1215,9 +1215,9 @@ async def test_run_surfaces_redacted_output_validation_error(
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model.enqueue([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
session = SimpleListSession(
|
||||
session_id="redacted-run",
|
||||
history=[{"role": "user", "content": _MODEL_OUTPUT_SECRET}],
|
||||
@@ -1243,9 +1243,9 @@ def test_run_sync_surfaces_redacted_output_validation_error_without_runner_data(
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model.enqueue([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
session = SimpleListSession(
|
||||
session_id="redacted-run-sync",
|
||||
history=[{"role": "user", "content": _MODEL_OUTPUT_SECRET}],
|
||||
@@ -1273,7 +1273,7 @@ async def test_run_preserves_diagnostic_wrapper_traceback_locals(
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False)
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
|
||||
diagnostic_input = "DIAGNOSTIC_RUNNER_INPUT_SECRET"
|
||||
model = FakeModel(initial_output=[get_text_message('{"answer": "missing count"}')])
|
||||
model = ScriptedModel(steps=[[get_text_message('{"answer": "missing count"}')]])
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
session = SimpleListSession(session_id="diagnostic-runner")
|
||||
|
||||
@@ -1292,7 +1292,7 @@ def test_run_sync_preserves_diagnostic_wrapper_traceback_locals(
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False)
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
|
||||
diagnostic_input = "DIAGNOSTIC_RUNNER_SYNC_INPUT_SECRET"
|
||||
model = FakeModel(initial_output=[get_text_message('{"answer": "missing count"}')])
|
||||
model = ScriptedModel(steps=[[get_text_message('{"answer": "missing count"}')]])
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
session = SimpleListSession(session_id="diagnostic-runner-sync")
|
||||
|
||||
@@ -1311,9 +1311,9 @@ async def test_streamed_run_surfaces_redacted_output_validation_error(
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
model.set_next_output([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model.enqueue([get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
result = Runner.run_streamed(agent, "go")
|
||||
|
||||
with pytest.raises(ModelBehaviorError) as exc_info:
|
||||
@@ -1334,7 +1334,7 @@ async def test_streamed_run_loop_exception_follows_model_data_policy(
|
||||
redacted: bool,
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
|
||||
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
result = Runner.run_streamed(agent, "go")
|
||||
|
||||
@@ -1378,7 +1378,7 @@ async def test_streamed_output_guardrail_omits_run_data_from_redacted_error(
|
||||
AgentOutputSchema(_RequiredOutput).validate_json(payload)
|
||||
raise AssertionError("validation should fail") # pragma: no cover
|
||||
|
||||
model = FakeModel(initial_output=[get_text_message(_MODEL_OUTPUT_SECRET)])
|
||||
model = ScriptedModel(steps=[[get_text_message(_MODEL_OUTPUT_SECRET)]])
|
||||
agent = Agent(
|
||||
name="A",
|
||||
model=model,
|
||||
@@ -1434,7 +1434,7 @@ async def test_streamed_session_error_after_output_guardrail_respects_redaction(
|
||||
raise AssertionError("validation should fail") # pragma: no cover
|
||||
|
||||
caplog.set_level(logging.ERROR, logger="openai.agents")
|
||||
model = FakeModel(initial_output=[get_text_message(_MODEL_OUTPUT_SECRET)])
|
||||
model = ScriptedModel(steps=[[get_text_message(_MODEL_OUTPUT_SECRET)]])
|
||||
agent = Agent(
|
||||
name="A",
|
||||
model=model,
|
||||
@@ -1536,7 +1536,7 @@ async def test_streamed_session_hostile_error_after_redacted_output_guardrail_is
|
||||
|
||||
agent = Agent(
|
||||
name="A",
|
||||
model=FakeModel(initial_output=[get_text_message(_MODEL_OUTPUT_SECRET)]),
|
||||
model=ScriptedModel(steps=[[get_text_message(_MODEL_OUTPUT_SECRET)]]),
|
||||
output_guardrails=[OutputGuardrail(guardrail_function=output_guardrail)],
|
||||
)
|
||||
result = Runner.run_streamed(agent, "go", session=FailingFinalTurnSession())
|
||||
@@ -1569,7 +1569,7 @@ async def test_streamed_input_guardrail_omits_run_data_from_redacted_error(
|
||||
AgentOutputSchema(_RequiredOutput).validate_json(payload)
|
||||
raise AssertionError("validation should fail") # pragma: no cover
|
||||
|
||||
model = FakeModel(initial_output=[get_text_message("unused")])
|
||||
model = ScriptedModel(steps=[[get_text_message("unused")]])
|
||||
agent = Agent(
|
||||
name="A",
|
||||
model=model,
|
||||
@@ -1596,7 +1596,7 @@ async def test_invalid_final_output_handler_receives_detached_redacted_error(
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
|
||||
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
retained_errors: list[ModelBehaviorError] = []
|
||||
|
||||
@@ -1636,7 +1636,7 @@ async def test_invalid_final_output_handler_invalid_fallback_preserves_redaction
|
||||
) -> None:
|
||||
fallback_secret = "INVALID_HANDLER_FALLBACK_SECRET"
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
|
||||
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
|
||||
def invalid_fallback(_data: RunErrorHandlerInput[None]) -> dict[str, str]:
|
||||
@@ -1681,10 +1681,8 @@ async def test_invalid_final_output_handler_fallback_serialization_follows_redac
|
||||
) -> None:
|
||||
fallback_secret = "PERMISSIVE_HANDLER_FALLBACK_SECRET"
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_text_message(f'{{"payload": "{_MODEL_OUTPUT_SECRET}", "count": "invalid"}}')
|
||||
]
|
||||
model = ScriptedModel(
|
||||
steps=[[get_text_message(f'{{"payload": "{_MODEL_OUTPUT_SECRET}", "count": "invalid"}}')]]
|
||||
)
|
||||
agent = Agent(name="A", model=model, output_type=_PermissiveFallbackOutput)
|
||||
|
||||
@@ -1743,7 +1741,7 @@ async def test_empty_final_output_handler_fallback_serialization_follows_redacti
|
||||
) -> None:
|
||||
fallback_secret = "EMPTY_HANDLER_FALLBACK_SECRET"
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
|
||||
model = FakeModel(initial_output=[])
|
||||
model = ScriptedModel(steps=[[]])
|
||||
agent = Agent(name="A", model=model, output_type=_PermissiveFallbackOutput)
|
||||
|
||||
def permissive_fallback(_data: RunErrorHandlerInput[None]) -> RunErrorHandlerResult:
|
||||
@@ -1792,7 +1790,7 @@ async def test_invalid_final_output_handler_failure_preserves_redaction(
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
|
||||
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
|
||||
def fail(data: RunErrorHandlerInput[None]) -> None:
|
||||
@@ -1831,7 +1829,7 @@ async def test_invalid_final_output_handler_hostile_failure_preserves_redaction(
|
||||
) -> None:
|
||||
handler_secret = "HOSTILE_HANDLER_FAILURE_SECRET"
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True)
|
||||
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
|
||||
def fail(_data: RunErrorHandlerInput[None]) -> None:
|
||||
@@ -1870,7 +1868,7 @@ async def test_invalid_final_output_handler_failure_preserves_diagnostic_context
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False)
|
||||
model = FakeModel(initial_output=[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')])
|
||||
model = ScriptedModel(steps=[[get_text_message(f'{{"answer": "{_MODEL_OUTPUT_SECRET}"}}')]])
|
||||
agent = Agent(name="A", model=model, output_type=_RequiredOutput)
|
||||
|
||||
def fail(_data: RunErrorHandlerInput[None]) -> None:
|
||||
@@ -1914,8 +1912,8 @@ async def test_multiturn_output_validation_error_run_data_follows_redaction_poli
|
||||
def record_value(value: str) -> str:
|
||||
return "recorded"
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
|
||||
+111
-110
@@ -30,6 +30,7 @@ from agents import (
|
||||
)
|
||||
from agents.agent import ToolsToFinalOutputResult
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import FunctionToolResult, function_tool
|
||||
from examples.financial_research_agent.agents.verifier_agent import (
|
||||
VerificationIssue,
|
||||
@@ -55,7 +56,6 @@ from examples.sandbox.sandbox_agents_as_tools import (
|
||||
from examples.tools.web_search_filters import _normalized_source_urls
|
||||
from examples.web_search_utils import extract_url_citations, extract_web_search_source_urls
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_final_output_message,
|
||||
get_function_tool_call,
|
||||
@@ -341,16 +341,16 @@ class OutlineCheckerOutput:
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_as_judge_loop_handles_dataclass_feedback() -> None:
|
||||
"""Mimics the llm_as_a_judge example: loop until the evaluator passes the outline."""
|
||||
outline_model = FakeModel()
|
||||
outline_model.add_multiple_turn_outputs(
|
||||
outline_model = ScriptedModel()
|
||||
outline_model.extend(
|
||||
[
|
||||
[get_text_message("Outline v1")],
|
||||
[get_text_message("Outline v2")],
|
||||
]
|
||||
)
|
||||
|
||||
judge_model = FakeModel()
|
||||
judge_model.add_multiple_turn_outputs(
|
||||
judge_model = ScriptedModel()
|
||||
judge_model.extend(
|
||||
[
|
||||
[
|
||||
get_final_output_message(
|
||||
@@ -400,14 +400,14 @@ async def test_llm_as_judge_loop_handles_dataclass_feedback() -> None:
|
||||
|
||||
assert latest_outline == "Outline v2"
|
||||
assert len(conversation) == 4
|
||||
assert judge_model.last_turn_args["input"] == conversation
|
||||
assert judge_model.calls[-1].input == conversation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parallel_translation_flow_reuses_runner_outputs() -> None:
|
||||
"""Covers the parallelization example by feeding multiple translations into a picker agent."""
|
||||
translation_model = FakeModel()
|
||||
translation_model.add_multiple_turn_outputs(
|
||||
translation_model = ScriptedModel()
|
||||
translation_model.extend(
|
||||
[
|
||||
[get_text_message("Uno")],
|
||||
[get_text_message("Dos")],
|
||||
@@ -416,8 +416,8 @@ async def test_parallel_translation_flow_reuses_runner_outputs() -> None:
|
||||
)
|
||||
spanish_agent = Agent(name="spanish_agent", model=translation_model)
|
||||
|
||||
picker_model = FakeModel()
|
||||
picker_model.set_next_output([get_text_message("Pick: Dos")])
|
||||
picker_model = ScriptedModel()
|
||||
picker_model.enqueue([get_text_message("Pick: Dos")])
|
||||
picker_agent = Agent(name="picker", model=picker_model)
|
||||
|
||||
translations: list[str] = []
|
||||
@@ -433,7 +433,7 @@ async def test_parallel_translation_flow_reuses_runner_outputs() -> None:
|
||||
|
||||
assert translations == ["Uno", "Dos", "Tres"]
|
||||
assert picker_result.final_output == "Pick: Dos"
|
||||
assert picker_model.last_turn_args["input"] == [
|
||||
assert picker_model.calls[-1].input == [
|
||||
{"content": f"Input: Hello\n\nTranslations:\n{combined}", "role": "user"}
|
||||
]
|
||||
|
||||
@@ -441,18 +441,18 @@ async def test_parallel_translation_flow_reuses_runner_outputs() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_deterministic_story_flow_stops_when_checker_blocks() -> None:
|
||||
"""Mimics deterministic flow: stop early when quality gate fails."""
|
||||
outline_model = FakeModel()
|
||||
outline_model.set_next_output([get_text_message("Outline v1")])
|
||||
checker_model = FakeModel()
|
||||
checker_model.set_next_output(
|
||||
outline_model = ScriptedModel()
|
||||
outline_model.enqueue([get_text_message("Outline v1")])
|
||||
checker_model = ScriptedModel()
|
||||
checker_model.enqueue(
|
||||
[
|
||||
get_final_output_message(
|
||||
json.dumps({"response": {"good_quality": False, "is_scifi": True}})
|
||||
)
|
||||
]
|
||||
)
|
||||
story_model = FakeModel()
|
||||
story_model.set_next_output(RuntimeError("story should not run"))
|
||||
story_model = ScriptedModel()
|
||||
story_model.enqueue(RuntimeError("story should not run"))
|
||||
|
||||
outline_agent = Agent(name="outline", model=outline_model)
|
||||
checker_agent = Agent(
|
||||
@@ -474,24 +474,24 @@ async def test_deterministic_story_flow_stops_when_checker_blocks() -> None:
|
||||
assert decision.is_scifi is True
|
||||
if decision.good_quality and decision.is_scifi:
|
||||
await Runner.run(story_agent, outline_result.final_output)
|
||||
assert story_model.first_turn_args is None, "story agent should never be invoked when gated"
|
||||
assert not story_model.calls, "story agent should never be invoked when gated"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deterministic_story_flow_runs_story_on_pass() -> None:
|
||||
"""Mimics deterministic flow: run full path when checker approves."""
|
||||
outline_model = FakeModel()
|
||||
outline_model.set_next_output([get_text_message("Outline ready")])
|
||||
checker_model = FakeModel()
|
||||
checker_model.set_next_output(
|
||||
outline_model = ScriptedModel()
|
||||
outline_model.enqueue([get_text_message("Outline ready")])
|
||||
checker_model = ScriptedModel()
|
||||
checker_model.enqueue(
|
||||
[
|
||||
get_final_output_message(
|
||||
json.dumps({"response": {"good_quality": True, "is_scifi": True}})
|
||||
)
|
||||
]
|
||||
)
|
||||
story_model = FakeModel()
|
||||
story_model.set_next_output([get_text_message("Final story")])
|
||||
story_model = ScriptedModel()
|
||||
story_model.enqueue([get_text_message("Final story")])
|
||||
|
||||
outline_agent = Agent(name="outline", model=outline_model)
|
||||
checker_agent = Agent(
|
||||
@@ -513,14 +513,14 @@ async def test_deterministic_story_flow_runs_story_on_pass() -> None:
|
||||
|
||||
story_result = await Runner.run(story_agent, outline_result.final_output)
|
||||
assert story_result.final_output == "Final story"
|
||||
assert story_model.last_turn_args["input"] == [{"content": "Outline ready", "role": "user"}]
|
||||
assert story_model.calls[-1].input == [{"content": "Outline ready", "role": "user"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_stream_emits_text_and_updates_inputs() -> None:
|
||||
"""Mimics routing example stream: text deltas flow through and input history updates."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Bonjour")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Bonjour")])
|
||||
triage_agent = Agent(name="triage_agent", model=model)
|
||||
|
||||
streamed = Runner.run_streamed(triage_agent, input="Salut")
|
||||
@@ -557,8 +557,8 @@ class MathHomeworkOutput(BaseModel):
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_guardrail_agent_trips_and_returns_info() -> None:
|
||||
"""Mimics math guardrail example: guardrail agent runs and trips before main agent completes."""
|
||||
guardrail_model = FakeModel()
|
||||
guardrail_model.set_next_output(
|
||||
guardrail_model = ScriptedModel()
|
||||
guardrail_model.enqueue(
|
||||
[
|
||||
get_final_output_message(
|
||||
json.dumps({"reasoning": "math detected", "is_math_homework": True})
|
||||
@@ -577,8 +577,8 @@ async def test_input_guardrail_agent_trips_and_returns_info() -> None:
|
||||
output_info=output, tripwire_triggered=output.is_math_homework
|
||||
)
|
||||
|
||||
main_model = FakeModel()
|
||||
main_model.set_next_output([get_text_message("Should not run")])
|
||||
main_model = ScriptedModel()
|
||||
main_model.enqueue([get_text_message("Should not run")])
|
||||
main_agent = Agent(name="main", model=main_model, input_guardrails=[math_guardrail])
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered) as excinfo:
|
||||
@@ -610,8 +610,8 @@ async def test_output_guardrail_blocks_sensitive_data() -> None:
|
||||
tripwire_triggered=contains_phone,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[
|
||||
get_final_output_message(
|
||||
json.dumps(
|
||||
@@ -642,8 +642,8 @@ async def test_output_guardrail_blocks_sensitive_data() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_guardrail_style_cancel_after_threshold() -> None:
|
||||
"""Mimics streaming guardrail example: stop streaming once threshold is reached."""
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[
|
||||
get_text_message("Chunk1 "),
|
||||
get_text_message("Chunk2 "),
|
||||
@@ -673,8 +673,8 @@ async def test_streaming_guardrail_style_cancel_after_threshold() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_cancel_after_turn_allows_turn_completion() -> None:
|
||||
"""Ensure cancel(after_turn) lets the current turn finish and final_output is populated."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Hello"), get_text_message("World")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Hello"), get_text_message("World")])
|
||||
agent = Agent(name="talkative", model=model)
|
||||
|
||||
streamed = Runner.run_streamed(agent, input="Hi")
|
||||
@@ -696,12 +696,12 @@ async def test_streaming_cancel_after_turn_allows_turn_completion() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_handoff_emits_agent_updated_event() -> None:
|
||||
"""Mimics routing handoff stream: emits AgentUpdatedStreamEvent and switches agent."""
|
||||
delegate_model = FakeModel()
|
||||
delegate_model.set_next_output([get_text_message("delegate reply")])
|
||||
delegate_model = ScriptedModel()
|
||||
delegate_model.enqueue([get_text_message("delegate reply")])
|
||||
delegate_agent = Agent(name="delegate", model=delegate_model)
|
||||
|
||||
triage_model = FakeModel()
|
||||
triage_model.set_next_output(
|
||||
triage_model = ScriptedModel()
|
||||
triage_model.enqueue(
|
||||
[
|
||||
get_text_message("triage summary"),
|
||||
get_handoff_tool_call(delegate_agent),
|
||||
@@ -749,8 +749,8 @@ async def test_agent_as_tool_streaming_example_collects_events() -> None:
|
||||
|
||||
billing_tool.on_invoke_tool = fake_invoke
|
||||
|
||||
main_model = FakeModel()
|
||||
main_model.add_multiple_turn_outputs(
|
||||
main_model = ScriptedModel()
|
||||
main_model.extend(
|
||||
[
|
||||
[get_function_tool_call("billing_agent", json.dumps({"input": "Need bill"}))],
|
||||
[get_text_message("Final answer")],
|
||||
@@ -776,8 +776,8 @@ async def test_agent_as_tool_streaming_example_collects_events() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sandbox_agents_as_tools_example_serializes_structured_reviews() -> None:
|
||||
pricing_model = FakeModel()
|
||||
pricing_model.set_next_output(
|
||||
pricing_model = ScriptedModel()
|
||||
pricing_model.enqueue(
|
||||
[
|
||||
get_final_output_message(
|
||||
json.dumps(
|
||||
@@ -793,8 +793,8 @@ async def test_sandbox_agents_as_tools_example_serializes_structured_reviews() -
|
||||
)
|
||||
]
|
||||
)
|
||||
rollout_model = FakeModel()
|
||||
rollout_model.set_next_output(
|
||||
rollout_model = ScriptedModel()
|
||||
rollout_model.enqueue(
|
||||
[
|
||||
get_final_output_message(
|
||||
json.dumps(
|
||||
@@ -812,8 +812,8 @@ async def test_sandbox_agents_as_tools_example_serializes_structured_reviews() -
|
||||
)
|
||||
]
|
||||
)
|
||||
orchestrator_model = FakeModel()
|
||||
orchestrator_model.add_multiple_turn_outputs(
|
||||
orchestrator_model = ScriptedModel()
|
||||
orchestrator_model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -878,7 +878,7 @@ async def test_sandbox_agents_as_tools_example_serializes_structured_reviews() -
|
||||
assert result.final_output == "Recommendation complete"
|
||||
outer_second_turn_input = cast(
|
||||
list[dict[str, Any]],
|
||||
orchestrator_model.last_turn_args["input"],
|
||||
orchestrator_model.calls[-1].input,
|
||||
)
|
||||
outer_tool_outputs = [
|
||||
item for item in outer_second_turn_input if item.get("type") == "function_call_output"
|
||||
@@ -962,8 +962,8 @@ async def test_forcing_tool_use_behaviors_align_with_example() -> None:
|
||||
return f"{city}: Sunny"
|
||||
|
||||
# default: run_llm_again -> model responds after tool call
|
||||
default_model = FakeModel()
|
||||
default_model.add_multiple_turn_outputs(
|
||||
default_model = ScriptedModel()
|
||||
default_model.extend(
|
||||
[
|
||||
[
|
||||
get_text_message("Tool call coming"),
|
||||
@@ -986,8 +986,8 @@ async def test_forcing_tool_use_behaviors_align_with_example() -> None:
|
||||
assert len(default_result.raw_responses) == 2
|
||||
|
||||
# first_tool: stop_on_first_tool -> final output from first tool result
|
||||
first_model = FakeModel()
|
||||
first_model.set_next_output(
|
||||
first_model = ScriptedModel()
|
||||
first_model.enqueue(
|
||||
[
|
||||
get_text_message("Tool call coming"),
|
||||
get_function_tool_call("get_weather", json.dumps({"city": "Paris"})),
|
||||
@@ -1014,8 +1014,8 @@ async def test_forcing_tool_use_behaviors_align_with_example() -> None:
|
||||
is_final_output=True, final_output=f"Custom:{results[0].output}"
|
||||
)
|
||||
|
||||
custom_model = FakeModel()
|
||||
custom_model.set_next_output(
|
||||
custom_model = ScriptedModel()
|
||||
custom_model.enqueue(
|
||||
[
|
||||
get_text_message("Tool call coming"),
|
||||
get_function_tool_call("get_weather", json.dumps({"city": "Berlin"})),
|
||||
@@ -1037,12 +1037,12 @@ async def test_forcing_tool_use_behaviors_align_with_example() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_multi_turn_continues_with_handoff_agent() -> None:
|
||||
"""Mimics routing example multi-turn: first handoff, then continue with delegated agent."""
|
||||
delegate_model = FakeModel()
|
||||
delegate_model.set_next_output([get_text_message("Bonjour")])
|
||||
delegate_model = ScriptedModel()
|
||||
delegate_model.enqueue([get_text_message("Bonjour")])
|
||||
delegate_agent = Agent(name="delegate", model=delegate_model)
|
||||
|
||||
triage_model = FakeModel()
|
||||
triage_model.add_multiple_turn_outputs(
|
||||
triage_model = ScriptedModel()
|
||||
triage_model.extend(
|
||||
[
|
||||
[get_handoff_tool_call(delegate_agent)],
|
||||
[get_text_message("handoff completed")],
|
||||
@@ -1055,13 +1055,13 @@ async def test_routing_multi_turn_continues_with_handoff_agent() -> None:
|
||||
assert first_result.last_agent == delegate_agent
|
||||
|
||||
# Next user turn continues with delegate.
|
||||
delegate_model.set_next_output([get_text_message("Encore?")])
|
||||
delegate_model.enqueue([get_text_message("Encore?")])
|
||||
follow_up_input = first_result.to_input_list()
|
||||
follow_up_input.append({"role": "user", "content": "Encore!"})
|
||||
|
||||
second_result = await Runner.run(delegate_agent, follow_up_input)
|
||||
assert second_result.final_output == "Encore?"
|
||||
assert delegate_model.last_turn_args["input"] == follow_up_input
|
||||
assert delegate_model.calls[-1].input == follow_up_input
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1084,19 +1084,19 @@ async def test_agents_as_tools_conditional_enabling_matches_preference() -> None
|
||||
]
|
||||
|
||||
for preference, expected_tools in scenarios:
|
||||
spanish_model = FakeModel()
|
||||
spanish_model.set_next_output([get_text_message("ES hola")])
|
||||
spanish_model = ScriptedModel()
|
||||
spanish_model.enqueue([get_text_message("ES hola")])
|
||||
spanish_agent = Agent(name="spanish", model=spanish_model)
|
||||
|
||||
french_model = FakeModel()
|
||||
french_model.set_next_output([get_text_message("FR bonjour")])
|
||||
french_model = ScriptedModel()
|
||||
french_model.enqueue([get_text_message("FR bonjour")])
|
||||
french_agent = Agent(name="french", model=french_model)
|
||||
|
||||
italian_model = FakeModel()
|
||||
italian_model.set_next_output([get_text_message("IT ciao")])
|
||||
italian_model = ScriptedModel()
|
||||
italian_model.enqueue([get_text_message("IT ciao")])
|
||||
italian_agent = Agent(name="italian", model=italian_model)
|
||||
|
||||
orchestrator_model = FakeModel()
|
||||
orchestrator_model = ScriptedModel()
|
||||
# Build tool calls only for expected tools to avoid missing-tool errors.
|
||||
tool_calls = [
|
||||
get_function_tool_call(
|
||||
@@ -1106,7 +1106,7 @@ async def test_agents_as_tools_conditional_enabling_matches_preference() -> None
|
||||
)
|
||||
for tool_name in sorted(expected_tools)
|
||||
]
|
||||
orchestrator_model.add_multiple_turn_outputs([tool_calls, [get_text_message("Done")]])
|
||||
orchestrator_model.extend([tool_calls, [get_text_message("Done")]])
|
||||
|
||||
context = AppContext(language_preference=preference)
|
||||
|
||||
@@ -1137,35 +1137,35 @@ async def test_agents_as_tools_conditional_enabling_matches_preference() -> None
|
||||
|
||||
assert result.final_output == "Done"
|
||||
assert (
|
||||
spanish_model.first_turn_args is not None
|
||||
bool(spanish_model.calls)
|
||||
if "respond_spanish" in expected_tools
|
||||
else spanish_model.first_turn_args is None
|
||||
else not spanish_model.calls
|
||||
)
|
||||
assert (
|
||||
french_model.first_turn_args is not None
|
||||
bool(french_model.calls)
|
||||
if "respond_french" in expected_tools
|
||||
else french_model.first_turn_args is None
|
||||
else not french_model.calls
|
||||
)
|
||||
assert (
|
||||
italian_model.first_turn_args is not None
|
||||
bool(italian_model.calls)
|
||||
if "respond_italian" in expected_tools
|
||||
else italian_model.first_turn_args is None
|
||||
else not italian_model.calls
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agents_as_tools_orchestrator_runs_multiple_translations() -> None:
|
||||
"""Orchestrator calls multiple translation agent tools then summarizes."""
|
||||
spanish_model = FakeModel()
|
||||
spanish_model.set_next_output([get_text_message("ES hola")])
|
||||
spanish_model = ScriptedModel()
|
||||
spanish_model.enqueue([get_text_message("ES hola")])
|
||||
spanish_agent = Agent(name="spanish", model=spanish_model)
|
||||
|
||||
french_model = FakeModel()
|
||||
french_model.set_next_output([get_text_message("FR bonjour")])
|
||||
french_model = ScriptedModel()
|
||||
french_model.enqueue([get_text_message("FR bonjour")])
|
||||
french_agent = Agent(name="french", model=french_model)
|
||||
|
||||
orchestrator_model = FakeModel()
|
||||
orchestrator_model.add_multiple_turn_outputs(
|
||||
orchestrator_model = ScriptedModel()
|
||||
orchestrator_model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -1197,8 +1197,8 @@ async def test_agents_as_tools_orchestrator_runs_multiple_translations() -> None
|
||||
result = await Runner.run(orchestrator, "Hi")
|
||||
|
||||
assert result.final_output == "Summary complete"
|
||||
assert spanish_model.last_turn_args["input"] == [{"content": "Hi", "role": "user"}]
|
||||
assert french_model.last_turn_args["input"] == [{"content": "Hi", "role": "user"}]
|
||||
assert spanish_model.calls[-1].input == [{"content": "Hi", "role": "user"}]
|
||||
assert french_model.calls[-1].input == [{"content": "Hi", "role": "user"}]
|
||||
assert len(result.raw_responses) == 3
|
||||
|
||||
|
||||
@@ -1209,14 +1209,15 @@ async def test_agents_as_tools_subagent_cancellation_preserves_parent_final_outp
|
||||
async def _cancel_tool() -> str:
|
||||
raise asyncio.CancelledError("tool-cancelled")
|
||||
|
||||
success_model = FakeModel()
|
||||
success_model.set_next_output([get_text_message("Status: ok")])
|
||||
success_model = ScriptedModel()
|
||||
success_model.enqueue([get_text_message("Status: ok")])
|
||||
success_agent = Agent(name="status", model=success_model)
|
||||
|
||||
observability_model = FakeModel()
|
||||
observability_model.set_next_output(
|
||||
observability_model = ScriptedModel()
|
||||
observability_model.enqueue(
|
||||
[get_function_tool_call("cancel_tool", "{}", call_id="inner_cancel")]
|
||||
)
|
||||
observability_model.enqueue([])
|
||||
observability_agent = Agent(
|
||||
name="observability",
|
||||
model=observability_model,
|
||||
@@ -1224,8 +1225,8 @@ async def test_agents_as_tools_subagent_cancellation_preserves_parent_final_outp
|
||||
model_settings=ModelSettings(tool_choice="required"),
|
||||
)
|
||||
|
||||
orchestrator_model = FakeModel()
|
||||
orchestrator_model.add_multiple_turn_outputs(
|
||||
orchestrator_model = ScriptedModel()
|
||||
orchestrator_model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -1257,11 +1258,11 @@ async def test_agents_as_tools_subagent_cancellation_preserves_parent_final_outp
|
||||
|
||||
assert result.final_output == "Summary complete"
|
||||
assert len(result.raw_responses) == 2
|
||||
assert success_model.last_turn_args["input"] == [{"content": "Hi", "role": "user"}]
|
||||
assert observability_model.first_turn_args is not None
|
||||
assert observability_model.first_turn_args["input"] == [{"content": "Hi", "role": "user"}]
|
||||
assert success_model.calls[-1].input == [{"content": "Hi", "role": "user"}]
|
||||
assert bool(observability_model.calls)
|
||||
assert observability_model.calls[0].input == [{"content": "Hi", "role": "user"}]
|
||||
|
||||
second_turn_input = cast(list[dict[str, Any]], orchestrator_model.last_turn_args["input"])
|
||||
second_turn_input = cast(list[dict[str, Any]], orchestrator_model.calls[-1].input)
|
||||
tool_outputs = [
|
||||
item for item in second_turn_input if item.get("type") == "function_call_output"
|
||||
]
|
||||
@@ -1294,12 +1295,12 @@ async def test_agents_as_tools_streaming_subagent_cancellation_preserves_parent_
|
||||
async def on_stream(event: AgentToolStreamEvent) -> None:
|
||||
received_events.append(event)
|
||||
|
||||
status_model = FakeModel()
|
||||
status_model.set_next_output([get_text_message("Status: ok")])
|
||||
status_model = ScriptedModel()
|
||||
status_model.enqueue([get_text_message("Status: ok")])
|
||||
status_agent = Agent(name="status", model=status_model)
|
||||
|
||||
observability_model = FakeModel()
|
||||
observability_model.add_multiple_turn_outputs(
|
||||
observability_model = ScriptedModel()
|
||||
observability_model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call("ok_tool", "{}", call_id="inner_ok"),
|
||||
@@ -1318,8 +1319,8 @@ async def test_agents_as_tools_streaming_subagent_cancellation_preserves_parent_
|
||||
model_settings=ModelSettings(tool_choice="required"),
|
||||
)
|
||||
|
||||
orchestrator_model = FakeModel()
|
||||
orchestrator_model.add_multiple_turn_outputs(
|
||||
orchestrator_model = ScriptedModel()
|
||||
orchestrator_model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -1356,12 +1357,12 @@ async def test_agents_as_tools_streaming_subagent_cancellation_preserves_parent_
|
||||
assert result.final_output == "Summary complete"
|
||||
assert len(result.raw_responses) == 2
|
||||
assert received_events, "on_stream should confirm the nested streaming path ran"
|
||||
assert status_model.last_turn_args["input"] == [{"content": "Hi", "role": "user"}]
|
||||
assert observability_model.last_turn_args is not None
|
||||
assert status_model.calls[-1].input == [{"content": "Hi", "role": "user"}]
|
||||
assert bool(observability_model.calls)
|
||||
|
||||
nested_second_turn_input = cast(
|
||||
list[dict[str, Any]],
|
||||
observability_model.last_turn_args["input"],
|
||||
observability_model.calls[-1].input,
|
||||
)
|
||||
nested_tool_outputs = [
|
||||
item for item in nested_second_turn_input if item.get("type") == "function_call_output"
|
||||
@@ -1383,7 +1384,7 @@ async def test_agents_as_tools_streaming_subagent_cancellation_preserves_parent_
|
||||
|
||||
outer_second_turn_input = cast(
|
||||
list[dict[str, Any]],
|
||||
orchestrator_model.last_turn_args["input"],
|
||||
orchestrator_model.calls[-1].input,
|
||||
)
|
||||
outer_tool_outputs = [
|
||||
item for item in outer_second_turn_input if item.get("type") == "function_call_output"
|
||||
@@ -1409,12 +1410,12 @@ async def test_agents_as_tools_failure_error_function_none_reraises_cancelled_er
|
||||
async def _cancel_tool() -> str:
|
||||
raise asyncio.CancelledError("tool-cancelled")
|
||||
|
||||
status_model = FakeModel()
|
||||
status_model.set_next_output([get_text_message("Status: ok")])
|
||||
status_model = ScriptedModel()
|
||||
status_model.enqueue([get_text_message("Status: ok")])
|
||||
status_agent = Agent(name="status", model=status_model)
|
||||
|
||||
observability_model = FakeModel()
|
||||
observability_model.set_next_output(
|
||||
observability_model = ScriptedModel()
|
||||
observability_model.enqueue(
|
||||
[get_function_tool_call("cancel_tool", "{}", call_id="inner_cancel")]
|
||||
)
|
||||
observability_agent = Agent(
|
||||
@@ -1426,8 +1427,8 @@ async def test_agents_as_tools_failure_error_function_none_reraises_cancelled_er
|
||||
model_settings=ModelSettings(tool_choice="required"),
|
||||
)
|
||||
|
||||
orchestrator_model = FakeModel()
|
||||
orchestrator_model.set_next_output(
|
||||
orchestrator_model = ScriptedModel()
|
||||
orchestrator_model.enqueue(
|
||||
[
|
||||
get_function_tool_call(
|
||||
"status_agent",
|
||||
|
||||
+18
-18
@@ -8,9 +8,9 @@ import pytest
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from agents import Agent, RunContextWrapper, RunHooks, Runner, TContext, Tool
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool_context import ToolContext
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_final_output_message,
|
||||
get_function_tool,
|
||||
@@ -75,7 +75,7 @@ class RunHooksForTests(RunHooks):
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_agent_hooks():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(name="test_2", model=model)
|
||||
agent_3 = Agent(
|
||||
@@ -87,12 +87,12 @@ async def test_non_streamed_agent_hooks():
|
||||
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_text_message("user_message")])
|
||||
model.enqueue([get_text_message("user_message")])
|
||||
output = await Runner.run(agent_3, input="user_message", hooks=hooks)
|
||||
assert hooks.events == {"on_agent_start": 1, "on_agent_end": 1}, f"{output}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
[get_text_message("done")],
|
||||
@@ -103,7 +103,7 @@ async def test_non_streamed_agent_hooks():
|
||||
assert len(set(hooks.tool_context_ids)) == 1
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -127,7 +127,7 @@ async def test_non_streamed_agent_hooks():
|
||||
}, f"got unexpected event count: {hooks.events}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
@@ -161,7 +161,7 @@ async def test_non_streamed_agent_hooks():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_agent_hooks():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(name="test_2", model=model)
|
||||
agent_3 = Agent(
|
||||
@@ -173,14 +173,14 @@ async def test_streamed_agent_hooks():
|
||||
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_text_message("user_message")])
|
||||
model.enqueue([get_text_message("user_message")])
|
||||
output = Runner.run_streamed(agent_3, input="user_message", hooks=hooks)
|
||||
async for _ in output.stream_events():
|
||||
pass
|
||||
assert hooks.events == {"on_agent_start": 1, "on_agent_end": 1}, f"{output}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -204,7 +204,7 @@ async def test_streamed_agent_hooks():
|
||||
}, f"got unexpected event count: {hooks.events}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
@@ -243,7 +243,7 @@ class Foo(TypedDict):
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_non_streamed_agent_hooks():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(name="test_2", model=model)
|
||||
agent_3 = Agent(
|
||||
@@ -256,12 +256,12 @@ async def test_structured_output_non_streamed_agent_hooks():
|
||||
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
output = await Runner.run(agent_3, input="user_message", hooks=hooks)
|
||||
assert hooks.events == {"on_agent_start": 1, "on_agent_end": 1}, f"{output}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -284,7 +284,7 @@ async def test_structured_output_non_streamed_agent_hooks():
|
||||
}, f"got unexpected event count: {hooks.events}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
@@ -316,7 +316,7 @@ async def test_structured_output_non_streamed_agent_hooks():
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_streamed_agent_hooks():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(name="test_2", model=model)
|
||||
agent_3 = Agent(
|
||||
@@ -329,14 +329,14 @@ async def test_structured_output_streamed_agent_hooks():
|
||||
|
||||
agent_1.handoffs.append(agent_3)
|
||||
|
||||
model.set_next_output([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
model.enqueue([get_final_output_message(json.dumps({"a": "b"}))])
|
||||
output = Runner.run_streamed(agent_3, input="user_message", hooks=hooks)
|
||||
async for _ in output.stream_events():
|
||||
pass
|
||||
assert hooks.events == {"on_agent_start": 1, "on_agent_end": 1}, f"{output}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
@@ -360,7 +360,7 @@ async def test_structured_output_streamed_agent_hooks():
|
||||
}, f"got unexpected event count: {hooks.events}"
|
||||
hooks.reset()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a tool call
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")],
|
||||
|
||||
+111
-127
@@ -24,8 +24,8 @@ from agents import (
|
||||
from agents.guardrail import input_guardrail, output_guardrail
|
||||
from agents.result import RunResultStreaming
|
||||
from agents.run_internal.guardrails import run_input_guardrails, run_input_guardrails_with_queue
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_function_tool_call, get_text_message
|
||||
from .testing_processor import fetch_events
|
||||
|
||||
@@ -365,14 +365,14 @@ async def test_parallel_guardrail_runs_concurrently_with_agent():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
input_guardrails=[parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
result = await Runner.run(agent, "test input")
|
||||
|
||||
@@ -380,7 +380,7 @@ async def test_parallel_guardrail_runs_concurrently_with_agent():
|
||||
assert result.final_output is not None
|
||||
assert len(result.input_guardrail_results) == 1
|
||||
assert result.input_guardrail_results[0].output.output_info == "parallel_ok"
|
||||
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
|
||||
assert bool(model.calls), "Model should have been called in parallel mode"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -399,14 +399,14 @@ async def test_parallel_guardrail_runs_concurrently_with_agent_streaming():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="streaming_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
input_guardrails=[parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello from stream")])
|
||||
model.enqueue([get_text_message("hello from stream")])
|
||||
|
||||
result = Runner.run_streamed(agent, "test input")
|
||||
|
||||
@@ -416,7 +416,7 @@ async def test_parallel_guardrail_runs_concurrently_with_agent_streaming():
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert received_events is True
|
||||
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
|
||||
assert bool(model.calls), "Model should have been called in parallel mode"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -435,21 +435,21 @@ async def test_blocking_guardrail_prevents_agent_execution():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
input_guardrails=[blocking_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered) as exc_info:
|
||||
await Runner.run(agent, "test input")
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert exc_info.value.guardrail_result.output.output_info == "security_violation"
|
||||
assert model.first_turn_args is None, "Model should not have been called"
|
||||
assert not model.calls, "Model should not have been called"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -468,14 +468,14 @@ async def test_blocking_guardrail_prevents_agent_execution_streaming():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="streaming_agent",
|
||||
instructions="Reply with a long message",
|
||||
input_guardrails=[blocking_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
result = Runner.run_streamed(agent, "test input")
|
||||
|
||||
@@ -484,7 +484,7 @@ async def test_blocking_guardrail_prevents_agent_execution_streaming():
|
||||
pass
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert model.first_turn_args is None, "Model should not have been called"
|
||||
assert not model.calls, "Model should not have been called"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -516,7 +516,7 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="agent_with_tools",
|
||||
instructions="Call the fast_tool immediately",
|
||||
@@ -524,8 +524,8 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution():
|
||||
input_guardrails=[slow_parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_function_tool_call("fast_tool", arguments="{}")])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_function_tool_call("fast_tool", arguments="{}")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered):
|
||||
await Runner.run(agent, "trigger guardrail")
|
||||
@@ -534,7 +534,7 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution():
|
||||
assert tool_was_executed is True, (
|
||||
"Expected tool to execute before slow parallel guardrail triggered"
|
||||
)
|
||||
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
|
||||
assert bool(model.calls), "Model should have been called in parallel mode"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -553,7 +553,7 @@ async def test_parallel_guardrail_trip_cancels_model_task():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
original_get_response = model.get_response
|
||||
|
||||
async def slow_get_response(*args, **kwargs):
|
||||
@@ -573,7 +573,7 @@ async def test_parallel_guardrail_trip_cancels_model_task():
|
||||
input_guardrails=[tripwire_after_model_starts],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("should_not_finish")])
|
||||
model.enqueue([get_text_message("should_not_finish")])
|
||||
|
||||
with patch.object(model, "get_response", side_effect=slow_get_response):
|
||||
with pytest.raises(InputGuardrailTripwireTriggered):
|
||||
@@ -600,7 +600,7 @@ async def test_parallel_guardrail_trip_compat_mode_does_not_cancel_model_task():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
original_get_response = model.get_response
|
||||
|
||||
async def slow_get_response(*args, **kwargs):
|
||||
@@ -620,7 +620,7 @@ async def test_parallel_guardrail_trip_compat_mode_does_not_cancel_model_task():
|
||||
input_guardrails=[tripwire_after_model_starts],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("should_finish_without_cancel")])
|
||||
model.enqueue([get_text_message("should_finish_without_cancel")])
|
||||
|
||||
with patch.object(model, "get_response", side_effect=slow_get_response):
|
||||
with patch(
|
||||
@@ -663,7 +663,7 @@ async def test_model_error_cancels_parallel_input_guardrail_task():
|
||||
guardrail_cancelled.set()
|
||||
raise
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
async def boom_get_response(*args, **kwargs):
|
||||
# Only blow up once the guardrail is genuinely mid-flight.
|
||||
@@ -704,7 +704,7 @@ async def test_parallel_guardrail_non_tripwire_error_not_swallowed():
|
||||
await asyncio.wait_for(model_started.wait(), timeout=1)
|
||||
raise ValueError("guardrail boom")
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
original_get_response = model.get_response
|
||||
|
||||
async def slow_get_response(*args, **kwargs):
|
||||
@@ -724,7 +724,7 @@ async def test_parallel_guardrail_non_tripwire_error_not_swallowed():
|
||||
input_guardrails=[raising_parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("should_not_finish")])
|
||||
model.enqueue([get_text_message("should_not_finish")])
|
||||
|
||||
with patch.object(model, "get_response", side_effect=slow_get_response):
|
||||
with pytest.raises(ValueError, match="guardrail boom"):
|
||||
@@ -748,7 +748,7 @@ async def test_parallel_guardrail_error_cancels_streaming_model():
|
||||
await asyncio.wait_for(model_started.wait(), timeout=1)
|
||||
raise ValueError("guardrail boom")
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
async def blocking_stream_response(*args, **kwargs):
|
||||
model_started.set()
|
||||
@@ -797,7 +797,7 @@ async def test_model_error_before_guardrail_error_preserves_stream_finalization(
|
||||
await asyncio.wait_for(raise_guardrail_error.wait(), timeout=1)
|
||||
raise ValueError("guardrail boom")
|
||||
|
||||
class BlockingCleanupFakeModel(FakeModel):
|
||||
class BlockingCleanupScriptedModel(ScriptedModel):
|
||||
async def _cleanup_on_run_end(self, owner: object) -> None:
|
||||
model_cleanup_started.set()
|
||||
try:
|
||||
@@ -807,8 +807,8 @@ async def test_model_error_before_guardrail_error_preserves_stream_finalization(
|
||||
model_cleanup_cancelled.set()
|
||||
raise
|
||||
|
||||
model = BlockingCleanupFakeModel(tracing_enabled=True)
|
||||
model.set_next_output(RuntimeError("model boom"))
|
||||
model = BlockingCleanupScriptedModel(emit_traces=True)
|
||||
model.enqueue(RuntimeError("model boom"))
|
||||
|
||||
agent = Agent(
|
||||
name="streaming_model_error_agent",
|
||||
@@ -875,7 +875,7 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution_streaming():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="agent_with_tools",
|
||||
instructions="Call the fast_tool immediately",
|
||||
@@ -883,8 +883,8 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution_streaming():
|
||||
input_guardrails=[slow_parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_function_tool_call("fast_tool", arguments="{}")])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_function_tool_call("fast_tool", arguments="{}")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
|
||||
result = Runner.run_streamed(agent, "trigger guardrail")
|
||||
|
||||
@@ -896,7 +896,7 @@ async def test_parallel_guardrail_may_not_prevent_tool_execution_streaming():
|
||||
assert tool_was_executed is True, (
|
||||
"Expected tool to execute before slow parallel guardrail triggered"
|
||||
)
|
||||
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
|
||||
assert bool(model.calls), "Model should have been called in parallel mode"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -922,7 +922,7 @@ async def test_parallel_guardrail_trip_before_tool_execution_stops_streaming_tur
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
original_stream_response = model.stream_response
|
||||
|
||||
async def delayed_stream_response(*args, **kwargs):
|
||||
@@ -939,8 +939,8 @@ async def test_parallel_guardrail_trip_before_tool_execution_stops_streaming_tur
|
||||
input_guardrails=[tripwire_before_tool_execution],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_function_tool_call("dangerous_tool", arguments="{}")])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_function_tool_call("dangerous_tool", arguments="{}")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
|
||||
with patch.object(model, "stream_response", side_effect=delayed_stream_response):
|
||||
result = Runner.run_streamed(agent, "trigger guardrail")
|
||||
@@ -952,7 +952,7 @@ async def test_parallel_guardrail_trip_before_tool_execution_stops_streaming_tur
|
||||
assert model_started.is_set() is True
|
||||
assert guardrail_tripped.is_set() is True
|
||||
assert tool_was_executed is False
|
||||
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
|
||||
assert bool(model.calls), "Model should have been called in parallel mode"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -996,7 +996,7 @@ async def test_parallel_guardrail_trip_with_slow_cancel_sibling_stops_streaming_
|
||||
slow_cancel_finished.set()
|
||||
raise
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
original_stream_response = model.stream_response
|
||||
|
||||
async def delayed_stream_response(*args, **kwargs):
|
||||
@@ -1013,8 +1013,8 @@ async def test_parallel_guardrail_trip_with_slow_cancel_sibling_stops_streaming_
|
||||
input_guardrails=[tripwire_before_tool_execution, slow_to_cancel_guardrail],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_function_tool_call("dangerous_tool", arguments="{}")])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_function_tool_call("dangerous_tool", arguments="{}")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
|
||||
with patch.object(model, "stream_response", side_effect=delayed_stream_response):
|
||||
result = Runner.run_streamed(agent, "trigger guardrail")
|
||||
@@ -1033,7 +1033,7 @@ async def test_parallel_guardrail_trip_with_slow_cancel_sibling_stops_streaming_
|
||||
assert slow_cancel_started.is_set() is True
|
||||
assert slow_cancel_finished.is_set() is True
|
||||
assert tool_was_executed is False
|
||||
assert model.first_turn_args is not None, "Model should have been called in parallel mode"
|
||||
assert bool(model.calls), "Model should have been called in parallel mode"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1059,7 +1059,7 @@ async def test_blocking_guardrail_prevents_tool_execution():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="agent_with_tools",
|
||||
instructions="Call the dangerous_tool immediately",
|
||||
@@ -1067,14 +1067,14 @@ async def test_blocking_guardrail_prevents_tool_execution():
|
||||
input_guardrails=[security_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_function_tool_call("dangerous_tool", arguments="{}")])
|
||||
model.enqueue([get_function_tool_call("dangerous_tool", arguments="{}")])
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered):
|
||||
await Runner.run(agent, "trigger guardrail")
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert tool_was_executed is False
|
||||
assert model.first_turn_args is None, "Model should not have been called"
|
||||
assert not model.calls, "Model should not have been called"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1100,7 +1100,7 @@ async def test_blocking_guardrail_prevents_tool_execution_streaming():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="agent_with_tools",
|
||||
instructions="Call the dangerous_tool immediately",
|
||||
@@ -1108,7 +1108,7 @@ async def test_blocking_guardrail_prevents_tool_execution_streaming():
|
||||
input_guardrails=[security_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_function_tool_call("dangerous_tool", arguments="{}")])
|
||||
model.enqueue([get_function_tool_call("dangerous_tool", arguments="{}")])
|
||||
|
||||
result = Runner.run_streamed(agent, "trigger guardrail")
|
||||
|
||||
@@ -1118,7 +1118,7 @@ async def test_blocking_guardrail_prevents_tool_execution_streaming():
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert tool_was_executed is False
|
||||
assert model.first_turn_args is None, "Model should not have been called"
|
||||
assert not model.calls, "Model should not have been called"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1137,20 +1137,20 @@ async def test_parallel_guardrail_passes_agent_continues():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'success'",
|
||||
input_guardrails=[parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("success")])
|
||||
model.enqueue([get_text_message("success")])
|
||||
|
||||
result = await Runner.run(agent, "test input")
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert result.final_output is not None
|
||||
assert model.first_turn_args is not None, "Model should have been called"
|
||||
assert bool(model.calls), "Model should have been called"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1169,14 +1169,14 @@ async def test_parallel_guardrail_passes_agent_continues_streaming():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'success'",
|
||||
input_guardrails=[parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("success")])
|
||||
model.enqueue([get_text_message("success")])
|
||||
|
||||
result = Runner.run_streamed(agent, "test input")
|
||||
|
||||
@@ -1186,7 +1186,7 @@ async def test_parallel_guardrail_passes_agent_continues_streaming():
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert received_events is True
|
||||
assert model.first_turn_args is not None, "Model should have been called"
|
||||
assert bool(model.calls), "Model should have been called"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1205,20 +1205,20 @@ async def test_blocking_guardrail_passes_agent_continues():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'success'",
|
||||
input_guardrails=[blocking_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("success")])
|
||||
model.enqueue([get_text_message("success")])
|
||||
|
||||
result = await Runner.run(agent, "test input")
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert result.final_output is not None
|
||||
assert model.first_turn_args is not None, "Model should have been called after guardrail passed"
|
||||
assert bool(model.calls), "Model should have been called after guardrail passed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1237,14 +1237,14 @@ async def test_blocking_guardrail_passes_agent_continues_streaming():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'success'",
|
||||
input_guardrails=[blocking_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("success")])
|
||||
model.enqueue([get_text_message("success")])
|
||||
|
||||
result = Runner.run_streamed(agent, "test input")
|
||||
|
||||
@@ -1254,7 +1254,7 @@ async def test_blocking_guardrail_passes_agent_continues_streaming():
|
||||
|
||||
assert guardrail_executed is True
|
||||
assert received_events is True
|
||||
assert model.first_turn_args is not None, "Model should have been called after guardrail passed"
|
||||
assert bool(model.calls), "Model should have been called after guardrail passed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1289,7 +1289,7 @@ async def test_mixed_blocking_and_parallel_guardrails():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
original_get_response = model.get_response
|
||||
|
||||
@@ -1306,7 +1306,7 @@ async def test_mixed_blocking_and_parallel_guardrails():
|
||||
input_guardrails=[blocking_check, parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with patch.object(model, "get_response", side_effect=tracked_get_response):
|
||||
result = await Runner.run(agent, "test input")
|
||||
@@ -1324,9 +1324,7 @@ async def test_mixed_blocking_and_parallel_guardrails():
|
||||
"Model called while parallel guardrail still running"
|
||||
)
|
||||
assert parallel_finished.is_set() is True, "Parallel guardrail should have completed"
|
||||
assert model.first_turn_args is not None, (
|
||||
"Model should have been called after blocking guardrails passed"
|
||||
)
|
||||
assert bool(model.calls), "Model should have been called after blocking guardrails passed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1361,7 +1359,7 @@ async def test_mixed_blocking_and_parallel_guardrails_streaming():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
original_stream_response = model.stream_response
|
||||
|
||||
@@ -1379,7 +1377,7 @@ async def test_mixed_blocking_and_parallel_guardrails_streaming():
|
||||
input_guardrails=[blocking_check, parallel_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with patch.object(model, "stream_response", side_effect=tracked_stream_response):
|
||||
result = Runner.run_streamed(agent, "test input")
|
||||
@@ -1399,9 +1397,7 @@ async def test_mixed_blocking_and_parallel_guardrails_streaming():
|
||||
"Model called while parallel guardrail still running"
|
||||
)
|
||||
assert parallel_finished.is_set() is True, "Parallel guardrail should have completed"
|
||||
assert model.first_turn_args is not None, (
|
||||
"Model should have been called after blocking guardrails passed"
|
||||
)
|
||||
assert bool(model.calls), "Model should have been called after blocking guardrails passed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1432,7 +1428,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
original_get_response = model.get_response
|
||||
|
||||
@@ -1446,7 +1442,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent():
|
||||
input_guardrails=[first_blocking_check, second_blocking_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with patch.object(model, "get_response", side_effect=tracked_get_response):
|
||||
result = await Runner.run(agent, "test input")
|
||||
@@ -1466,9 +1462,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent():
|
||||
assert timestamps["second_blocking_end"] <= timestamps["model_called"], (
|
||||
"Second blocking guardrail must complete before model is called"
|
||||
)
|
||||
assert model.first_turn_args is not None, (
|
||||
"Model should have been called after all blocking guardrails passed"
|
||||
)
|
||||
assert bool(model.calls), "Model should have been called after all blocking guardrails passed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1499,7 +1493,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent_streaming():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
original_stream_response = model.stream_response
|
||||
|
||||
@@ -1514,7 +1508,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent_streaming():
|
||||
input_guardrails=[first_blocking_check, second_blocking_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with patch.object(model, "stream_response", side_effect=tracked_stream_response):
|
||||
result = Runner.run_streamed(agent, "test input")
|
||||
@@ -1536,9 +1530,7 @@ async def test_multiple_blocking_guardrails_complete_before_agent_streaming():
|
||||
assert timestamps["second_blocking_end"] <= timestamps["model_called"], (
|
||||
"Second blocking guardrail must complete before model is called"
|
||||
)
|
||||
assert model.first_turn_args is not None, (
|
||||
"Model should have been called after all blocking guardrails passed"
|
||||
)
|
||||
assert bool(model.calls), "Model should have been called after all blocking guardrails passed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1575,14 +1567,14 @@ async def test_multiple_blocking_guardrails_one_triggers():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="multi_blocking_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
input_guardrails=[first_blocking_check, second_blocking_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered):
|
||||
await Runner.run(agent, "test input")
|
||||
@@ -1593,9 +1585,7 @@ async def test_multiple_blocking_guardrails_one_triggers():
|
||||
assert "first_blocking_end" in timestamps
|
||||
assert "second_blocking_start" in timestamps
|
||||
assert "second_blocking_end" in timestamps
|
||||
assert model.first_turn_args is None, (
|
||||
"Model should not have been called when guardrail triggered"
|
||||
)
|
||||
assert not model.calls, "Model should not have been called when guardrail triggered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1632,14 +1622,14 @@ async def test_multiple_blocking_guardrails_one_triggers_streaming():
|
||||
tripwire_triggered=True,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="multi_blocking_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
input_guardrails=[first_blocking_check, second_blocking_check],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
result = Runner.run_streamed(agent, "test input")
|
||||
|
||||
@@ -1653,9 +1643,7 @@ async def test_multiple_blocking_guardrails_one_triggers_streaming():
|
||||
assert "first_blocking_end" in timestamps
|
||||
assert "second_blocking_start" in timestamps
|
||||
assert "second_blocking_end" in timestamps
|
||||
assert model.first_turn_args is None, (
|
||||
"Model should not have been called when guardrail triggered"
|
||||
)
|
||||
assert not model.calls, "Model should not have been called when guardrail triggered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1685,22 +1673,22 @@ async def test_guardrail_via_agent_and_run_config_equivalent():
|
||||
tripwire_triggered=False,
|
||||
)
|
||||
|
||||
model1 = FakeModel()
|
||||
model1 = ScriptedModel()
|
||||
agent_with_guardrail = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
input_guardrails=[agent_level_check],
|
||||
model=model1,
|
||||
)
|
||||
model1.set_next_output([get_text_message("hello")])
|
||||
model1.enqueue([get_text_message("hello")])
|
||||
|
||||
model2 = FakeModel()
|
||||
model2 = ScriptedModel()
|
||||
agent_without_guardrail = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
model=model2,
|
||||
)
|
||||
model2.set_next_output([get_text_message("hello")])
|
||||
model2.enqueue([get_text_message("hello")])
|
||||
run_config = RunConfig(input_guardrails=[config_level_check])
|
||||
|
||||
result1 = await Runner.run(agent_with_guardrail, "test input")
|
||||
@@ -1714,8 +1702,8 @@ async def test_guardrail_via_agent_and_run_config_equivalent():
|
||||
assert result2.input_guardrail_results[0].output.output_info == "config_level_passed"
|
||||
assert result1.final_output is not None
|
||||
assert result2.final_output is not None
|
||||
assert model1.first_turn_args is not None
|
||||
assert model2.first_turn_args is not None
|
||||
assert bool(model1.calls)
|
||||
assert bool(model2.calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1760,14 +1748,14 @@ async def test_blocking_guardrail_cancels_remaining_on_trigger():
|
||||
slow_guardrail_cancelled = True
|
||||
raise
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
input_guardrails=[fast_guardrail_that_triggers, slow_guardrail_that_should_be_cancelled],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered):
|
||||
await asyncio.wait_for(Runner.run(agent, "test input"), timeout=5)
|
||||
@@ -1780,9 +1768,7 @@ async def test_blocking_guardrail_cancels_remaining_on_trigger():
|
||||
assert slow_guardrail_executed is False, "Slow guardrail should NOT have completed execution"
|
||||
|
||||
# Verify agent never started
|
||||
assert model.first_turn_args is None, (
|
||||
"Model should not have been called when guardrail triggered"
|
||||
)
|
||||
assert not model.calls, "Model should not have been called when guardrail triggered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1827,14 +1813,14 @@ async def test_blocking_guardrail_cancels_remaining_on_trigger_streaming():
|
||||
slow_guardrail_cancelled = True
|
||||
raise
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_agent",
|
||||
instructions="Reply with 'hello'",
|
||||
input_guardrails=[fast_guardrail_that_triggers, slow_guardrail_that_should_be_cancelled],
|
||||
model=model,
|
||||
)
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
result = Runner.run_streamed(agent, "test input")
|
||||
|
||||
@@ -1853,9 +1839,7 @@ async def test_blocking_guardrail_cancels_remaining_on_trigger_streaming():
|
||||
assert slow_guardrail_executed is False, "Slow guardrail should NOT have completed execution"
|
||||
|
||||
# Verify agent never started
|
||||
assert model.first_turn_args is None, (
|
||||
"Model should not have been called when guardrail triggered"
|
||||
)
|
||||
assert not model.calls, "Model should not have been called when guardrail triggered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1882,7 +1866,7 @@ async def test_streaming_input_guardrail_exception_awaits_cancelled_siblings():
|
||||
await slow_started.wait()
|
||||
raise RuntimeError("guardrail failed")
|
||||
|
||||
agent = Agent(name="test_agent", model=FakeModel())
|
||||
agent = Agent(name="test_agent", model=ScriptedModel())
|
||||
context = RunContextWrapper(context=None)
|
||||
streamed_result = RunResultStreaming(
|
||||
"test input",
|
||||
@@ -2021,7 +2005,7 @@ def _ordered_input_guardrails(
|
||||
]
|
||||
|
||||
|
||||
def _tripwire_agent(model: FakeModel, *, run_in_parallel: bool) -> Agent[Any]:
|
||||
def _tripwire_agent(model: ScriptedModel, *, run_in_parallel: bool) -> Agent[Any]:
|
||||
return Agent(
|
||||
name="guardrail_results_agent",
|
||||
model=model,
|
||||
@@ -2039,8 +2023,8 @@ def _result_names(results: list[Any]) -> list[str]:
|
||||
@pytest.mark.parametrize("run_in_parallel", [False, True])
|
||||
async def test_input_guardrail_tripwire_reports_results(run_in_parallel: bool):
|
||||
"""Runner.run() reports every completed guardrail result on the raised tripwire."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered) as exc_info:
|
||||
await Runner.run(_tripwire_agent(model, run_in_parallel=run_in_parallel), "test input")
|
||||
@@ -2055,8 +2039,8 @@ async def test_input_guardrail_tripwire_reports_results(run_in_parallel: bool):
|
||||
@pytest.mark.parametrize("run_in_parallel", [False, True])
|
||||
async def test_input_guardrail_tripwire_reports_results_streamed(run_in_parallel: bool):
|
||||
"""The streamed path reports the same results, including on the streamed result object."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
result = Runner.run_streamed(
|
||||
_tripwire_agent(model, run_in_parallel=run_in_parallel), "test input"
|
||||
@@ -2073,8 +2057,8 @@ async def test_input_guardrail_tripwire_reports_results_streamed(run_in_parallel
|
||||
|
||||
def test_input_guardrail_tripwire_reports_results_sync():
|
||||
"""Runner.run_sync() matches the async entry points."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered) as exc_info:
|
||||
Runner.run_sync(_tripwire_agent(model, run_in_parallel=False), "test input")
|
||||
@@ -2087,8 +2071,8 @@ def test_input_guardrail_tripwire_reports_results_sync():
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_guardrail_results_reported_on_success():
|
||||
"""Passing guardrails still land on the successful result exactly once."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
agent = Agent(
|
||||
name="guardrail_results_agent",
|
||||
model=model,
|
||||
@@ -2134,8 +2118,8 @@ async def test_input_guardrail_exception_reports_completed_results_streamed(
|
||||
run_in_parallel: bool,
|
||||
):
|
||||
"""A streamed guardrail raising a non-tripwire error still reports earlier results."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
agent = Agent(
|
||||
name="guardrail_results_agent",
|
||||
model=model,
|
||||
@@ -2180,7 +2164,7 @@ def _ordered_output_guardrails(
|
||||
]
|
||||
|
||||
|
||||
def _output_tripwire_agent(model: FakeModel) -> Agent[Any]:
|
||||
def _output_tripwire_agent(model: ScriptedModel) -> Agent[Any]:
|
||||
return Agent(
|
||||
name="output_guardrail_results_agent",
|
||||
model=model,
|
||||
@@ -2191,8 +2175,8 @@ def _output_tripwire_agent(model: FakeModel) -> Agent[Any]:
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_tripwire_reports_results():
|
||||
"""Runner.run() reports every completed output guardrail result on the raised tripwire."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with pytest.raises(OutputGuardrailTripwireTriggered) as exc_info:
|
||||
await Runner.run(_output_tripwire_agent(model), "test input")
|
||||
@@ -2206,8 +2190,8 @@ async def test_output_guardrail_tripwire_reports_results():
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_tripwire_reports_results_streamed():
|
||||
"""The streamed path reports the same results, including on the streamed result object."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
result = Runner.run_streamed(_output_tripwire_agent(model), "test input")
|
||||
with pytest.raises(OutputGuardrailTripwireTriggered) as exc_info:
|
||||
@@ -2222,8 +2206,8 @@ async def test_output_guardrail_tripwire_reports_results_streamed():
|
||||
|
||||
def test_output_guardrail_tripwire_reports_results_sync():
|
||||
"""Runner.run_sync() matches the async entry points."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
|
||||
with pytest.raises(OutputGuardrailTripwireTriggered) as exc_info:
|
||||
Runner.run_sync(_output_tripwire_agent(model), "test input")
|
||||
@@ -2236,8 +2220,8 @@ def test_output_guardrail_tripwire_reports_results_sync():
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_results_reported_on_success():
|
||||
"""Passing output guardrails still land on the successful result exactly once."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
agent = Agent(
|
||||
name="output_guardrail_results_agent",
|
||||
model=model,
|
||||
@@ -2270,8 +2254,8 @@ async def test_output_guardrail_exception_reports_completed_results():
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_exception_reports_completed_results_streamed():
|
||||
"""A streamed output guardrail raising a non-tripwire error still reports earlier results."""
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("hello")])
|
||||
agent = Agent(
|
||||
name="output_guardrail_results_agent",
|
||||
model=model,
|
||||
|
||||
@@ -63,8 +63,9 @@ from agents.run_internal.items import (
|
||||
from agents.run_internal.session_persistence import (
|
||||
resolve_nested_history_owned_session_item_refs,
|
||||
)
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .model_test_helpers import get_exact_output_stream_step
|
||||
from .test_responses import get_function_tool_call, get_handoff_tool_call, get_text_message
|
||||
from .utils.simple_session import SimpleListSession
|
||||
|
||||
@@ -528,16 +529,14 @@ class TestHandoffHistoryDuplicationFix:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_to_input_list_normalized_uses_filtered_continuation_after_nested_handoff() -> None:
|
||||
triage_model = FakeModel()
|
||||
delegate_model = FakeModel()
|
||||
triage_model = ScriptedModel()
|
||||
delegate_model = ScriptedModel()
|
||||
|
||||
delegate = Agent(name="delegate", model=delegate_model)
|
||||
triage = Agent(name="triage", model=triage_model, handoffs=[delegate])
|
||||
|
||||
triage_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
|
||||
)
|
||||
delegate_model.add_multiple_turn_outputs(
|
||||
triage_model.extend([[get_text_message("triage summary"), get_handoff_tool_call(delegate)]])
|
||||
delegate_model.extend(
|
||||
[
|
||||
[get_text_message("resolution")],
|
||||
[get_text_message("followup answer")],
|
||||
@@ -567,13 +566,13 @@ async def test_to_input_list_normalized_uses_filtered_continuation_after_nested_
|
||||
assert "function_call" not in normalized_types
|
||||
assert "function_call_output" not in normalized_types
|
||||
|
||||
replay_model = FakeModel()
|
||||
replay_model = ScriptedModel()
|
||||
replay_agent = Agent(name="replay", model=replay_model)
|
||||
replay_model.add_multiple_turn_outputs([[get_text_message("replayed")]])
|
||||
replay_model.extend([[get_text_message("replayed")]])
|
||||
replay_result = await Runner.run(replay_agent, input=preserve_all_input)
|
||||
|
||||
assert replay_model.first_turn_args is not None
|
||||
replay_input = replay_model.first_turn_args["input"]
|
||||
assert bool(replay_model.calls)
|
||||
replay_input = replay_model.calls[0].input
|
||||
assert isinstance(replay_input, list)
|
||||
assert sum(_input_item_text(item) == "triage summary" for item in replay_input) == 1
|
||||
assert replay_result.final_output == "replayed"
|
||||
@@ -582,7 +581,7 @@ async def test_to_input_list_normalized_uses_filtered_continuation_after_nested_
|
||||
follow_up_result = await Runner.run(delegate, input=follow_up_input)
|
||||
|
||||
assert follow_up_result.final_output == "followup answer"
|
||||
assert delegate_model.last_turn_args["input"] == follow_up_input
|
||||
assert delegate_model.calls[-1].input == follow_up_input
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -590,8 +589,8 @@ async def test_to_input_list_normalized_keeps_delegate_tool_items_after_nested_h
|
||||
async def lookup_weather(city: str) -> str:
|
||||
return f"weather:{city}"
|
||||
|
||||
triage_model = FakeModel()
|
||||
delegate_model = FakeModel()
|
||||
triage_model = ScriptedModel()
|
||||
delegate_model = ScriptedModel()
|
||||
|
||||
delegate = Agent(
|
||||
name="delegate",
|
||||
@@ -600,10 +599,8 @@ async def test_to_input_list_normalized_keeps_delegate_tool_items_after_nested_h
|
||||
)
|
||||
triage = Agent(name="triage", model=triage_model, handoffs=[delegate])
|
||||
|
||||
triage_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
|
||||
)
|
||||
delegate_model.add_multiple_turn_outputs(
|
||||
triage_model.extend([[get_text_message("triage summary"), get_handoff_tool_call(delegate)]])
|
||||
delegate_model.extend(
|
||||
[
|
||||
[
|
||||
get_text_message("delegate preamble"),
|
||||
@@ -659,8 +656,8 @@ async def test_to_input_list_normalized_uses_custom_filter_input_items() -> None
|
||||
)
|
||||
)
|
||||
|
||||
triage_model = FakeModel()
|
||||
delegate_model = FakeModel()
|
||||
triage_model = ScriptedModel()
|
||||
delegate_model = ScriptedModel()
|
||||
|
||||
delegate = Agent(name="delegate", model=delegate_model)
|
||||
triage = Agent(
|
||||
@@ -669,10 +666,8 @@ async def test_to_input_list_normalized_uses_custom_filter_input_items() -> None
|
||||
handoffs=[handoff(delegate, input_filter=keep_messages_only)],
|
||||
)
|
||||
|
||||
triage_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
|
||||
)
|
||||
delegate_model.add_multiple_turn_outputs([[get_text_message("resolution")]])
|
||||
triage_model.extend([[get_text_message("triage summary"), get_handoff_tool_call(delegate)]])
|
||||
delegate_model.extend([[get_text_message("resolution")]])
|
||||
|
||||
result = await Runner.run(triage, input="user_question")
|
||||
preserve_all_input = result.to_input_list()
|
||||
@@ -702,18 +697,16 @@ async def test_non_nested_filtered_handoff_does_not_add_occurrence_lineage(
|
||||
def identity_filter(data: HandoffInputData) -> HandoffInputData:
|
||||
return data
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=identity_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
if streamed:
|
||||
streamed_result = Runner.run_streamed(first_agent, input="start")
|
||||
@@ -745,18 +738,16 @@ async def test_custom_filter_summary_shape_does_not_claim_equal_session_item() -
|
||||
input_items=(),
|
||||
)
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=custom_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(first_agent, input="start")
|
||||
|
||||
@@ -766,14 +757,12 @@ async def test_custom_filter_summary_shape_does_not_claim_equal_session_item() -
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapper_reset_does_not_change_nested_history_ownership() -> None:
|
||||
"""Replay ownership must not depend on wrappers that are current after the run."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
set_conversation_history_wrappers(start="<<START>>", end="<<END>>")
|
||||
try:
|
||||
@@ -797,18 +786,16 @@ async def test_public_nested_history_filter_preserves_ownership(chained: bool) -
|
||||
return remove_all_tools(nest_handoff_history(data))
|
||||
|
||||
input_filter = chained_filter if chained else nest_handoff_history
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=input_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(first_agent, input="start")
|
||||
|
||||
@@ -824,18 +811,16 @@ async def test_public_nested_history_filter_preserves_ownership_after_deepcopy()
|
||||
assert not isinstance(nested.input_history, str)
|
||||
return nested.clone(input_history=deepcopy(nested.input_history))
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=copied_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(first_agent, input="start")
|
||||
|
||||
@@ -856,18 +841,16 @@ async def test_public_nested_history_filter_preserves_ownership_after_dict_rebui
|
||||
)
|
||||
)
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=rebuilt_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(first_agent, input="start")
|
||||
|
||||
@@ -891,18 +874,16 @@ async def test_public_nested_history_filter_rebases_owned_input_after_insertion(
|
||||
)
|
||||
return nested.clone(input_history=(inserted, *rebuilt_history))
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=inserting_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
if streamed:
|
||||
streamed_result = Runner.run_streamed(first_agent, input="start")
|
||||
@@ -930,18 +911,16 @@ async def test_public_nested_history_filter_data_rebuild_drops_private_ownership
|
||||
input_items=nested.input_items,
|
||||
)
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=rebuilt_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(first_agent, input="start")
|
||||
|
||||
@@ -956,18 +935,16 @@ async def test_public_nested_history_filter_preserves_ownership_after_new_items_
|
||||
nested = nest_handoff_history(data)
|
||||
return nested.clone(new_items=deepcopy(nested.new_items))
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=copied_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(first_agent, input="start")
|
||||
|
||||
@@ -987,18 +964,16 @@ async def test_public_nested_history_filter_does_not_own_equal_replacement() ->
|
||||
)
|
||||
return nested.clone(new_items=(replacement, *nested.new_items[1:]))
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=replacement_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(first_agent, input="start")
|
||||
|
||||
@@ -1060,18 +1035,16 @@ async def test_nested_history_preserves_repeated_run_item_reference_occurrences(
|
||||
message = data.new_items[0]
|
||||
return nest_handoff_history(data.clone(new_items=(message, message, *data.new_items[1:])))
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=duplicate_message_filter)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
if streamed:
|
||||
streamed_result = Runner.run_streamed(first_agent, input="start")
|
||||
@@ -1091,8 +1064,8 @@ async def test_nested_history_retains_forwarded_pre_handoff_item_provenance(
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
"""Lossless items from earlier turns must retain one replay occurrence."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
tool_search_call = ResponseToolSearchCall(
|
||||
@@ -1111,13 +1084,14 @@ async def test_nested_history_retains_forwarded_pre_handoff_item_provenance(
|
||||
tools=[],
|
||||
type="tool_search_output",
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
first_output = [tool_search_call, tool_search_output]
|
||||
first_model.extend(
|
||||
[
|
||||
[tool_search_call, tool_search_output],
|
||||
get_exact_output_stream_step(first_output) if streamed else first_output,
|
||||
[get_handoff_tool_call(second_agent)],
|
||||
]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
run_result: RunResult | RunResultStreaming
|
||||
|
||||
@@ -1156,14 +1130,12 @@ async def test_nested_history_retains_forwarded_pre_handoff_item_provenance(
|
||||
@pytest.mark.asyncio
|
||||
async def test_to_input_list_during_active_stream_does_not_mutate_input() -> None:
|
||||
"""Inspecting an active stream must not mutate its eventual public input."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = Runner.run_streamed(
|
||||
first_agent,
|
||||
@@ -1186,14 +1158,12 @@ async def test_to_input_list_during_active_stream_does_not_mutate_input() -> Non
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_history_ownership_remaps_after_new_items_insertion() -> None:
|
||||
"""A caller inserting a public new_items entry must not make ownership drop the new item."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(
|
||||
first_agent,
|
||||
@@ -1210,14 +1180,12 @@ async def test_nested_history_ownership_remaps_after_new_items_insertion() -> No
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_history_input_removal_does_not_claim_an_unmarked_equal_occurrence() -> None:
|
||||
"""An equal replacement input must not retain ownership of the removed occurrence."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(
|
||||
first_agent,
|
||||
@@ -1237,14 +1205,12 @@ async def test_nested_history_input_removal_does_not_claim_an_unmarked_equal_occ
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_history_new_item_removal_does_not_claim_an_equal_item() -> None:
|
||||
"""Removing the owned RunItem must not transfer ownership to an equal RunItem."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(
|
||||
first_agent,
|
||||
@@ -1272,14 +1238,12 @@ async def test_nested_history_new_item_removal_does_not_claim_an_equal_item() ->
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_history_ownership_survives_result_new_items_copy() -> None:
|
||||
"""Copying result run items must not replay a nested session occurrence twice."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(
|
||||
first_agent,
|
||||
@@ -1299,14 +1263,12 @@ async def test_nested_history_input_copy_does_not_infer_occurrence_ownership(
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
"""An unmarked public-input copy must remain distinct from its session occurrence."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result: RunResult | RunResultStreaming
|
||||
if streamed:
|
||||
@@ -1341,14 +1303,12 @@ async def test_nested_history_input_copy_and_reorder_does_not_infer_ownership(
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
"""Payload equality must not transfer ownership after a copied input reorder."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("owned once"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("owned once"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
|
||||
result: RunResult | RunResultStreaming
|
||||
@@ -1380,14 +1340,12 @@ async def test_result_input_mutation_does_not_change_state_snapshot_ownership(
|
||||
def approval_tool() -> str:
|
||||
return "approved"
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model, tools=[approval_tool])
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("owned once"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs(
|
||||
first_model.extend([[get_text_message("owned once"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="approval")],
|
||||
[get_text_message("done")],
|
||||
@@ -1438,14 +1396,12 @@ async def test_result_input_mutation_does_not_change_state_snapshot_ownership(
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_history_ownership_revalidates_after_input_removal() -> None:
|
||||
"""Removing an owned input occurrence must restore its session copy during replay."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
result = await Runner.run(
|
||||
first_agent,
|
||||
@@ -1566,9 +1522,9 @@ def test_nested_history_normalizes_forwarded_status_before_ownership() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_handoff_preserves_prior_nested_history_ownership(streamed: bool) -> None:
|
||||
"""A later non-nesting handoff must not clear ownership established by an earlier handoff."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
final_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
final_model = ScriptedModel()
|
||||
final_agent = Agent(name="final", model=final_model)
|
||||
second_agent = Agent(
|
||||
name="second",
|
||||
@@ -1576,13 +1532,9 @@ async def test_plain_handoff_preserves_prior_nested_history_ownership(streamed:
|
||||
handoffs=[handoff(final_agent, nest_handoff_history=False)],
|
||||
)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("first message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("second message"), get_handoff_tool_call(final_agent)]]
|
||||
)
|
||||
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("first message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("second message"), get_handoff_tool_call(final_agent)]])
|
||||
final_model.extend([[get_text_message("done")]])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
|
||||
if streamed:
|
||||
@@ -1609,9 +1561,9 @@ async def test_non_nested_copying_filter_preserves_prior_nested_history_ownershi
|
||||
return data
|
||||
return data.clone(input_history=deepcopy(data.input_history))
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
final_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
final_model = ScriptedModel()
|
||||
final_agent = Agent(name="final", model=final_model)
|
||||
second_agent = Agent(
|
||||
name="second",
|
||||
@@ -1625,13 +1577,9 @@ async def test_non_nested_copying_filter_preserves_prior_nested_history_ownershi
|
||||
],
|
||||
)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("first message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("second message"), get_handoff_tool_call(final_agent)]]
|
||||
)
|
||||
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("first message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("second message"), get_handoff_tool_call(final_agent)]])
|
||||
final_model.extend([[get_text_message("done")]])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
|
||||
if streamed:
|
||||
@@ -1660,9 +1608,9 @@ async def test_copied_custom_input_items_keep_session_occurrence_for_later_nesti
|
||||
def copy_model_items(data: HandoffInputData) -> HandoffInputData:
|
||||
return data.clone(input_items=deepcopy(data.new_items))
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
final_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
final_model = ScriptedModel()
|
||||
final_agent = Agent(name="final", model=final_model)
|
||||
second_agent = Agent(name="second", model=second_model, handoffs=[final_agent])
|
||||
first_agent = Agent(
|
||||
@@ -1670,11 +1618,9 @@ async def test_copied_custom_input_items_keep_session_occurrence_for_later_nesti
|
||||
model=first_model,
|
||||
handoffs=[handoff(second_agent, input_filter=copy_model_items)],
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("copied once"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_handoff_tool_call(final_agent)]])
|
||||
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("copied once"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_handoff_tool_call(final_agent)]])
|
||||
final_model.extend([[get_text_message("done")]])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
|
||||
if streamed:
|
||||
@@ -1697,9 +1643,9 @@ async def test_identity_filter_preserves_prior_nested_history_ownership(streamed
|
||||
def identity_filter(data: HandoffInputData) -> HandoffInputData:
|
||||
return data
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
final_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
final_model = ScriptedModel()
|
||||
final_agent = Agent(name="final", model=final_model)
|
||||
second_agent = Agent(
|
||||
name="second",
|
||||
@@ -1707,13 +1653,9 @@ async def test_identity_filter_preserves_prior_nested_history_ownership(streamed
|
||||
handoffs=[handoff(final_agent, input_filter=identity_filter)],
|
||||
)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("first message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("second message"), get_handoff_tool_call(final_agent)]]
|
||||
)
|
||||
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("first message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("second message"), get_handoff_tool_call(final_agent)]])
|
||||
final_model.extend([[get_text_message("done")]])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
|
||||
if streamed:
|
||||
@@ -1739,9 +1681,9 @@ async def test_nested_handoff_history_preserves_identical_messages_across_turns(
|
||||
def continue_work() -> str:
|
||||
return "continue"
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
final_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
final_model = ScriptedModel()
|
||||
final_agent = Agent(name="final", model=final_model)
|
||||
second_agent = Agent(
|
||||
name="second",
|
||||
@@ -1751,16 +1693,14 @@ async def test_nested_handoff_history_preserves_identical_messages_across_turns(
|
||||
)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("same"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs(
|
||||
first_model.extend([[get_text_message("same"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend(
|
||||
[
|
||||
[get_text_message("same"), get_function_tool_call("continue_work", "{}")],
|
||||
[get_text_message("same"), get_handoff_tool_call(final_agent)],
|
||||
]
|
||||
)
|
||||
final_model.add_multiple_turn_outputs([[get_text_message("same")]])
|
||||
final_model.extend([[get_text_message("same")]])
|
||||
|
||||
if streamed:
|
||||
streamed_result = Runner.run_streamed(
|
||||
@@ -1779,7 +1719,7 @@ async def test_nested_handoff_history_preserves_identical_messages_across_turns(
|
||||
)
|
||||
replay_input = result.to_input_list()
|
||||
|
||||
final_input = final_model.last_turn_args["input"]
|
||||
final_input = final_model.calls[-1].input
|
||||
summary = str(cast(dict[str, Any], final_input[0])["content"])
|
||||
assert summary.count("same") == 2
|
||||
assert sum(_input_item_text(item) == "same" for item in replay_input) == 4
|
||||
@@ -1788,15 +1728,13 @@ async def test_nested_handoff_history_preserves_identical_messages_across_turns(
|
||||
@pytest.mark.asyncio
|
||||
async def test_explicit_default_handoff_history_mapper_is_honored() -> None:
|
||||
"""An explicitly configured mapper should own the exact model input."""
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
first_model.extend([[get_text_message("handoff message"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
|
||||
await Runner.run(
|
||||
first_agent,
|
||||
@@ -1807,8 +1745,8 @@ async def test_explicit_default_handoff_history_mapper_is_honored() -> None:
|
||||
),
|
||||
)
|
||||
|
||||
assert second_model.first_turn_args is not None
|
||||
second_input = second_model.first_turn_args["input"]
|
||||
assert bool(second_model.calls)
|
||||
second_input = second_model.calls[0].input
|
||||
assert isinstance(second_input, list)
|
||||
assert len(second_input) == 1
|
||||
summary = str(cast(dict[str, Any], second_input[0])["content"])
|
||||
@@ -1824,9 +1762,9 @@ async def test_nested_handoff_history_partition_survives_interruption_resume() -
|
||||
def approval_tool() -> str:
|
||||
return "approved"
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
final_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
final_model = ScriptedModel()
|
||||
final_agent = Agent(name="final", model=final_model)
|
||||
second_agent = Agent(
|
||||
name="second",
|
||||
@@ -1837,16 +1775,14 @@ async def test_nested_handoff_history_partition_survives_interruption_resume() -
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[get_text_message("once"), get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs(
|
||||
first_model.extend([[get_text_message("once"), get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="approval")],
|
||||
[get_text_message("once"), get_handoff_tool_call(final_agent)],
|
||||
]
|
||||
)
|
||||
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
final_model.extend([[get_text_message("done")]])
|
||||
|
||||
interrupted = await Runner.run(first_agent, input="start", run_config=run_config)
|
||||
assert len(interrupted.interruptions) == 1
|
||||
@@ -1870,7 +1806,7 @@ async def test_nested_handoff_history_partition_survives_interruption_resume() -
|
||||
resumed = await Runner.run(first_agent, restored, run_config=run_config)
|
||||
|
||||
assert resumed.final_output == "done"
|
||||
final_input = final_model.last_turn_args["input"]
|
||||
final_input = final_model.calls[-1].input
|
||||
summary = str(cast(dict[str, Any], final_input[0])["content"])
|
||||
assert summary.count("once") == 1
|
||||
assert sum(_input_item_text(item) == "once" for item in resumed.to_input_list()) == 2
|
||||
@@ -1893,9 +1829,9 @@ async def test_pending_handoff_in_interrupted_turn_survives_run_state(
|
||||
def approval_tool() -> str:
|
||||
return "approved"
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
final_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
final_model = ScriptedModel()
|
||||
final_agent = Agent(name="final", model=final_model)
|
||||
second_agent = Agent(
|
||||
name="second",
|
||||
@@ -1910,8 +1846,8 @@ async def test_pending_handoff_in_interrupted_turn_survives_run_state(
|
||||
first_handoff.call_id = "first-handoff"
|
||||
final_handoff = cast(ResponseFunctionToolCall, get_handoff_tool_call(final_agent))
|
||||
final_handoff.call_id = "final-handoff"
|
||||
first_model.add_multiple_turn_outputs([[get_text_message("first once"), first_handoff]])
|
||||
second_model.add_multiple_turn_outputs(
|
||||
first_model.extend([[get_text_message("first once"), first_handoff]])
|
||||
second_model.extend(
|
||||
[
|
||||
[
|
||||
get_text_message("second once"),
|
||||
@@ -1920,7 +1856,7 @@ async def test_pending_handoff_in_interrupted_turn_survives_run_state(
|
||||
]
|
||||
]
|
||||
)
|
||||
final_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
final_model.extend([[get_text_message("done")]])
|
||||
|
||||
interrupted: RunResult | RunResultStreaming
|
||||
if streamed:
|
||||
@@ -1964,7 +1900,7 @@ async def test_pending_handoff_in_interrupted_turn_survives_run_state(
|
||||
== 1
|
||||
)
|
||||
if nest_handoff_history:
|
||||
final_input = final_model.last_turn_args["input"]
|
||||
final_input = final_model.calls[-1].input
|
||||
summary = str(cast(dict[str, Any], final_input[0])["content"])
|
||||
assert summary.count("first once") == 1
|
||||
assert summary.count("second once") == 1
|
||||
@@ -2008,8 +1944,8 @@ async def test_resumed_handoff_persists_all_staged_approval_outputs(
|
||||
nonlocal handoff_count
|
||||
handoff_count += 1
|
||||
|
||||
source_model = FakeModel()
|
||||
target_model = FakeModel()
|
||||
source_model = ScriptedModel()
|
||||
target_model = ScriptedModel()
|
||||
target_agent = Agent(name="target", model=target_model)
|
||||
source_agent = Agent(
|
||||
name="source",
|
||||
@@ -2030,8 +1966,8 @@ async def test_resumed_handoff_persists_all_staged_approval_outputs(
|
||||
second_call.id = "item-second"
|
||||
handoff_call.id = "item-handoff"
|
||||
handoff_call.call_id = "handoff"
|
||||
source_model.add_multiple_turn_outputs([[first_call, second_call, handoff_call]])
|
||||
target_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
source_model.extend([[first_call, second_call, handoff_call]])
|
||||
target_model.extend([[get_text_message("done")]])
|
||||
|
||||
run_config = RunConfig(nest_handoff_history=nest_handoff_history)
|
||||
hooks = RecordingHooks()
|
||||
@@ -2152,8 +2088,8 @@ async def test_nested_history_resume_to_final_preserves_status_less_ownership(
|
||||
def approval_tool() -> str:
|
||||
return "approved"
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model, tools=[approval_tool])
|
||||
first_agent = Agent(name="first", model=first_model, handoffs=[second_agent])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
@@ -2164,10 +2100,8 @@ async def test_nested_history_resume_to_final_preserves_status_less_ownership(
|
||||
status=None,
|
||||
type="message",
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
[[status_less_message, get_handoff_tool_call(second_agent)]]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs(
|
||||
first_model.extend([[status_less_message, get_handoff_tool_call(second_agent)]])
|
||||
second_model.extend(
|
||||
[
|
||||
[get_function_tool_call("approval_tool", "{}", call_id="approval")],
|
||||
[get_text_message("done")],
|
||||
@@ -2211,10 +2145,8 @@ async def test_nested_history_resume_to_final_preserves_status_less_ownership(
|
||||
assert final_output == "done"
|
||||
assert sum(_input_item_text(item) == "once" for item in replay_input) == 1
|
||||
assert all("_agents_nested_history_token" not in item for item in replay_input)
|
||||
assert second_model.last_turn_args is not None
|
||||
assert all(
|
||||
"_agents_nested_history_token" not in item for item in second_model.last_turn_args["input"]
|
||||
)
|
||||
assert bool(second_model.calls)
|
||||
assert all("_agents_nested_history_token" not in item for item in second_model.calls[-1].input)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
|
||||
@@ -2234,8 +2166,8 @@ async def test_first_nested_handoff_after_restore_uses_explicit_occurrence_linea
|
||||
def approval_tool() -> str:
|
||||
return "approved"
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
second_agent = Agent(name="second", model=second_model)
|
||||
first_agent = Agent(
|
||||
name="first",
|
||||
@@ -2259,17 +2191,18 @@ async def test_first_nested_handoff_after_restore_uses_explicit_occurrence_linea
|
||||
tools=[],
|
||||
type="tool_search_output",
|
||||
)
|
||||
first_model.add_multiple_turn_outputs(
|
||||
first_output = [
|
||||
tool_search_call,
|
||||
tool_search_output,
|
||||
get_function_tool_call("approval_tool", "{}", call_id="approval"),
|
||||
]
|
||||
first_model.extend(
|
||||
[
|
||||
[
|
||||
tool_search_call,
|
||||
tool_search_output,
|
||||
get_function_tool_call("approval_tool", "{}", call_id="approval"),
|
||||
],
|
||||
get_exact_output_stream_step(first_output) if streamed else first_output,
|
||||
[get_handoff_tool_call(second_agent)],
|
||||
]
|
||||
)
|
||||
second_model.add_multiple_turn_outputs([[get_text_message("done")]])
|
||||
second_model.extend([[get_text_message("done")]])
|
||||
run_config = RunConfig(nest_handoff_history=True)
|
||||
interrupted: RunResult | RunResultStreaming
|
||||
|
||||
|
||||
@@ -73,6 +73,7 @@ from agents.run_internal.tool_planning import (
|
||||
execute_mcp_approval_requests,
|
||||
)
|
||||
from agents.run_state import RunState as RunStateClass
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import FunctionTool, HostedMCPTool
|
||||
from agents.tool_guardrails import (
|
||||
ToolGuardrailFunctionOutput,
|
||||
@@ -83,7 +84,6 @@ from agents.tool_guardrails import (
|
||||
)
|
||||
from agents.usage import Usage
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .mcp.helpers import FakeMCPServer
|
||||
from .test_responses import get_text_message
|
||||
from .utils.hitl import (
|
||||
@@ -321,12 +321,12 @@ async def test_nested_agent_tool_resumes_after_rejection() -> None:
|
||||
async def inner_hitl_tool() -> str:
|
||||
return "ok"
|
||||
|
||||
inner_model = FakeModel()
|
||||
inner_model = ScriptedModel()
|
||||
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool])
|
||||
inner_call_first = make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")
|
||||
inner_call_retry = make_function_tool_call(inner_hitl_tool.name, call_id="inner-2")
|
||||
inner_final = get_text_message("done")
|
||||
inner_model.add_multiple_turn_outputs(
|
||||
inner_model.extend(
|
||||
[
|
||||
[inner_call_first],
|
||||
[inner_call_retry],
|
||||
@@ -340,12 +340,12 @@ async def test_nested_agent_tool_resumes_after_rejection() -> None:
|
||||
needs_approval=True,
|
||||
)
|
||||
|
||||
outer_model = FakeModel()
|
||||
outer_model = ScriptedModel()
|
||||
outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool])
|
||||
outer_call = make_function_tool_call(
|
||||
agent_tool.name, call_id="outer-1", arguments='{"input":"hi"}'
|
||||
)
|
||||
outer_model.add_multiple_turn_outputs([[outer_call]])
|
||||
outer_model.extend([[outer_call]])
|
||||
|
||||
first = await Runner.run(outer_agent, "start")
|
||||
assert first.interruptions, "agent tool should request approval first"
|
||||
@@ -391,23 +391,23 @@ async def test_changed_nested_parent_fails_before_tool_inventory_callbacks() ->
|
||||
async def observer() -> str:
|
||||
return "unused"
|
||||
|
||||
inner_model = FakeModel()
|
||||
inner_model.add_multiple_turn_outputs(
|
||||
[[make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")]]
|
||||
)
|
||||
inner_model = ScriptedModel()
|
||||
inner_model.extend([[make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")]])
|
||||
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool])
|
||||
agent_tool = inner_agent.as_tool(
|
||||
tool_name="inner_agent_tool",
|
||||
tool_description="Inner agent tool with HITL",
|
||||
needs_approval=True,
|
||||
)
|
||||
outer_model = FakeModel(
|
||||
initial_output=[
|
||||
make_function_tool_call(
|
||||
agent_tool.name,
|
||||
call_id="outer-1",
|
||||
arguments='{"input":"safe"}',
|
||||
)
|
||||
outer_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
make_function_tool_call(
|
||||
agent_tool.name,
|
||||
call_id="outer-1",
|
||||
arguments='{"input":"safe"}',
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool, observer])
|
||||
@@ -438,9 +438,9 @@ async def test_nested_agent_tool_interruptions_remain_distinct_across_outer_call
|
||||
async def inner_hitl_tool() -> str:
|
||||
return "ok"
|
||||
|
||||
inner_model = FakeModel()
|
||||
inner_model = ScriptedModel()
|
||||
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool])
|
||||
inner_model.add_multiple_turn_outputs(
|
||||
inner_model.extend(
|
||||
[
|
||||
[make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")],
|
||||
[make_function_tool_call(inner_hitl_tool.name, call_id="inner-2")],
|
||||
@@ -453,9 +453,9 @@ async def test_nested_agent_tool_interruptions_remain_distinct_across_outer_call
|
||||
needs_approval=False,
|
||||
)
|
||||
|
||||
outer_model = FakeModel()
|
||||
outer_model = ScriptedModel()
|
||||
outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool])
|
||||
outer_model.add_multiple_turn_outputs(
|
||||
outer_model.extend(
|
||||
[
|
||||
[
|
||||
make_function_tool_call(
|
||||
@@ -488,11 +488,9 @@ async def test_nested_agent_tool_does_not_inherit_parent_approvals() -> None:
|
||||
async def inner_shared_tool() -> str:
|
||||
return "inner"
|
||||
|
||||
inner_model = FakeModel()
|
||||
inner_model = ScriptedModel()
|
||||
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_shared_tool])
|
||||
inner_model.add_multiple_turn_outputs(
|
||||
[[make_function_tool_call(inner_shared_tool.name, call_id="dup")]]
|
||||
)
|
||||
inner_model.extend([[make_function_tool_call(inner_shared_tool.name, call_id="dup")]])
|
||||
|
||||
agent_tool = inner_agent.as_tool(
|
||||
tool_name="inner_agent_tool",
|
||||
@@ -500,9 +498,9 @@ async def test_nested_agent_tool_does_not_inherit_parent_approvals() -> None:
|
||||
needs_approval=False,
|
||||
)
|
||||
|
||||
outer_model = FakeModel()
|
||||
outer_model = ScriptedModel()
|
||||
outer_agent = Agent(name="Outer", model=outer_model, tools=[outer_shared_tool, agent_tool])
|
||||
outer_model.add_multiple_turn_outputs(
|
||||
outer_model.extend(
|
||||
[
|
||||
[make_function_tool_call(outer_shared_tool.name, call_id="dup")],
|
||||
[
|
||||
@@ -568,7 +566,7 @@ async def test_resume_does_not_duplicate_pending_shell_approvals() -> None:
|
||||
call_id = extract_tool_call_id(raw_call)
|
||||
assert call_id, "shell call must have a call_id"
|
||||
|
||||
model.set_next_output([raw_call])
|
||||
model.enqueue([raw_call])
|
||||
first = await Runner.run(agent, "run shell")
|
||||
assert first.interruptions, "shell tool should require approval"
|
||||
|
||||
@@ -626,7 +624,8 @@ async def test_route_local_shell_calls_to_remote_shell_tool():
|
||||
action={"type": "exec", "command": ["echo", "test"], "env": {}}, # type: ignore[arg-type]
|
||||
status="in_progress",
|
||||
)
|
||||
model.set_next_output([local_shell_call])
|
||||
model.enqueue([local_shell_call])
|
||||
model.enqueue([])
|
||||
|
||||
await Runner.run(agent, "run local shell")
|
||||
|
||||
@@ -654,7 +653,7 @@ async def test_preserve_max_turns_when_resuming_from_runresult_state():
|
||||
tool = function_tool(test_tool, needs_approval=require_approval)
|
||||
model, agent = make_model_and_agent(tools=[tool])
|
||||
|
||||
model.add_multiple_turn_outputs([[make_function_tool_call("test_tool", call_id="call-1")]])
|
||||
model.extend([[make_function_tool_call("test_tool", call_id="call-1")]])
|
||||
|
||||
result1 = await Runner.run(agent, "call test_tool", max_turns=20)
|
||||
assert result1.interruptions, "should have an interruption"
|
||||
@@ -662,7 +661,7 @@ async def test_preserve_max_turns_when_resuming_from_runresult_state():
|
||||
state = approve_first_interruption(result1, always_approve=True)
|
||||
|
||||
# Provide 10 more turns (turns 2-11) to ensure we exceed the default 10 but not 20.
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_text_message(f"turn {i + 2}"), # Text message first (doesn't finish)
|
||||
@@ -671,6 +670,7 @@ async def test_preserve_max_turns_when_resuming_from_runresult_state():
|
||||
for i in range(10)
|
||||
]
|
||||
)
|
||||
model.enqueue([])
|
||||
|
||||
result2 = await Runner.run(agent, state)
|
||||
assert result2 is not None, "Run should complete successfully with max_turns=20 from state"
|
||||
@@ -687,7 +687,7 @@ async def test_current_turn_not_preserved_in_to_state():
|
||||
model, agent = make_model_and_agent(tools=[tool])
|
||||
|
||||
# Model emits a tool call requiring approval
|
||||
model.set_next_output([make_function_tool_call("test_tool", call_id="call-1")])
|
||||
model.enqueue([make_function_tool_call("test_tool", call_id="call-1")])
|
||||
|
||||
# First turn with interruption
|
||||
result1 = await Runner.run(agent, "call test_tool")
|
||||
@@ -859,7 +859,7 @@ async def test_preserve_persisted_item_counter_when_resuming_streamed_runs():
|
||||
]
|
||||
|
||||
# Set up model to return final output immediately (so the run completes)
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
|
||||
result = Runner.run_streamed(agent, state)
|
||||
|
||||
@@ -910,7 +910,7 @@ async def test_function_needs_approval_invalid_type_raises() -> None:
|
||||
return "ok"
|
||||
|
||||
model, agent = make_model_and_agent(tools=[bad_tool])
|
||||
model.set_next_output([make_function_tool_call("bad_tool")])
|
||||
model.enqueue([make_function_tool_call("bad_tool")])
|
||||
|
||||
with pytest.raises(UserError, match="needs_approval"):
|
||||
await Runner.run(agent, "run invalid")
|
||||
@@ -951,9 +951,7 @@ async def test_callable_function_approval_fails_closed_for_invalid_arguments(
|
||||
needs_approval=needs_approval,
|
||||
)
|
||||
model, agent = make_model_and_agent(tools=[tool])
|
||||
model.set_next_output(
|
||||
[make_function_tool_call(tool.name, arguments=arguments, call_id="call-invalid")]
|
||||
)
|
||||
model.enqueue([make_function_tool_call(tool.name, arguments=arguments, call_id="call-invalid")])
|
||||
|
||||
result = await Runner.run(agent, "send an email")
|
||||
|
||||
@@ -986,7 +984,7 @@ async def test_callable_function_approval_receives_valid_object_arguments() -> N
|
||||
)
|
||||
arguments = '{"subject": "status update"}'
|
||||
model, agent = make_model_and_agent(tools=[tool])
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[make_function_tool_call(tool.name, arguments=arguments, call_id="call-valid")],
|
||||
[get_text_message("done")],
|
||||
@@ -1058,7 +1056,7 @@ async def test_agent_as_tool_with_nested_approvals_propagates() -> None:
|
||||
spanish_agent.tools = [get_current_timestamp]
|
||||
|
||||
# Spanish agent will first request timestamp, then return text.
|
||||
nested_model.add_multiple_turn_outputs(
|
||||
nested_model.extend(
|
||||
[
|
||||
[make_function_tool_call("get_current_timestamp")],
|
||||
[get_text_message("hola")],
|
||||
@@ -1066,7 +1064,7 @@ async def test_agent_as_tool_with_nested_approvals_propagates() -> None:
|
||||
)
|
||||
|
||||
# Orchestrator model will call the spanish agent tool.
|
||||
orchestrator_model = FakeModel()
|
||||
orchestrator_model = ScriptedModel()
|
||||
orchestrator = Agent(
|
||||
name="orchestrator",
|
||||
tools=[
|
||||
@@ -1079,7 +1077,7 @@ async def test_agent_as_tool_with_nested_approvals_propagates() -> None:
|
||||
model=orchestrator_model,
|
||||
)
|
||||
|
||||
orchestrator_model.add_multiple_turn_outputs(
|
||||
orchestrator_model.extend(
|
||||
[
|
||||
[
|
||||
make_function_tool_call(
|
||||
@@ -1132,7 +1130,7 @@ async def test_nested_agent_tool_continuation_runs_outer_callbacks_once(streamed
|
||||
return "inner output"
|
||||
|
||||
nested_agent.tools = [inner_tool]
|
||||
nested_model.add_multiple_turn_outputs(
|
||||
nested_model.extend(
|
||||
[
|
||||
[
|
||||
make_function_tool_call(
|
||||
@@ -1171,8 +1169,8 @@ async def test_nested_agent_tool_continuation_runs_outer_callbacks_once(streamed
|
||||
outer_tool.tool_output_guardrails = [track_output]
|
||||
outer_tool.custom_data_extractor = extract_custom_data
|
||||
|
||||
outer_model = FakeModel()
|
||||
outer_model.add_multiple_turn_outputs(
|
||||
outer_model = ScriptedModel()
|
||||
outer_model.extend(
|
||||
[
|
||||
[
|
||||
make_function_tool_call(
|
||||
@@ -1789,9 +1787,9 @@ async def test_execute_path_skips_needs_approval_checker_when_status_resolved()
|
||||
async def sensitive(value: str) -> str:
|
||||
return f"ran:{value}"
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="agent", model=model, tools=[sensitive])
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[make_function_tool_call(sensitive.name, call_id="call-1", arguments='{"value":"x"}')],
|
||||
[get_text_message("done")],
|
||||
@@ -1827,14 +1825,14 @@ async def test_resume_checkpoints_tool_output_before_tool_use_behavior_failure()
|
||||
def failing_behavior(_ctx: Any, _results: Any) -> Any:
|
||||
raise RuntimeError("tool use behavior failed")
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="agent",
|
||||
model=model,
|
||||
tools=[sensitive],
|
||||
tool_use_behavior=failing_behavior,
|
||||
)
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[make_function_tool_call(sensitive.name, call_id="call-1", arguments='{"value":"x"}')],
|
||||
[get_text_message("done")],
|
||||
|
||||
@@ -1,24 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
Model,
|
||||
ModelResponse,
|
||||
ModelSettings,
|
||||
OpenAIConversationsSession,
|
||||
Runner,
|
||||
Usage,
|
||||
function_tool,
|
||||
)
|
||||
from agents.items import TResponseInputItem, TResponseStreamEvent
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.testing import ModelCall, ModelStep, ScriptedModel, function_call
|
||||
from tests.test_responses import get_text_message
|
||||
from tests.utils.hitl import HITL_REJECTION_MSG
|
||||
from tests.utils.simple_session import SimpleListSession
|
||||
@@ -69,67 +64,32 @@ class ScenarioResult:
|
||||
items: list[TResponseInputItem]
|
||||
|
||||
|
||||
class ScenarioModel(Model):
|
||||
def __init__(self) -> None:
|
||||
self._counter = 0
|
||||
def make_scenario_model() -> ScriptedModel:
|
||||
call_counter = 0
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Any],
|
||||
output_schema: Any,
|
||||
handoffs: list[Any],
|
||||
tracing: Any,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: Any | None,
|
||||
) -> ModelResponse:
|
||||
if input_has_rejection(input):
|
||||
return ModelResponse(
|
||||
output=[get_text_message(HITL_REJECTION_MSG)],
|
||||
usage=Usage(),
|
||||
response_id="resp-test",
|
||||
)
|
||||
tool_choice = model_settings.tool_choice
|
||||
def respond(call: ModelCall):
|
||||
nonlocal call_counter
|
||||
if input_has_rejection(call.input):
|
||||
return [get_text_message(HITL_REJECTION_MSG)]
|
||||
tool_choice = call.model_settings.tool_choice
|
||||
tool_name = tool_choice if isinstance(tool_choice, str) else TOOL_ECHO
|
||||
self._counter += 1
|
||||
call_id = f"call_{self._counter}"
|
||||
query = extract_user_message(input)
|
||||
tool_call = ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
name=tool_name,
|
||||
call_id=call_id,
|
||||
arguments=json.dumps({"query": query}),
|
||||
)
|
||||
return ModelResponse(output=[tool_call], usage=Usage(), response_id="resp-test")
|
||||
call_counter += 1
|
||||
return [
|
||||
function_call(
|
||||
tool_name,
|
||||
{"query": extract_user_message(call.input)},
|
||||
call_id=f"call_{call_counter}",
|
||||
)
|
||||
]
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Any],
|
||||
output_schema: Any,
|
||||
handoffs: list[Any],
|
||||
tracing: Any,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: Any | None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
if False:
|
||||
yield cast(TResponseStreamEvent, {})
|
||||
raise RuntimeError("Streaming is not supported in this scenario.")
|
||||
return ScriptedModel([ModelStep.respond(respond) for _ in range(4)])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_session_hitl_scenario() -> None:
|
||||
execute_counts.clear()
|
||||
session = SimpleListSession(session_id="memory")
|
||||
model = ScenarioModel()
|
||||
model = make_scenario_model()
|
||||
|
||||
steps = [
|
||||
ScenarioStep(
|
||||
@@ -233,7 +193,7 @@ async def test_openai_conversations_session_hitl_scenario() -> None:
|
||||
rehydrated_session = OpenAIConversationsSession(
|
||||
conversation_id="conv_test", openai_client=typed_client
|
||||
)
|
||||
model = ScenarioModel()
|
||||
model = make_scenario_model()
|
||||
|
||||
steps = [
|
||||
ScenarioStep(
|
||||
@@ -280,7 +240,7 @@ async def test_openai_conversations_session_hitl_scenario() -> None:
|
||||
|
||||
async def run_scenario_step(
|
||||
session: Any,
|
||||
model: ScenarioModel,
|
||||
model: ScriptedModel,
|
||||
step: ScenarioStep,
|
||||
) -> ScenarioResult:
|
||||
agent = Agent(
|
||||
|
||||
@@ -27,8 +27,8 @@ from agents import (
|
||||
)
|
||||
from agents.items import TResponseInputItem, TResponseOutputItem
|
||||
from agents.stream_events import RunItemStreamEvent
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_function_tool_call, get_text_message
|
||||
from .utils.simple_session import SimpleListSession
|
||||
|
||||
@@ -62,7 +62,7 @@ def _message_texts(items: list[TResponseInputItem]) -> list[str]:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_raises_without_handler() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="Invalid JSON"):
|
||||
@@ -71,7 +71,7 @@ async def test_invalid_final_output_raises_without_handler() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_returns_validated_fallback() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
def handler(data: RunErrorHandlerInput[None]) -> FinalOutput:
|
||||
@@ -96,7 +96,7 @@ async def test_invalid_final_output_handler_returns_validated_fallback() -> None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_can_skip_fallback_history() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
result = await Runner.run(
|
||||
@@ -119,7 +119,7 @@ async def test_invalid_final_output_handler_rejects_invalid_fallback(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False)
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
with pytest.warns(UserWarning, match="Pydantic serializer warnings"):
|
||||
@@ -133,7 +133,7 @@ async def test_invalid_final_output_handler_rejects_invalid_fallback(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_can_decline_recovery() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="Invalid JSON"):
|
||||
@@ -146,7 +146,7 @@ async def test_invalid_final_output_handler_can_decline_recovery() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_does_not_catch_other_model_behavior_errors() -> None:
|
||||
model = FakeModel(initial_output=[get_function_tool_call("missing_tool")])
|
||||
model = ScriptedModel(steps=[[get_function_tool_call("missing_tool")]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
handler_called = False
|
||||
|
||||
@@ -170,8 +170,8 @@ async def test_invalid_final_output_handler_does_not_catch_other_model_behavior_
|
||||
async def test_empty_structured_output_handler_avoids_another_model_turn(
|
||||
invalid_output: list[TResponseOutputItem],
|
||||
) -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([invalid_output, [get_text_message('{"summary":"unused"}')]])
|
||||
model = ScriptedModel()
|
||||
model.extend([invalid_output, [get_text_message('{"summary":"unused"}')]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
def handler(data: RunErrorHandlerInput[None]) -> FinalOutput:
|
||||
@@ -188,7 +188,7 @@ async def test_empty_structured_output_handler_avoids_another_model_turn(
|
||||
)
|
||||
|
||||
assert result.final_output == FinalOutput(summary="safe fallback")
|
||||
assert len(model.turn_outputs) == 1
|
||||
assert model.remaining_steps == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -199,19 +199,19 @@ async def test_empty_structured_output_handler_avoids_another_model_turn(
|
||||
async def test_empty_structured_output_without_fallback_keeps_existing_next_turn_behavior(
|
||||
error_handlers: RunErrorHandlers[None] | None,
|
||||
) -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[], [get_text_message('{"summary":"second turn"}')]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[], [get_text_message('{"summary":"second turn"}')]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
result = await Runner.run(agent, input="user_message", error_handlers=error_handlers)
|
||||
|
||||
assert result.final_output == FinalOutput(summary="second turn")
|
||||
assert not model.turn_outputs
|
||||
assert model.remaining_steps == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_invalid_final_output_emits_exact_fallback_item() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
session = SimpleListSession()
|
||||
|
||||
@@ -246,8 +246,8 @@ async def test_streamed_invalid_final_output_emits_exact_fallback_item() -> None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_empty_structured_output_handler_avoids_another_model_turn() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[], [get_text_message('{"summary":"unused"}')]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[], [get_text_message('{"summary":"unused"}')]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
result = Runner.run_streamed(
|
||||
@@ -258,7 +258,7 @@ async def test_streamed_empty_structured_output_handler_avoids_another_model_tur
|
||||
events = [event async for event in result.stream_events()]
|
||||
|
||||
assert result.final_output == FinalOutput(summary="safe fallback")
|
||||
assert len(model.turn_outputs) == 1
|
||||
assert model.remaining_steps == 1
|
||||
assert any(
|
||||
isinstance(event, RunItemStreamEvent)
|
||||
and event.name == "message_output_created"
|
||||
@@ -270,7 +270,7 @@ async def test_streamed_empty_structured_output_handler_avoids_another_model_tur
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_fallback_runs_hooks_and_output_guardrails() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
model = ScriptedModel(steps=[[get_text_message("not valid json")]])
|
||||
hooks = RecordingRunHooks()
|
||||
guarded_outputs: list[Any] = []
|
||||
|
||||
@@ -315,8 +315,8 @@ async def test_invalid_final_output_fallback_does_not_retry_or_replay_tools(
|
||||
side_effects.append(value)
|
||||
return f"recorded:{value}"
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -373,4 +373,4 @@ async def test_invalid_final_output_fallback_does_not_retry_or_replay_tools(
|
||||
|
||||
assert final_output == FinalOutput(summary="safe fallback")
|
||||
assert side_effects == ["once"]
|
||||
assert len(model.turn_outputs) == 2
|
||||
assert model.remaining_steps == 2
|
||||
|
||||
@@ -28,8 +28,9 @@ from agents import (
|
||||
from agents.items import ToolCallOutputItem
|
||||
from agents.run_internal.run_loop import LocalShellAction, ToolRunLocalShellCall
|
||||
from agents.run_state import RunState
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.model_test_helpers import get_response_obj
|
||||
|
||||
from .fake_model import FakeModel, get_response_obj
|
||||
from .test_responses import get_text_message
|
||||
|
||||
|
||||
@@ -47,7 +48,7 @@ class RecordingLocalShellExecutor:
|
||||
|
||||
async def _create_serialized_local_shell_state() -> tuple[LocalShellTool, dict[str, Any]]:
|
||||
tool = LocalShellTool(executor=RecordingLocalShellExecutor(output="shell result"))
|
||||
initial_model = FakeModel()
|
||||
initial_model = ScriptedModel()
|
||||
initial_agent = Agent(name="shell-agent", model=initial_model, tools=[tool])
|
||||
local_shell_call = LocalShellCall(
|
||||
id="lsh_test",
|
||||
@@ -62,7 +63,7 @@ async def _create_serialized_local_shell_state() -> tuple[LocalShellTool, dict[s
|
||||
status="completed",
|
||||
type="local_shell_call",
|
||||
)
|
||||
initial_model.add_multiple_turn_outputs(
|
||||
initial_model.extend(
|
||||
[
|
||||
[get_text_message("running shell"), local_shell_call],
|
||||
[get_text_message("shell complete")],
|
||||
@@ -158,7 +159,7 @@ async def test_runner_executes_local_shell_calls() -> None:
|
||||
executor = RecordingLocalShellExecutor(output="shell result")
|
||||
tool = LocalShellTool(executor=executor)
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="shell-agent", model=model, tools=[tool])
|
||||
|
||||
action = LocalShellCallAction(
|
||||
@@ -176,7 +177,7 @@ async def test_runner_executes_local_shell_calls() -> None:
|
||||
type="local_shell_call",
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("running shell"), local_shell_call],
|
||||
[get_text_message("shell complete")],
|
||||
@@ -188,7 +189,8 @@ async def test_runner_executes_local_shell_calls() -> None:
|
||||
assert len(executor.calls) == 1
|
||||
request = executor.calls[0]
|
||||
assert isinstance(request, LocalShellCommandRequest)
|
||||
assert request.data is local_shell_call
|
||||
assert request.data == local_shell_call
|
||||
assert request.data is not local_shell_call
|
||||
|
||||
items = result.new_items
|
||||
assert len(items) == 4
|
||||
@@ -201,7 +203,8 @@ async def test_runner_executes_local_shell_calls() -> None:
|
||||
|
||||
tool_call_item = items[1]
|
||||
assert tool_call_item.type == "tool_call_item"
|
||||
assert tool_call_item.raw_item is local_shell_call
|
||||
assert tool_call_item.raw_item == local_shell_call
|
||||
assert tool_call_item.raw_item is not local_shell_call
|
||||
|
||||
local_shell_output = items[2]
|
||||
assert isinstance(local_shell_output, ToolCallOutputItem)
|
||||
@@ -334,8 +337,8 @@ async def test_run_state_preserves_official_local_shell_original_input(
|
||||
"id": "lsh_output_123",
|
||||
"output": "shell result",
|
||||
}
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[get_text_message("complete")]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[get_text_message("complete")]])
|
||||
agent = Agent(name="shell-agent", model=model)
|
||||
result = await Runner.run(agent, input=[original_input])
|
||||
serialized = json.loads(json.dumps(result.to_state().to_json()))
|
||||
|
||||
+30
-32
@@ -18,8 +18,8 @@ from agents import (
|
||||
UserError,
|
||||
)
|
||||
from agents.stream_events import RunItemStreamEvent
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_function_tool,
|
||||
get_function_tool_call,
|
||||
@@ -30,7 +30,7 @@ from .test_responses import (
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_max_turns():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -39,7 +39,7 @@ async def test_non_streamed_max_turns():
|
||||
|
||||
func_output = json.dumps({"a": "b"})
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("1"), get_function_tool_call("some_function", func_output, "1")],
|
||||
[get_text_message("2"), get_function_tool_call("some_function", func_output, "2")],
|
||||
@@ -54,7 +54,7 @@ async def test_non_streamed_max_turns():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_max_turns_none_disables_limit():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -63,7 +63,7 @@ async def test_non_streamed_max_turns_none_disables_limit():
|
||||
|
||||
func_output = json.dumps({"a": "b"})
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("1"), get_function_tool_call("some_function", func_output, "1")],
|
||||
[get_text_message("2"), get_function_tool_call("some_function", func_output, "2")],
|
||||
@@ -81,7 +81,7 @@ async def test_non_streamed_max_turns_none_disables_limit():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_max_turns():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -89,7 +89,7 @@ async def test_streamed_max_turns():
|
||||
)
|
||||
func_output = json.dumps({"a": "b"})
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_text_message("1"),
|
||||
@@ -121,7 +121,7 @@ async def test_streamed_max_turns():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_max_turns_none_disables_limit():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -129,7 +129,7 @@ async def test_streamed_max_turns_none_disables_limit():
|
||||
)
|
||||
func_output = json.dumps({"a": "b"})
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("1"), get_function_tool_call("some_function", func_output, "1")],
|
||||
[get_text_message("2"), get_function_tool_call("some_function", func_output, "2")],
|
||||
@@ -157,19 +157,19 @@ class FooModel(BaseModel):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_structured_output_refusal_raises_without_retry():
|
||||
model = FakeModel(initial_output=[get_refusal_message("I cannot help with that request.")])
|
||||
model = ScriptedModel(steps=[[get_refusal_message("I cannot help with that request.")]])
|
||||
agent = Agent(name="test_1", model=model, output_type=FooModel)
|
||||
|
||||
with pytest.raises(ModelRefusalError) as exc_info:
|
||||
await Runner.run(agent, input="user_message", max_turns=3)
|
||||
|
||||
assert exc_info.value.refusal == "I cannot help with that request."
|
||||
assert not model.turn_outputs
|
||||
assert model.remaining_steps == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_refusal_handler_returns_structured_output():
|
||||
model = FakeModel(initial_output=[get_refusal_message("I cannot help with that request.")])
|
||||
model = ScriptedModel(steps=[[get_refusal_message("I cannot help with that request.")]])
|
||||
agent = Agent(name="test_1", model=model, output_type=FooModel)
|
||||
|
||||
def handler(data):
|
||||
@@ -194,7 +194,7 @@ async def test_non_streamed_refusal_handler_returns_structured_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_refusal_handler_can_skip_history():
|
||||
model = FakeModel(initial_output=[get_refusal_message("I cannot help with that request.")])
|
||||
model = ScriptedModel(steps=[[get_refusal_message("I cannot help with that request.")]])
|
||||
agent = Agent(name="test_1", model=model)
|
||||
|
||||
result = await Runner.run(
|
||||
@@ -214,7 +214,7 @@ async def test_non_streamed_refusal_handler_can_skip_history():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_refusal_handler_returns_output():
|
||||
model = FakeModel(initial_output=[get_refusal_message("I cannot help with that request.")])
|
||||
model = ScriptedModel(steps=[[get_refusal_message("I cannot help with that request.")]])
|
||||
agent = Agent(name="test_1", model=model)
|
||||
|
||||
result = Runner.run_streamed(
|
||||
@@ -237,7 +237,7 @@ async def test_streamed_refusal_handler_returns_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_non_streamed_max_turns():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -245,7 +245,7 @@ async def test_structured_output_non_streamed_max_turns():
|
||||
tools=[get_function_tool("tool_1", "result")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool_1")],
|
||||
[get_function_tool_call("tool_1")],
|
||||
@@ -260,7 +260,7 @@ async def test_structured_output_non_streamed_max_turns():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_streamed_max_turns():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -268,7 +268,7 @@ async def test_structured_output_streamed_max_turns():
|
||||
tools=[get_function_tool("tool_1", "result")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool_1")],
|
||||
[get_function_tool_call("tool_1")],
|
||||
@@ -285,7 +285,7 @@ async def test_structured_output_streamed_max_turns():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_max_turns_handler_invalid_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -303,7 +303,7 @@ async def test_structured_output_max_turns_handler_invalid_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_max_turns_handler_pydantic_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -324,7 +324,7 @@ async def test_structured_output_max_turns_handler_pydantic_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_max_turns_handler_list_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -344,7 +344,7 @@ async def test_structured_output_max_turns_handler_list_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_max_turns_handler_returns_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test_1", model=model)
|
||||
|
||||
result = await Runner.run(
|
||||
@@ -364,7 +364,7 @@ async def test_non_streamed_max_turns_handler_returns_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_max_turns_handler_skip_history():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test_1", model=model)
|
||||
|
||||
result = await Runner.run(
|
||||
@@ -385,7 +385,7 @@ async def test_non_streamed_max_turns_handler_skip_history():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_max_turns_handler_raw_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test_1", model=model)
|
||||
|
||||
result = await Runner.run(
|
||||
@@ -401,7 +401,7 @@ async def test_non_streamed_max_turns_handler_raw_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_max_turns_handler_raw_dict_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test_1", model=model)
|
||||
|
||||
result = await Runner.run(
|
||||
@@ -416,7 +416,7 @@ async def test_non_streamed_max_turns_handler_raw_dict_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_max_turns_handler_returns_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test_1", model=model)
|
||||
|
||||
result = Runner.run_streamed(
|
||||
@@ -439,7 +439,7 @@ async def test_streamed_max_turns_handler_returns_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_max_turns_handler_pydantic_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -466,7 +466,7 @@ async def test_streamed_max_turns_handler_pydantic_output():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_max_turns_handler_list_output():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
@@ -492,15 +492,13 @@ async def test_streamed_max_turns_handler_list_output():
|
||||
|
||||
async def _run_max_turns_handler_with_session(streamed: bool) -> list[str]:
|
||||
"""Run one tool turn, trip max turns, and return the session's persisted item types."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="test_1",
|
||||
model=model,
|
||||
tools=[get_function_tool("some_function", "result")],
|
||||
)
|
||||
model.add_multiple_turn_outputs(
|
||||
[[get_function_tool_call("some_function", json.dumps({"a": "b"}))]]
|
||||
)
|
||||
model.extend([[get_function_tool_call("some_function", json.dumps({"a": "b"}))]])
|
||||
session = SQLiteSession("max-turns-handler", ":memory:")
|
||||
try:
|
||||
if streamed:
|
||||
|
||||
+13
-13
@@ -6,20 +6,20 @@ from pydantic import BaseModel
|
||||
|
||||
from agents import Agent, RunContextWrapper, RunErrorDetails, Runner, RunResult
|
||||
from agents.agent_output import _WRAPPER_DICT_KEY
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.util._pretty_print import (
|
||||
pretty_print_result,
|
||||
pretty_print_run_error_details,
|
||||
pretty_print_run_result_streaming,
|
||||
)
|
||||
from tests.fake_model import FakeModel
|
||||
|
||||
from .test_responses import get_final_output_message, get_text_message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pretty_result():
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Hi there")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Hi there")])
|
||||
|
||||
agent = Agent(name="test_agent", model=model)
|
||||
result = await Runner.run(agent, input="Hello")
|
||||
@@ -92,8 +92,8 @@ RunErrorDetails:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pretty_run_result_streaming():
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Hi there")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Hi there")])
|
||||
|
||||
agent = Agent(name="test_agent", model=model)
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -122,8 +122,8 @@ class Foo(BaseModel):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pretty_run_result_structured_output():
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[
|
||||
get_text_message("Test"),
|
||||
get_final_output_message(Foo(bar="Hi there").model_dump_json()),
|
||||
@@ -150,8 +150,8 @@ RunResult:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pretty_run_result_streaming_structured_output():
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[
|
||||
get_text_message("Test"),
|
||||
get_final_output_message(Foo(bar="Hi there").model_dump_json()),
|
||||
@@ -184,8 +184,8 @@ RunResultStreaming:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pretty_run_result_list_structured_output():
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[
|
||||
get_text_message("Test"),
|
||||
get_final_output_message(
|
||||
@@ -219,8 +219,8 @@ RunResult:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pretty_run_result_streaming_list_structured_output():
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[
|
||||
get_text_message("Test"),
|
||||
get_final_output_message(
|
||||
|
||||
@@ -41,8 +41,8 @@ from agents.items import (
|
||||
)
|
||||
from agents.mcp.util import MCPUtil
|
||||
from agents.run_internal import run_loop
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.usage import Usage
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.mcp.helpers import FakeMCPServer
|
||||
from tests.mcp.model_compat import Tool as MCPTool
|
||||
from tests.test_responses import get_function_tool_call
|
||||
@@ -76,7 +76,7 @@ def _make_hosted_mcp_list_tools(server_label: str, tool_name: str) -> McpListToo
|
||||
|
||||
|
||||
def test_process_model_response_shell_call_without_tool_raises() -> None:
|
||||
agent = Agent(name="no-shell", model=FakeModel())
|
||||
agent = Agent(name="no-shell", model=ScriptedModel())
|
||||
shell_call = make_shell_call("shell-1")
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="shell tool"):
|
||||
@@ -111,7 +111,7 @@ def test_process_model_response_dispatches_falsy_shell_tool() -> None:
|
||||
|
||||
|
||||
def test_process_model_response_sets_title_for_local_mcp_function_tool() -> None:
|
||||
agent = Agent(name="local-mcp", model=FakeModel())
|
||||
agent = Agent(name="local-mcp", model=ScriptedModel())
|
||||
mcp_tool = MCPTool(name="search_docs", inputSchema={}, description=None, title="Search Docs")
|
||||
function_tool = MCPUtil.to_function_tool(
|
||||
mcp_tool,
|
||||
@@ -142,7 +142,7 @@ def test_process_model_response_sets_title_for_local_mcp_function_tool() -> None
|
||||
|
||||
|
||||
def test_process_model_response_uses_mcp_list_tools_metadata_for_hosted_mcp_calls() -> None:
|
||||
agent = Agent(name="hosted-mcp", model=FakeModel())
|
||||
agent = Agent(name="hosted-mcp", model=ScriptedModel())
|
||||
hosted_tool = HostedMCPTool(
|
||||
tool_config=cast(
|
||||
Any,
|
||||
@@ -186,7 +186,7 @@ def test_process_model_response_uses_mcp_list_tools_metadata_for_hosted_mcp_call
|
||||
|
||||
def test_process_model_response_skips_local_shell_execution_for_hosted_environment() -> None:
|
||||
shell_tool = ShellTool(environment={"type": "container_auto"})
|
||||
agent = Agent(name="hosted-shell", model=FakeModel(), tools=[shell_tool])
|
||||
agent = Agent(name="hosted-shell", model=ScriptedModel(), tools=[shell_tool])
|
||||
shell_call = make_shell_call("shell-hosted-1")
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
@@ -213,7 +213,7 @@ def test_process_model_response_sanitizes_shell_call_model_object() -> None:
|
||||
action=cast(Any, {"commands": ["echo hi"], "timeout_ms": 1000}),
|
||||
)
|
||||
shell_tool = ShellTool(environment={"type": "container_auto"})
|
||||
agent = Agent(name="hosted-shell-model", model=FakeModel(), tools=[shell_tool])
|
||||
agent = Agent(name="hosted-shell-model", model=ScriptedModel(), tools=[shell_tool])
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -252,7 +252,7 @@ def test_process_model_response_preserves_shell_call_output() -> None:
|
||||
}
|
||||
],
|
||||
}
|
||||
agent = Agent(name="shell-output", model=FakeModel())
|
||||
agent = Agent(name="shell-output", model=ScriptedModel())
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -288,7 +288,7 @@ def test_process_model_response_sanitizes_shell_call_output_model_object() -> No
|
||||
],
|
||||
),
|
||||
)
|
||||
agent = Agent(name="shell-output-model", model=FakeModel())
|
||||
agent = Agent(name="shell-output-model", model=ScriptedModel())
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -322,7 +322,7 @@ def test_process_model_response_sanitizes_shell_call_output_model_object() -> No
|
||||
|
||||
|
||||
def test_process_model_response_apply_patch_call_without_tool_raises() -> None:
|
||||
agent = Agent(name="no-apply", model=FakeModel())
|
||||
agent = Agent(name="no-apply", model=ScriptedModel())
|
||||
apply_patch_call = make_apply_patch_dict("apply-1", diff="-old\n+new\n")
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="apply_patch tool"):
|
||||
@@ -338,7 +338,7 @@ def test_process_model_response_apply_patch_call_without_tool_raises() -> None:
|
||||
def test_process_model_response_sanitizes_apply_patch_call_model_object() -> None:
|
||||
editor = RecordingEditor()
|
||||
apply_patch_tool = ApplyPatchTool(editor=editor)
|
||||
agent = Agent(name="apply-agent-model", model=FakeModel(), tools=[apply_patch_tool])
|
||||
agent = Agent(name="apply-agent-model", model=ScriptedModel(), tools=[apply_patch_tool])
|
||||
apply_patch_call = ResponseApplyPatchToolCall(
|
||||
type="apply_patch_call",
|
||||
id="ap_call_1",
|
||||
@@ -380,7 +380,7 @@ def test_process_model_response_sanitizes_apply_patch_call_model_object() -> Non
|
||||
def test_process_model_response_queues_apply_patch_call() -> None:
|
||||
editor = RecordingEditor()
|
||||
apply_patch_tool = ApplyPatchTool(editor=editor)
|
||||
agent = Agent(name="apply-agent", model=FakeModel(), tools=[apply_patch_tool])
|
||||
agent = Agent(name="apply-agent", model=ScriptedModel(), tools=[apply_patch_tool])
|
||||
apply_patch_call = make_apply_patch_dict("apply-1")
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
@@ -420,7 +420,7 @@ def test_process_model_response_dispatches_falsy_apply_patch_tool() -> None:
|
||||
def test_process_model_response_queues_hosted_apply_patch_from_custom_tool_call() -> None:
|
||||
editor = RecordingEditor()
|
||||
apply_patch_tool = ApplyPatchTool(editor=editor)
|
||||
agent = Agent(name="apply-agent-custom", model=FakeModel(), tools=[apply_patch_tool])
|
||||
agent = Agent(name="apply-agent-custom", model=ScriptedModel(), tools=[apply_patch_tool])
|
||||
custom_call = ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
name="apply_patch",
|
||||
@@ -456,7 +456,7 @@ def test_process_model_response_queues_custom_tool_call_for_custom_tool() -> Non
|
||||
on_invoke_tool=lambda _ctx, raw_input: raw_input,
|
||||
format={"type": "text"},
|
||||
)
|
||||
agent = Agent(name="custom-agent", model=FakeModel(), tools=[custom_tool])
|
||||
agent = Agent(name="custom-agent", model=ScriptedModel(), tools=[custom_tool])
|
||||
custom_call = ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
name="raw_editor",
|
||||
@@ -487,7 +487,7 @@ def test_process_model_response_prefers_namespaced_function_over_apply_patch_fal
|
||||
tools=[function_tool(lambda payload: payload, name_override="apply_patch_lookup")],
|
||||
)[0]
|
||||
all_tools: list[Tool] = [namespaced_tool]
|
||||
agent = Agent(name="billing-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="billing-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -511,7 +511,7 @@ def test_process_model_response_prefers_namespaced_function_over_apply_patch_fal
|
||||
|
||||
|
||||
def test_process_model_response_handles_compaction_item() -> None:
|
||||
agent = Agent(name="compaction-agent", model=FakeModel())
|
||||
agent = Agent(name="compaction-agent", model=ScriptedModel())
|
||||
compaction_item = ResponseCompactionItem(
|
||||
id="comp-1",
|
||||
encrypted_content="enc",
|
||||
@@ -537,7 +537,7 @@ def test_process_model_response_handles_compaction_item() -> None:
|
||||
|
||||
|
||||
def test_process_model_response_classifies_tool_search_items() -> None:
|
||||
agent = Agent(name="tool-search-agent", model=FakeModel())
|
||||
agent = Agent(name="tool-search-agent", model=ScriptedModel())
|
||||
tool_search_call = construct_type(
|
||||
type_=ResponseOutputItem,
|
||||
value={
|
||||
@@ -604,7 +604,7 @@ def test_process_model_response_uses_namespace_for_duplicate_function_names() ->
|
||||
tools=[billing_tool],
|
||||
)
|
||||
all_tools: list[Tool] = [*crm_namespace, *billing_namespace]
|
||||
agent = Agent(name="billing-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="billing-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -633,7 +633,7 @@ def test_process_model_response_collapses_synthetic_deferred_namespace_in_tools_
|
||||
name_override="get_weather",
|
||||
defer_loading=True,
|
||||
)
|
||||
agent = Agent(name="weather-agent", model=FakeModel(), tools=[deferred_tool])
|
||||
agent = Agent(name="weather-agent", model=ScriptedModel(), tools=[deferred_tool])
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -670,7 +670,7 @@ def test_process_model_response_rejects_bare_name_for_duplicate_namespaced_funct
|
||||
tools=[billing_tool],
|
||||
)
|
||||
all_tools: list[Tool] = [*crm_namespace, *billing_namespace]
|
||||
agent = Agent(name="billing-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="billing-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="Tool lookup_account not found"):
|
||||
run_loop.process_model_response(
|
||||
@@ -688,7 +688,7 @@ def test_process_model_response_uses_last_duplicate_top_level_function() -> None
|
||||
first_tool = function_tool(lambda customer_id: f"first:{customer_id}", name_override="lookup")
|
||||
second_tool = function_tool(lambda customer_id: f"second:{customer_id}", name_override="lookup")
|
||||
all_tools: list[Tool] = [first_tool, second_tool]
|
||||
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -707,7 +707,7 @@ def test_process_model_response_rejects_reserved_same_name_namespace_shape() ->
|
||||
invalid_tool._tool_namespace = "lookup_account"
|
||||
invalid_tool._tool_namespace_description = "Same-name namespace"
|
||||
all_tools: list[Tool] = [invalid_tool]
|
||||
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
with pytest.raises(UserError, match="synthetic namespace `lookup_account.lookup_account`"):
|
||||
run_loop.process_model_response(
|
||||
@@ -740,7 +740,7 @@ def test_process_model_response_rejects_qualified_name_collision_with_dotted_top
|
||||
tools=[function_tool(lambda customer_id: customer_id, name_override="lookup_account")],
|
||||
)[0]
|
||||
all_tools: list[Tool] = [dotted_top_level_tool, namespaced_tool]
|
||||
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
with pytest.raises(UserError, match="qualified name `crm.lookup_account`"):
|
||||
run_loop.process_model_response(
|
||||
@@ -771,7 +771,7 @@ def test_process_model_response_prefers_visible_top_level_function_over_deferred
|
||||
defer_loading=True,
|
||||
)
|
||||
all_tools: list[Tool] = [visible_tool, deferred_tool]
|
||||
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -801,7 +801,7 @@ def test_process_model_response_uses_internal_lookup_key_for_deferred_top_level_
|
||||
defer_loading=True,
|
||||
)
|
||||
all_tools: list[Tool] = [visible_tool, deferred_tool]
|
||||
agent = Agent(name="lookup-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="lookup-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -832,7 +832,7 @@ def test_process_model_response_preserves_synthetic_namespace_for_deferred_top_l
|
||||
defer_loading=True,
|
||||
)
|
||||
all_tools: list[Tool] = [deferred_tool]
|
||||
agent = Agent(name="weather-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="weather-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -858,10 +858,10 @@ def test_process_model_response_prefers_namespaced_function_over_handoff_name_co
|
||||
description="Billing tools",
|
||||
tools=[billing_tool],
|
||||
)
|
||||
handoff_target = Agent(name="lookup-agent", model=FakeModel())
|
||||
handoff_target = Agent(name="lookup-agent", model=ScriptedModel())
|
||||
lookup_handoff: Handoff = handoff(handoff_target, tool_name_override="lookup_account")
|
||||
all_tools: list[Tool] = [*billing_namespace]
|
||||
agent = Agent(name="billing-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="billing-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
agent=agent,
|
||||
@@ -890,7 +890,7 @@ def test_process_model_response_prefers_namespaced_function_over_handoff_name_co
|
||||
def test_process_model_response_rejects_mismatched_function_namespace() -> None:
|
||||
bare_tool = function_tool(lambda customer_id: customer_id, name_override="lookup_account")
|
||||
all_tools: list[Tool] = [bare_tool]
|
||||
agent = Agent(name="bare-agent", model=FakeModel(), tools=all_tools)
|
||||
agent = Agent(name="bare-agent", model=ScriptedModel(), tools=all_tools)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="crm.lookup_account"):
|
||||
run_loop.process_model_response(
|
||||
@@ -911,7 +911,7 @@ def test_process_model_response_rejects_mismatched_function_namespace() -> None:
|
||||
|
||||
|
||||
def test_process_model_response_collects_missing_function_tool_when_opted_in() -> None:
|
||||
agent = Agent(name="test", model=FakeModel(), tools=[function_tool(lambda: "ok")])
|
||||
agent = Agent(name="test", model=ScriptedModel(), tools=[function_tool(lambda: "ok")])
|
||||
missing_call = get_function_tool_call("missing_tool", "{}", call_id="call_missing")
|
||||
|
||||
processed = run_loop.process_model_response(
|
||||
|
||||
@@ -70,9 +70,10 @@ from agents.memory import SQLiteSession
|
||||
from agents.models.chatcmpl_converter import Converter as ChatCompletionsConverter
|
||||
from agents.models.openai_responses import Converter as ResponsesConverter
|
||||
from agents.run_internal.turn_resolution import process_model_response
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool_context import ToolContext
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .model_test_helpers import get_exact_output_stream_step
|
||||
from .test_responses import get_handoff_tool_call, get_text_message
|
||||
|
||||
PROGRAM_CALL_ID = "call_program"
|
||||
@@ -493,7 +494,7 @@ async def test_schema_backed_direct_tool_preserves_argument_error_formatter() ->
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_preserves_direct_error_for_schema_backed_tool() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
direct_call = ResponseFunctionToolCall(
|
||||
id="function_item",
|
||||
call_id=FUNCTION_CALL_ID,
|
||||
@@ -501,7 +502,7 @@ async def test_runner_preserves_direct_error_for_schema_backed_tool() -> None:
|
||||
arguments='{"sku":"A-1"}',
|
||||
type="function_call",
|
||||
)
|
||||
model.add_multiple_turn_outputs([[direct_call], [get_text_message("inventory lookup failed")]])
|
||||
model.extend([[direct_call], [get_text_message("inventory lookup failed")]])
|
||||
|
||||
@function_tool(allowed_callers=["direct", "programmatic"])
|
||||
def lookup_inventory(sku: str) -> InventoryOutput:
|
||||
@@ -526,7 +527,7 @@ async def test_runner_preserves_direct_error_for_schema_backed_tool() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_preserves_direct_default_timeout_for_schema_backed_tool() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
direct_call = ResponseFunctionToolCall(
|
||||
id="function_item",
|
||||
call_id=FUNCTION_CALL_ID,
|
||||
@@ -534,7 +535,7 @@ async def test_runner_preserves_direct_default_timeout_for_schema_backed_tool()
|
||||
arguments='{"sku":"A-1"}',
|
||||
type="function_call",
|
||||
)
|
||||
model.add_multiple_turn_outputs([[direct_call], [get_text_message("timed out")]])
|
||||
model.extend([[direct_call], [get_text_message("timed out")]])
|
||||
|
||||
@function_tool(allowed_callers=["direct", "programmatic"], timeout=0.01)
|
||||
async def lookup_inventory(sku: str) -> InventoryOutput:
|
||||
@@ -588,8 +589,8 @@ async def test_schema_backed_function_tool_accepts_conforming_custom_error_outpu
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_schema_backed_programmatic_tool_accepts_conforming_custom_timeout_output() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("timeout handled")],
|
||||
@@ -1190,8 +1191,8 @@ def test_process_model_response_rejects_program_items_without_programmatic_tool(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_rejects_program_item_without_programmatic_tool() -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([_program()])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([_program()])
|
||||
agent = Agent(name="inventory", model=model)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="programmatic_tool_calling tool"):
|
||||
@@ -1252,8 +1253,8 @@ def test_process_model_response_rejects_program_owned_calls_without_programmatic
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_does_not_execute_program_owned_call_without_programmatic_tool() -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([_function_call()])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([_function_call()])
|
||||
executed = False
|
||||
|
||||
@function_tool(allowed_callers=["programmatic"])
|
||||
@@ -1500,8 +1501,8 @@ def test_process_model_response_accepts_server_owned_parent_after_incomplete_out
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_does_not_execute_program_owned_call_without_parent_program() -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([_function_call()])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([_function_call()])
|
||||
executed = False
|
||||
|
||||
@function_tool(allowed_callers=["programmatic"])
|
||||
@@ -1891,12 +1892,12 @@ def test_process_model_response_accepts_allowed_program_owned_shell_output(
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streamed", [False, True])
|
||||
async def test_runner_executes_and_replays_programmatic_function_calls(streamed: bool) -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("42 units are available")],
|
||||
]
|
||||
outputs = [
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("42 units are available")],
|
||||
]
|
||||
model = ScriptedModel(
|
||||
[get_exact_output_stream_step(output) for output in outputs] if streamed else outputs
|
||||
)
|
||||
|
||||
@function_tool(allowed_callers=["programmatic"])
|
||||
@@ -1923,9 +1924,9 @@ async def test_runner_executes_and_replays_programmatic_function_calls(streamed:
|
||||
result = await Runner.run(agent, "Check inventory")
|
||||
|
||||
assert result.final_output == "42 units are available"
|
||||
assert model.first_turn_args is not None
|
||||
assert model.first_turn_args["model_settings"].tool_choice == "programmatic_tool_calling"
|
||||
assert model.last_turn_args["model_settings"].tool_choice is None
|
||||
assert bool(model.calls)
|
||||
assert model.calls[0].model_settings.tool_choice == "programmatic_tool_calling"
|
||||
assert model.calls[-1].model_settings.tool_choice is None
|
||||
|
||||
function_outputs = [
|
||||
item
|
||||
@@ -1941,18 +1942,21 @@ async def test_runner_executes_and_replays_programmatic_function_calls(streamed:
|
||||
}
|
||||
assert _caller_dict(raw_output["caller"]) == PROGRAM_CALLER
|
||||
|
||||
replayed_output = next(
|
||||
item
|
||||
for item in model.last_turn_args["input"]
|
||||
if isinstance(item, dict) and item.get("type") == "function_call_output"
|
||||
replayed_output = cast(
|
||||
dict[str, Any],
|
||||
next(
|
||||
item
|
||||
for item in model.calls[-1].input
|
||||
if isinstance(item, dict) and item.get("type") == "function_call_output"
|
||||
),
|
||||
)
|
||||
assert _caller_dict(replayed_output["caller"]) == PROGRAM_CALLER
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_typed_programmatic_tool_preserves_input_guardrail_rejection() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("request rejected")],
|
||||
@@ -1991,8 +1995,8 @@ async def test_typed_programmatic_tool_preserves_input_guardrail_rejection() ->
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_typed_programmatic_tool_preserves_default_timeout_result() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("request timed out")],
|
||||
@@ -2023,8 +2027,8 @@ async def test_typed_programmatic_tool_preserves_default_timeout_result() -> Non
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_typed_programmatic_tool_preserves_output_guardrail_rejection() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("request rejected")],
|
||||
@@ -2070,12 +2074,12 @@ async def test_typed_programmatic_tool_preserves_approval_rejection(
|
||||
serialize_state: bool,
|
||||
rejection_message: str | None,
|
||||
) -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("request rejected")],
|
||||
]
|
||||
outputs = [
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("request rejected")],
|
||||
]
|
||||
model = ScriptedModel(
|
||||
[get_exact_output_stream_step(output) for output in outputs] if streaming else outputs
|
||||
)
|
||||
|
||||
@function_tool(allowed_callers=["programmatic"], needs_approval=True)
|
||||
@@ -2114,19 +2118,22 @@ async def test_typed_programmatic_tool_preserves_approval_rejection(
|
||||
expected_message = rejection_message or "Tool execution was not approved."
|
||||
assert json.loads(function_outputs[0]["output"]) == {"error": expected_message}
|
||||
assert _caller_dict(function_outputs[0]["caller"]) == PROGRAM_CALLER
|
||||
assert model.last_turn_args is not None
|
||||
replayed_output = next(
|
||||
item
|
||||
for item in model.last_turn_args["input"]
|
||||
if isinstance(item, dict) and item.get("type") == "function_call_output"
|
||||
assert bool(model.calls)
|
||||
replayed_output = cast(
|
||||
dict[str, Any],
|
||||
next(
|
||||
item
|
||||
for item in model.calls[-1].input
|
||||
if isinstance(item, dict) and item.get("type") == "function_call_output"
|
||||
),
|
||||
)
|
||||
assert json.loads(replayed_output["output"]) == {"error": expected_message}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rebuilt_mapping_programmatic_approval_preserves_caller() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("done")],
|
||||
@@ -2164,8 +2171,8 @@ async def test_rebuilt_mapping_programmatic_approval_preserves_caller() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rebuilt_mapping_programmatic_approval_rechecks_caller_permissions() -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([_program(), _function_call()])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([_program(), _function_call()])
|
||||
executed = False
|
||||
|
||||
@function_tool(allowed_callers=["programmatic"], needs_approval=True)
|
||||
@@ -2197,8 +2204,8 @@ async def test_rebuilt_mapping_programmatic_approval_rechecks_caller_permissions
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("parent_state", ["missing", "completed"])
|
||||
async def test_rebuilt_programmatic_approval_requires_active_parent(parent_state: str) -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("done")],
|
||||
@@ -2250,8 +2257,8 @@ async def test_rebuilt_programmatic_approval_requires_active_parent(parent_state
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_typed_programmatic_tool_preserves_pre_approval_guardrail_rejection() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("request rejected")],
|
||||
@@ -2290,7 +2297,7 @@ async def test_typed_programmatic_tool_preserves_pre_approval_guardrail_rejectio
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_handles_multiple_pauses_from_one_program() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
second_call = ResponseFunctionToolCall(
|
||||
id="function_item_2",
|
||||
call_id="call_lookup_2",
|
||||
@@ -2299,7 +2306,7 @@ async def test_runner_handles_multiple_pauses_from_one_program() -> None:
|
||||
caller=CallerProgram(type="program", caller_id=PROGRAM_CALL_ID),
|
||||
type="function_call",
|
||||
)
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output("incomplete"), second_call],
|
||||
@@ -2334,13 +2341,13 @@ async def test_runner_handles_multiple_pauses_from_one_program() -> None:
|
||||
|
||||
assert result.final_output == "84 units are available"
|
||||
assert calls == ["A-1", "B-2"]
|
||||
assert model.last_turn_args["model_settings"].tool_choice is None
|
||||
assert model.calls[-1].model_settings.tool_choice is None
|
||||
assert len([item for item in result.new_items if isinstance(item, ToolCallOutputItem)]) == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_executes_programmatic_batch_calls_concurrently() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
batch_calls = [
|
||||
ResponseFunctionToolCall(
|
||||
id=f"function_item_{index}",
|
||||
@@ -2352,7 +2359,7 @@ async def test_runner_executes_programmatic_batch_calls_concurrently() -> None:
|
||||
)
|
||||
for index in range(9)
|
||||
]
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[_program(), *batch_calls],
|
||||
[_program_output(), get_text_message("batch complete")],
|
||||
@@ -2398,8 +2405,8 @@ async def test_runner_executes_programmatic_batch_calls_concurrently() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_previous_response_id_continuation_sends_only_program_function_output() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("done")],
|
||||
@@ -2419,8 +2426,8 @@ async def test_previous_response_id_continuation_sends_only_program_function_out
|
||||
result = await Runner.run(agent, "Check inventory", auto_previous_response_id=True)
|
||||
|
||||
assert result.final_output == "done"
|
||||
assert model.last_turn_args["previous_response_id"] == "resp-789"
|
||||
last_input = model.last_turn_args["input"]
|
||||
assert model.calls[-1].previous_response_id == "resp-789"
|
||||
last_input = model.calls[-1].input
|
||||
assert isinstance(last_input, list)
|
||||
assert len(last_input) == 1
|
||||
function_output = cast(dict[str, Any], last_input[0])
|
||||
@@ -2433,8 +2440,8 @@ async def test_previous_response_id_continuation_sends_only_program_function_out
|
||||
async def test_previous_response_id_continuation_accepts_server_owned_program_output(
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([_program_output(), get_text_message("done")])
|
||||
output = [_program_output(), get_text_message("done")]
|
||||
model = ScriptedModel([get_exact_output_stream_step(output) if streamed else output])
|
||||
|
||||
@function_tool(allowed_callers=["programmatic"])
|
||||
def lookup_inventory(sku: str) -> InventoryOutput:
|
||||
@@ -2470,13 +2477,13 @@ async def test_previous_response_id_continuation_accepts_server_owned_program_ou
|
||||
)
|
||||
|
||||
assert result.final_output == "done"
|
||||
assert model.last_turn_args["previous_response_id"] == "response_with_program_parent"
|
||||
assert model.calls[-1].previous_response_id == "response_with_program_parent"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_previous_response_id_continuation_accepts_repeated_program_pause() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_function_call()],
|
||||
[_program_output(), get_text_message("done")],
|
||||
@@ -2520,8 +2527,8 @@ async def test_previous_response_id_continuation_accepts_repeated_program_pause(
|
||||
async def test_run_state_round_trip_preserves_server_owned_program_parent(
|
||||
parent_source: str,
|
||||
) -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_function_call()],
|
||||
[_program_output(), get_text_message("done")],
|
||||
@@ -2566,13 +2573,13 @@ async def test_run_state_round_trip_preserves_server_owned_program_parent(
|
||||
|
||||
assert executed is True
|
||||
assert result.final_output == "done"
|
||||
assert model.last_turn_args["previous_response_id"] == "resp-789"
|
||||
assert model.calls[-1].previous_response_id == "resp-789"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_round_trip_preserves_program_history_and_caller() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_text_message("done")],
|
||||
@@ -2614,9 +2621,9 @@ async def test_sqlite_session_round_trip_preserves_program_history_and_caller()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_handoff_summarizes_complete_programmatic_transcript() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
delegate = Agent(name="delegate", model=model)
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[_program(), _function_call()],
|
||||
[_program_output(), get_handoff_tool_call(delegate)],
|
||||
@@ -2692,8 +2699,8 @@ async def test_non_function_programmatic_outputs_preserve_caller() -> None:
|
||||
caller = cast(Any, PROGRAM_CALLER)
|
||||
|
||||
async def run_tool(tool: Any, tool_call: Any) -> dict[str, Any]:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[_program(), tool_call], [get_text_message("done")]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[_program(), tool_call], [get_text_message("done")]])
|
||||
agent = Agent(
|
||||
name="tool agent",
|
||||
model=model,
|
||||
@@ -2752,13 +2759,13 @@ async def test_non_function_programmatic_outputs_preserve_caller() -> None:
|
||||
|
||||
apply_patch_output = await run_tool(
|
||||
ApplyPatchTool(editor=Editor(), allowed_callers=["programmatic"]),
|
||||
ResponseApplyPatchToolCall(
|
||||
id="apply_patch_item",
|
||||
call_id="call_apply_patch",
|
||||
operation=OperationCreateFile(type="create_file", path="example.txt", diff="hello"),
|
||||
status="completed",
|
||||
type="apply_patch_call",
|
||||
caller=caller,
|
||||
),
|
||||
{
|
||||
"id": "apply_patch_item",
|
||||
"call_id": "call_apply_patch",
|
||||
"operation": {"type": "create_file", "path": "example.txt", "diff": "hello"},
|
||||
"status": "completed",
|
||||
"type": "apply_patch_call",
|
||||
"caller": caller,
|
||||
},
|
||||
)
|
||||
assert _caller_dict(apply_patch_output["caller"]) == PROGRAM_CALLER
|
||||
|
||||
@@ -4,13 +4,13 @@ import pytest
|
||||
from openai.types.responses.response_create_params import ContextManagement, PromptCacheOptions
|
||||
|
||||
from agents import Agent, ModelSettings, RunConfig, Runner
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel, PromptCacheFakeModel
|
||||
from .test_responses import get_function_tool, get_function_tool_call, get_text_message
|
||||
from .utils.simple_session import SimpleListSession
|
||||
|
||||
|
||||
def _sent_prompt_cache_key(model: FakeModel, *, first_turn: bool = False) -> str | None:
|
||||
def _sent_prompt_cache_key(model: ScriptedModel, *, first_turn: bool = False) -> str | None:
|
||||
model_settings = _sent_model_settings(model, first_turn=first_turn)
|
||||
extra_args = model_settings.extra_args or {}
|
||||
value = extra_args.get("prompt_cache_key")
|
||||
@@ -18,23 +18,27 @@ def _sent_prompt_cache_key(model: FakeModel, *, first_turn: bool = False) -> str
|
||||
return value
|
||||
|
||||
|
||||
def _sent_model_settings(model: FakeModel, *, first_turn: bool = False) -> ModelSettings:
|
||||
args = model.first_turn_args if first_turn else model.last_turn_args
|
||||
assert args is not None
|
||||
model_settings = args["model_settings"]
|
||||
def _sent_model_settings(model: ScriptedModel, *, first_turn: bool = False) -> ModelSettings:
|
||||
call = model.calls[0] if first_turn else model.calls[-1]
|
||||
model_settings = call.model_settings
|
||||
assert isinstance(model_settings, ModelSettings)
|
||||
return model_settings
|
||||
|
||||
|
||||
class DefaultPromptCacheDisabledFakeModel(FakeModel):
|
||||
class PromptCacheScriptedModel(ScriptedModel):
|
||||
def _supports_default_prompt_cache_key(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class DefaultPromptCacheDisabledScriptedModel(ScriptedModel):
|
||||
def _supports_default_prompt_cache_key(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_generates_prompt_cache_key_by_default() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
await Runner.run(agent, "hi")
|
||||
@@ -46,21 +50,21 @@ async def test_runner_generates_prompt_cache_key_by_default() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_adds_prompt_cache_key_without_adding_model_call_keyword() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
await Runner.run(agent, "hi")
|
||||
|
||||
# PromptCacheFakeModel uses the public Model.get_response() signature. If the runner added
|
||||
# PromptCacheScriptedModel uses the public Model.get_response() signature. If the runner added
|
||||
# prompt_cache_key as a direct model-call keyword, this run would fail before this assertion.
|
||||
assert _sent_prompt_cache_key(model) is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_reuses_generated_prompt_cache_key_across_turns() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = PromptCacheScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("lookup", "{}")],
|
||||
[get_text_message("done")],
|
||||
@@ -78,8 +82,8 @@ async def test_runner_reuses_generated_prompt_cache_key_across_turns() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_skips_generated_prompt_cache_key_when_model_disables_default() -> None:
|
||||
model = DefaultPromptCacheDisabledFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = DefaultPromptCacheDisabledScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
await Runner.run(agent, "hi")
|
||||
@@ -89,8 +93,8 @@ async def test_runner_skips_generated_prompt_cache_key_when_model_disables_defau
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_respects_existing_extra_args_prompt_cache_key() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
@@ -106,8 +110,8 @@ async def test_runner_respects_existing_extra_args_prompt_cache_key() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_respects_existing_extra_body_prompt_cache_key() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
@@ -124,8 +128,8 @@ async def test_runner_respects_existing_extra_body_prompt_cache_key() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_generates_prompt_cache_key_with_unrelated_extra_args() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
model_settings = ModelSettings(extra_args={"service_tier": "flex"})
|
||||
agent = Agent(
|
||||
name="test",
|
||||
@@ -146,8 +150,8 @@ async def test_runner_generates_prompt_cache_key_with_unrelated_extra_args() ->
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_preserves_context_management_when_adding_prompt_cache_key() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
context_management: list[ContextManagement] = [
|
||||
{"type": "compaction", "compact_threshold": 200000}
|
||||
]
|
||||
@@ -170,8 +174,8 @@ async def test_runner_preserves_context_management_when_adding_prompt_cache_key(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_preserves_prompt_cache_options_when_adding_prompt_cache_key() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
prompt_cache_options: PromptCacheOptions = {"mode": "explicit", "ttl": "30m"}
|
||||
model_settings = ModelSettings(prompt_cache_options=prompt_cache_options)
|
||||
agent = Agent(name="test", model=model, model_settings=model_settings)
|
||||
@@ -188,8 +192,8 @@ async def test_runner_preserves_prompt_cache_options_when_adding_prompt_cache_ke
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_skips_generated_key_when_model_settings_has_prompt_cache_keys() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
@@ -206,8 +210,8 @@ async def test_runner_skips_generated_key_when_model_settings_has_prompt_cache_k
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_uses_group_id_as_stable_prompt_cache_key_boundary() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
await Runner.run(agent, "hi", run_config=RunConfig(group_id="thread-123"))
|
||||
@@ -219,8 +223,8 @@ async def test_runner_uses_group_id_as_stable_prompt_cache_key_boundary() -> Non
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_uses_session_id_as_stable_prompt_cache_key_boundary() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="test", model=model)
|
||||
session = SimpleListSession(session_id="session-123")
|
||||
|
||||
@@ -233,8 +237,8 @@ async def test_runner_uses_session_id_as_stable_prompt_cache_key_boundary() -> N
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_runner_generates_prompt_cache_key_by_default() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, "hi")
|
||||
@@ -248,8 +252,8 @@ async def test_streamed_runner_generates_prompt_cache_key_by_default() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_state_preserves_generated_prompt_cache_key_on_resume() -> None:
|
||||
model = PromptCacheFakeModel()
|
||||
model.set_next_output([get_text_message("first")])
|
||||
model = PromptCacheScriptedModel()
|
||||
model.enqueue([get_text_message("first")])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
first_result = await Runner.run(agent, "hi")
|
||||
@@ -257,7 +261,7 @@ async def test_run_state_preserves_generated_prompt_cache_key_on_resume() -> Non
|
||||
state = first_result.to_state()
|
||||
restored_state = await type(state).from_string(agent, state.to_string())
|
||||
|
||||
model.set_next_output([get_text_message("second")])
|
||||
model.enqueue([get_text_message("second")])
|
||||
await Runner.run(agent, restored_state)
|
||||
|
||||
assert first_key is not None
|
||||
|
||||
+11
-11
@@ -1,8 +1,8 @@
|
||||
import pytest
|
||||
|
||||
from agents import Agent, run_demo_loop
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_function_tool,
|
||||
get_function_tool_call,
|
||||
@@ -14,8 +14,8 @@ from .test_responses import (
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_demo_loop_conversation(monkeypatch, capsys):
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[get_text_message("hello")], [get_text_message("good")]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[get_text_message("hello")], [get_text_message("good")]])
|
||||
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
@@ -27,7 +27,7 @@ async def test_run_demo_loop_conversation(monkeypatch, capsys):
|
||||
output = capsys.readouterr().out
|
||||
assert "hello" in output
|
||||
assert "good" in output
|
||||
assert model.last_turn_args["input"] == [
|
||||
assert model.calls[-1].input == [
|
||||
get_text_input_item("Hi"),
|
||||
get_text_message("hello").model_dump(exclude_unset=True),
|
||||
get_text_input_item("How are you?"),
|
||||
@@ -36,7 +36,7 @@ async def test_run_demo_loop_conversation(monkeypatch, capsys):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_demo_loop_streaming(monkeypatch, capsys):
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
target_agent = Agent(name="target", model=model)
|
||||
agent = Agent(
|
||||
name="test",
|
||||
@@ -47,7 +47,7 @@ async def test_run_demo_loop_streaming(monkeypatch, capsys):
|
||||
|
||||
# A single user turn that exercises every streamed event branch:
|
||||
# a tool call, the tool output, a handoff (agent update), then a text answer.
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("foo", "{}")],
|
||||
[get_handoff_tool_call(target_agent)],
|
||||
@@ -69,7 +69,7 @@ async def test_run_demo_loop_streaming(monkeypatch, capsys):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_demo_loop_exits_on_eof(monkeypatch, capsys):
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
def raise_eof(_=" > ") -> str:
|
||||
@@ -80,13 +80,13 @@ async def test_run_demo_loop_exits_on_eof(monkeypatch, capsys):
|
||||
await run_demo_loop(agent, stream=False)
|
||||
|
||||
# The loop should terminate cleanly without ever invoking the model.
|
||||
assert model.last_turn_args == {}
|
||||
assert not model.calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_demo_loop_skips_empty_input(monkeypatch, capsys):
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[get_text_message("hello")]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[get_text_message("hello")]])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Empty lines are ignored; only the non-empty input reaches the runner.
|
||||
@@ -97,4 +97,4 @@ async def test_run_demo_loop_skips_empty_input(monkeypatch, capsys):
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert "hello" in output
|
||||
assert model.last_turn_args["input"] == [get_text_input_item("Hi")]
|
||||
assert model.calls[-1].input == [get_text_input_item("Hi")]
|
||||
|
||||
@@ -6,7 +6,7 @@ from openai.types.responses.response_usage import InputTokensDetails, OutputToke
|
||||
|
||||
from agents import ModelBehaviorError, ModelSettings, ModelTracing, OpenAIResponsesModel, trace
|
||||
from agents.tracing.span_data import ResponseSpanData
|
||||
from tests import fake_model
|
||||
from tests import model_test_helpers
|
||||
|
||||
from .testing_processor import assert_no_spans, fetch_normalized_spans, fetch_ordered_spans
|
||||
|
||||
@@ -49,7 +49,7 @@ class DummyResponse:
|
||||
def __aiter__(self):
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=fake_model.get_response_obj(self.output),
|
||||
response=model_test_helpers.get_response_obj(self.output),
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
@@ -249,7 +249,7 @@ async def test_stream_response_creates_trace(monkeypatch):
|
||||
async def __aiter__(self):
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=fake_model.get_response_obj([], "dummy-id-123"),
|
||||
response=model_test_helpers.get_response_obj([], "dummy-id-123"),
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
@@ -322,7 +322,7 @@ async def test_stream_response_failed_or_incomplete_terminal_event_creates_trace
|
||||
class DummyTerminalEvent:
|
||||
def __init__(self):
|
||||
self.type = terminal_event_type
|
||||
self.response = fake_model.get_response_obj([], "dummy-id-terminal")
|
||||
self.response = model_test_helpers.get_response_obj([], "dummy-id-terminal")
|
||||
self.sequence_number = 0
|
||||
|
||||
class DummyStream:
|
||||
@@ -391,7 +391,7 @@ async def test_stream_non_data_tracing_doesnt_set_response_id(monkeypatch):
|
||||
async def __aiter__(self):
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=fake_model.get_response_obj([], "dummy-id-123"),
|
||||
response=model_test_helpers.get_response_obj([], "dummy-id-123"),
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
@@ -467,7 +467,7 @@ async def test_stream_disabled_tracing_doesnt_create_span(monkeypatch):
|
||||
async def __aiter__(self):
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=fake_model.get_response_obj([], "dummy-id-123"),
|
||||
response=model_test_helpers.get_response_obj([], "dummy-id-123"),
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
|
||||
+8
-7
@@ -1,13 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import Agent, Runner
|
||||
from agents.run import AgentRunner, set_default_agent_runner
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_text_input_item, get_text_message
|
||||
|
||||
|
||||
@@ -16,7 +17,7 @@ async def test_static_run_methods_call_into_default_runner() -> None:
|
||||
runner = mock.Mock(spec=AgentRunner)
|
||||
set_default_agent_runner(runner)
|
||||
|
||||
agent = Agent(name="test", model=FakeModel())
|
||||
agent = Agent(name="test", model=ScriptedModel())
|
||||
await Runner.run(agent, input="test")
|
||||
runner.run.assert_called_once()
|
||||
|
||||
@@ -29,16 +30,16 @@ async def test_static_run_methods_call_into_default_runner() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_preserves_duplicate_user_messages() -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
input_items = [get_text_input_item("repeat"), get_text_input_item("repeat")]
|
||||
|
||||
await Runner.run(agent, input=input_items)
|
||||
|
||||
sent_input = model.last_turn_args["input"]
|
||||
sent_input = model.calls[-1].input
|
||||
assert isinstance(sent_input, list)
|
||||
assert len(sent_input) == 2
|
||||
assert sent_input[0]["content"] == "repeat"
|
||||
assert sent_input[1]["content"] == "repeat"
|
||||
assert cast(dict[str, Any], sent_input[0])["content"] == "repeat"
|
||||
assert cast(dict[str, Any], sent_input[1])["content"] == "repeat"
|
||||
|
||||
+25
-23
@@ -19,8 +19,8 @@ from agents.run import __all__ as run_exports
|
||||
from agents.run_config import SandboxConcurrencyLimits, SandboxRunConfig
|
||||
from agents.sandbox.manifest import Manifest
|
||||
from agents.sandbox.snapshot import NoopSnapshotSpec
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_text_message
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ class DummyProvider(ModelProvider):
|
||||
|
||||
def __init__(self, model_to_return: Model | None = None) -> None:
|
||||
self.last_requested: str | None = None
|
||||
self.model_to_return: Model = model_to_return or FakeModel()
|
||||
self.model_to_return: Model = model_to_return or ScriptedModel()
|
||||
|
||||
def get_model(self, model_name: str | None) -> Model:
|
||||
# record the requested model name and return our test model
|
||||
@@ -119,7 +119,7 @@ def test_run_config_rejects_unknown_first_party_dictionary_fields(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_accepts_dictionary_run_configuration() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("done")])
|
||||
model = ScriptedModel(steps=[[get_text_message("done")]])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
result = await Runner.run(
|
||||
@@ -138,8 +138,8 @@ async def test_model_provider_on_run_config_is_used_for_agent_model_name() -> No
|
||||
provided in the ``RunConfig``, the ``Runner`` should resolve the model using the
|
||||
``model_provider`` on the ``RunConfig``.
|
||||
"""
|
||||
fake_model = FakeModel(initial_output=[get_text_message("from-provider")])
|
||||
provider = DummyProvider(model_to_return=fake_model)
|
||||
scripted_model = ScriptedModel(steps=[[get_text_message("from-provider")]])
|
||||
provider = DummyProvider(model_to_return=scripted_model)
|
||||
agent = Agent(name="test", model="test-model")
|
||||
run_config = RunConfig(model_provider=provider)
|
||||
result = await Runner.run(agent, input="any", run_config=run_config)
|
||||
@@ -154,8 +154,8 @@ async def test_run_config_model_name_override_takes_precedence() -> None:
|
||||
When a model name string is set on the RunConfig, then that name should be looked up
|
||||
using the RunConfig's model_provider, and should override any model on the agent.
|
||||
"""
|
||||
fake_model = FakeModel(initial_output=[get_text_message("override-name")])
|
||||
provider = DummyProvider(model_to_return=fake_model)
|
||||
scripted_model = ScriptedModel(steps=[[get_text_message("override-name")]])
|
||||
provider = DummyProvider(model_to_return=scripted_model)
|
||||
agent = Agent(name="test", model="agent-model")
|
||||
run_config = RunConfig(model="override-name", model_provider=provider)
|
||||
result = await Runner.run(agent, input="any", run_config=run_config)
|
||||
@@ -179,14 +179,15 @@ async def test_run_config_model_name_override_uses_model_specific_default_settin
|
||||
than the default fallback model.
|
||||
"""
|
||||
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", "gpt-5.4-mini")
|
||||
fake_model = FakeModel(initial_output=[get_text_message("override-name")])
|
||||
provider = DummyProvider(model_to_return=fake_model)
|
||||
scripted_model = ScriptedModel(steps=[[get_text_message("override-name")]])
|
||||
provider = DummyProvider(model_to_return=scripted_model)
|
||||
agent = Agent(name="test")
|
||||
run_config = RunConfig(model=model_name, model_provider=provider)
|
||||
result = await Runner.run(agent, input="any", run_config=run_config)
|
||||
assert result.final_output == "override-name"
|
||||
assert fake_model.first_turn_args is not None
|
||||
model_settings = fake_model.first_turn_args["model_settings"]
|
||||
assert bool(scripted_model.calls)
|
||||
model_settings = scripted_model.calls[0].model_settings
|
||||
assert model_settings.reasoning is not None
|
||||
assert model_settings.reasoning.effort == reasoning_effort
|
||||
assert model_settings.verbosity == "low"
|
||||
|
||||
@@ -199,8 +200,8 @@ async def test_run_config_model_settings_override_implicit_model_specific_defaul
|
||||
RunConfig model settings should overlay the implicit defaults for the resolved model name.
|
||||
"""
|
||||
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", "gpt-5.4-mini")
|
||||
fake_model = FakeModel(initial_output=[get_text_message("override-name")])
|
||||
provider = DummyProvider(model_to_return=fake_model)
|
||||
scripted_model = ScriptedModel(steps=[[get_text_message("override-name")]])
|
||||
provider = DummyProvider(model_to_return=scripted_model)
|
||||
agent = Agent(name="test")
|
||||
run_config = RunConfig(
|
||||
model="gpt-5",
|
||||
@@ -209,8 +210,9 @@ async def test_run_config_model_settings_override_implicit_model_specific_defaul
|
||||
)
|
||||
result = await Runner.run(agent, input="any", run_config=run_config)
|
||||
assert result.final_output == "override-name"
|
||||
assert fake_model.first_turn_args is not None
|
||||
model_settings = fake_model.first_turn_args["model_settings"]
|
||||
assert bool(scripted_model.calls)
|
||||
model_settings = scripted_model.calls[0].model_settings
|
||||
assert model_settings.reasoning is not None
|
||||
assert model_settings.reasoning.effort == "low"
|
||||
assert model_settings.verbosity == "low"
|
||||
assert model_settings.temperature == 0.3
|
||||
@@ -222,11 +224,11 @@ async def test_run_config_model_override_object_takes_precedence() -> None:
|
||||
When a concrete Model instance is set on the RunConfig, then that instance should be
|
||||
returned by AgentRunner._get_model regardless of the agent's model.
|
||||
"""
|
||||
fake_model = FakeModel(initial_output=[get_text_message("override-object")])
|
||||
scripted_model = ScriptedModel(steps=[[get_text_message("override-object")]])
|
||||
agent = Agent(name="test", model="agent-model")
|
||||
run_config = RunConfig(model=fake_model)
|
||||
run_config = RunConfig(model=scripted_model)
|
||||
result = await Runner.run(agent, input="any", run_config=run_config)
|
||||
# Our FakeModel on the RunConfig should have been used.
|
||||
# The ScriptedModel on the RunConfig should have been used.
|
||||
assert result.final_output == "override-object"
|
||||
|
||||
|
||||
@@ -237,13 +239,13 @@ async def test_agent_model_object_is_used_when_present() -> None:
|
||||
not specify a model override, then that object should be used directly without
|
||||
consulting the RunConfig's model_provider.
|
||||
"""
|
||||
fake_model = FakeModel(initial_output=[get_text_message("from-agent-object")])
|
||||
scripted_model = ScriptedModel(steps=[[get_text_message("from-agent-object")]])
|
||||
provider = DummyProvider()
|
||||
agent = Agent(name="test", model=fake_model)
|
||||
agent = Agent(name="test", model=scripted_model)
|
||||
run_config = RunConfig(model_provider=provider)
|
||||
result = await Runner.run(agent, input="any", run_config=run_config)
|
||||
# The dummy provider should never have been called, and the output should come from
|
||||
# the FakeModel on the agent.
|
||||
# the ScriptedModel on the agent.
|
||||
assert provider.last_requested is None
|
||||
assert result.final_output == "from-agent-object"
|
||||
|
||||
@@ -352,7 +354,7 @@ def test_tool_name_collision_policy_rejects_invalid_value() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_dictionary_rejects_invalid_tool_name_collision_policy() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("done")])
|
||||
model = ScriptedModel(steps=[[get_text_message("done")]])
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
with pytest.raises(
|
||||
@@ -365,4 +367,4 @@ async def test_runner_dictionary_rejects_invalid_tool_name_collision_policy() ->
|
||||
run_config={"tool_name_collision_policy": cast(Any, "erorr")},
|
||||
)
|
||||
|
||||
assert model.first_turn_args is None
|
||||
assert not model.calls
|
||||
|
||||
@@ -3,16 +3,16 @@ import json
|
||||
import pytest
|
||||
|
||||
from agents import Agent, MaxTurnsExceeded, RunErrorDetails, Runner
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_function_tool, get_function_tool_call, get_text_message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_error_includes_data():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model, tools=[get_function_tool("foo", "res")])
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("1"), get_function_tool_call("foo", json.dumps({"a": "b"}))],
|
||||
[get_text_message("done")],
|
||||
@@ -29,9 +29,9 @@ async def test_run_error_includes_data():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_run_error_includes_data():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model, tools=[get_function_tool("foo", "res")])
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("1"), get_function_tool_call("foo", json.dumps({"a": "b"}))],
|
||||
[get_text_message("done")],
|
||||
|
||||
+30
-34
@@ -7,15 +7,14 @@ import pytest
|
||||
from agents.agent import Agent
|
||||
from agents.items import ItemHelpers, ModelResponse, TResponseInputItem
|
||||
from agents.lifecycle import AgentHooks, RunHooks
|
||||
from agents.models.interface import Model
|
||||
from agents.run import Runner
|
||||
from agents.run_context import AgentHookContext, RunContextWrapper, TContext
|
||||
from agents.run_internal.run_loop import validate_run_hooks
|
||||
from agents.testing import ModelStep, ScriptedModel
|
||||
from agents.tool import Tool, function_tool
|
||||
from agents.tool_context import ToolContext
|
||||
from tests.test_agent_llm_hooks import AgentHooksForTests
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_function_tool,
|
||||
get_function_tool_call,
|
||||
@@ -89,11 +88,11 @@ class RunHooksForTests(RunHooks):
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_run_hooks_with_llm():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
|
||||
# Simulate a single LLM call producing an output:
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
await Runner.run(agent, input="hello", hooks=hooks)
|
||||
# Expect one on_agent_start, one on_llm_start, one on_llm_end, and one on_agent_end
|
||||
assert hooks.events == {
|
||||
@@ -107,10 +106,10 @@ async def test_async_run_hooks_with_llm():
|
||||
# test_sync_run_hook_with_llm()
|
||||
def test_sync_run_hook_with_llm():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
|
||||
# Simulate a single LLM call producing an output:
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
Runner.run_sync(agent, input="hello", hooks=hooks)
|
||||
# Expect one on_agent_start, one on_llm_start, one on_llm_end, and one on_agent_end
|
||||
assert hooks.events == {
|
||||
@@ -125,10 +124,10 @@ def test_sync_run_hook_with_llm():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_run_hooks_with_llm():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
|
||||
# Simulate a single LLM call producing an output:
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
stream = Runner.run_streamed(agent, input="hello", hooks=hooks)
|
||||
|
||||
async for event in stream.stream_events():
|
||||
@@ -160,13 +159,13 @@ async def test_streamed_run_hooks_with_llm():
|
||||
async def test_async_run_hooks_with_agent_hooks_with_llm():
|
||||
hooks = RunHooksForTests()
|
||||
agent_hooks = AgentHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
agent = Agent(
|
||||
name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[], hooks=agent_hooks
|
||||
)
|
||||
# Simulate a single LLM call producing an output:
|
||||
model.set_next_output([get_text_message("hello")])
|
||||
model.enqueue([get_text_message("hello")])
|
||||
await Runner.run(agent, input="hello", hooks=hooks)
|
||||
# Expect one on_agent_start, one on_llm_start, one on_llm_end, and one on_agent_end
|
||||
assert hooks.events == {
|
||||
@@ -182,13 +181,13 @@ async def test_async_run_hooks_with_agent_hooks_with_llm():
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_hooks_llm_error_non_streaming(monkeypatch):
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
|
||||
|
||||
async def boom(*args, **kwargs):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(FakeModel, "get_response", boom, raising=True)
|
||||
monkeypatch.setattr(ScriptedModel, "get_response", boom, raising=True)
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
await Runner.run(agent, input="hello", hooks=hooks)
|
||||
@@ -206,7 +205,7 @@ class DummyAgentHooks(AgentHooks):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_run_rejects_agent_hooks():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="A", model=model)
|
||||
hooks = cast(RunHooks, DummyAgentHooks())
|
||||
|
||||
@@ -215,7 +214,7 @@ async def test_runner_run_rejects_agent_hooks():
|
||||
|
||||
|
||||
def test_runner_run_streamed_rejects_agent_hooks():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="A", model=model)
|
||||
hooks = cast(RunHooks, DummyAgentHooks())
|
||||
|
||||
@@ -228,13 +227,9 @@ def test_validate_run_hooks_rejects_non_hook_objects() -> None:
|
||||
validate_run_hooks(object())
|
||||
|
||||
|
||||
class BoomModel(Model):
|
||||
async def get_response(self, *a, **k):
|
||||
raise AssertionError("get_response should not be called in streaming test")
|
||||
|
||||
async def stream_response(self, *a, **k):
|
||||
yield {"foo": "bar"}
|
||||
raise RuntimeError("stream blew up")
|
||||
async def _failing_stream(_call):
|
||||
yield {"foo": "bar"}
|
||||
raise RuntimeError("stream blew up")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -244,7 +239,8 @@ async def test_streamed_run_hooks_llm_error(monkeypatch):
|
||||
but do NOT emit on_llm_end (current behavior), and the exception propagates.
|
||||
"""
|
||||
hooks = RunHooksForTests()
|
||||
agent = Agent(name="A", model=BoomModel(), tools=[get_function_tool("f", "res")], handoffs=[])
|
||||
model = ScriptedModel([ModelStep.stream(_failing_stream)])
|
||||
agent = Agent(name="A", model=model, tools=[get_function_tool("f", "res")], handoffs=[])
|
||||
|
||||
stream = Runner.run_streamed(agent, input="hello", hooks=hooks)
|
||||
|
||||
@@ -276,10 +272,10 @@ class RunHooksWithTurnInput(RunHooks):
|
||||
async def test_run_hooks_receives_turn_input_string():
|
||||
"""Test that on_agent_start receives turn_input when input is a string."""
|
||||
hooks = RunHooksWithTurnInput()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
model.set_next_output([get_text_message("response")])
|
||||
model.enqueue([get_text_message("response")])
|
||||
await Runner.run(agent, input="hello world", hooks=hooks)
|
||||
|
||||
assert len(hooks.captured_turn_inputs) == 1
|
||||
@@ -293,7 +289,7 @@ async def test_run_hooks_receives_turn_input_string():
|
||||
async def test_run_hooks_receives_turn_input_list():
|
||||
"""Test that on_agent_start receives turn_input when input is a list."""
|
||||
hooks = RunHooksWithTurnInput()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
input_items: list[Any] = [
|
||||
@@ -301,7 +297,7 @@ async def test_run_hooks_receives_turn_input_list():
|
||||
{"role": "user", "content": "second message"},
|
||||
]
|
||||
|
||||
model.set_next_output([get_text_message("response")])
|
||||
model.enqueue([get_text_message("response")])
|
||||
await Runner.run(agent, input=input_items, hooks=hooks)
|
||||
|
||||
assert len(hooks.captured_turn_inputs) == 1
|
||||
@@ -315,10 +311,10 @@ async def test_run_hooks_receives_turn_input_list():
|
||||
async def test_run_hooks_receives_turn_input_streamed():
|
||||
"""Test that on_agent_start receives turn_input in streamed mode."""
|
||||
hooks = RunHooksWithTurnInput()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
model.set_next_output([get_text_message("response")])
|
||||
model.enqueue([get_text_message("response")])
|
||||
result = Runner.run_streamed(agent, input="streamed input", hooks=hooks)
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
@@ -332,7 +328,7 @@ async def test_run_hooks_receives_turn_input_streamed():
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_hooks_count_tool_and_handoff_invocations():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(
|
||||
@@ -342,7 +338,7 @@ async def test_run_hooks_count_tool_and_handoff_invocations():
|
||||
tools=[get_function_tool("some_function", "result")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
|
||||
[get_text_message("a_message"), get_handoff_tool_call(agent_1)],
|
||||
@@ -362,7 +358,7 @@ async def test_run_hooks_count_tool_and_handoff_invocations():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_run_hooks_count_tool_and_handoff_invocations():
|
||||
hooks = RunHooksForTests()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
agent_1 = Agent(name="test_1", model=model)
|
||||
agent_2 = Agent(
|
||||
@@ -372,7 +368,7 @@ async def test_streamed_run_hooks_count_tool_and_handoff_invocations():
|
||||
tools=[get_function_tool("some_function", "result")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="call_1"),
|
||||
@@ -433,10 +429,10 @@ async def test_tool_end_hooks_receive_raw_function_tool_result():
|
||||
|
||||
run_hooks = RecordingRunHooks()
|
||||
agent_hooks = RecordingAgentHooks()
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="test", model=model, tools=[get_metadata], hooks=agent_hooks)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("get_metadata", "{}")],
|
||||
[get_text_message("done")],
|
||||
|
||||
@@ -30,8 +30,8 @@ from agents.run_internal.run_loop import (
|
||||
SingleStepResult,
|
||||
)
|
||||
from agents.run_state import RunState
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.usage import Usage
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_function_tool_call, get_text_message
|
||||
from tests.utils.hitl import (
|
||||
make_agent,
|
||||
@@ -44,7 +44,7 @@ from tests.utils.simple_session import SimpleListSession
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_interrupted_turn_final_output_short_circuit(monkeypatch) -> None:
|
||||
agent: Agent[dict[str, str]] = make_agent(model=FakeModel())
|
||||
agent: Agent[dict[str, str]] = make_agent(model=ScriptedModel())
|
||||
context_wrapper = make_context_wrapper()
|
||||
|
||||
async def fake_execute_tool_plan(*_: object, **__: object):
|
||||
|
||||
+100
-194
@@ -4,10 +4,9 @@ from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
@@ -39,7 +38,7 @@ from openai.types.responses.response_usage import InputTokensDetails
|
||||
from openai.types.responses.tool_param import Mcp
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from agents import Agent, Model, ModelSettings, RunConfig, RunHooks, Runner, handoff, trace
|
||||
from agents import Agent, ModelSettings, RunConfig, RunHooks, Runner, handoff, trace
|
||||
from agents._tool_invocation import tool_invocation_identity_and_scope
|
||||
from agents.computer import Computer
|
||||
from agents.exceptions import ModelBehaviorError, UserError
|
||||
@@ -66,7 +65,6 @@ from agents.items import (
|
||||
ToolSearchOutputItem,
|
||||
TResponseInputItem,
|
||||
TResponseOutputItem,
|
||||
TResponseStreamEvent,
|
||||
)
|
||||
from agents.run_context import RunContextWrapper
|
||||
from agents.run_error_handlers import RunErrorHandlerResult, RunErrorHandlers
|
||||
@@ -110,9 +108,8 @@ from agents.sandbox import Manifest
|
||||
from agents.sandbox.capabilities.capability import Capability
|
||||
from agents.sandbox.entries import BaseEntry, Mount, MountStrategyBase
|
||||
from agents.sandbox.sandboxes.unix_local import UnixLocalSandboxClient, UnixLocalSandboxSessionState
|
||||
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
|
||||
from agents.sandbox.snapshot import LocalSnapshot, NoopSnapshot
|
||||
from agents.sandbox.types import ExecResult
|
||||
from agents.sandbox.snapshot import LocalSnapshot
|
||||
from agents.testing import ModelCall, ModelStep, ScriptedModel, scripted_sandbox_session
|
||||
from agents.tool import (
|
||||
ApplyPatchTool,
|
||||
ComputerTool,
|
||||
@@ -135,9 +132,7 @@ from agents.tool_guardrails import (
|
||||
)
|
||||
from agents.tracing.traces import TraceState
|
||||
from agents.usage import Usage
|
||||
from tests.utils.factories import TestSessionState
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_final_output_message,
|
||||
get_function_tool_call,
|
||||
@@ -166,49 +161,6 @@ _NEXT_UNSUPPORTED_SCHEMA_VERSION = f"{_CURRENT_SCHEMA_MAJOR}.{int(_CURRENT_SCHEM
|
||||
TContext = TypeVar("TContext")
|
||||
|
||||
|
||||
class _IdentitySandboxSession(BaseSandboxSession):
|
||||
def __init__(self, root: str) -> None:
|
||||
self.state = TestSessionState(
|
||||
manifest=Manifest(root=root),
|
||||
snapshot=NoopSnapshot(id=f"snapshot:{root}"),
|
||||
)
|
||||
|
||||
async def start(self) -> None:
|
||||
return None
|
||||
|
||||
async def stop(self) -> None:
|
||||
return None
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
return None
|
||||
|
||||
async def running(self) -> bool:
|
||||
return True
|
||||
|
||||
async def read(self, path: Path, *, user: object = None) -> Any:
|
||||
_ = (path, user)
|
||||
raise AssertionError("read() should not be called")
|
||||
|
||||
async def write(self, path: Path, data: io.IOBase, *, user: object = None) -> None:
|
||||
_ = (path, data, user)
|
||||
raise AssertionError("write() should not be called")
|
||||
|
||||
async def _exec_internal(
|
||||
self,
|
||||
*command: Any,
|
||||
timeout: float | None = None,
|
||||
) -> ExecResult:
|
||||
_ = (command, timeout)
|
||||
raise AssertionError("_exec_internal() should not be called")
|
||||
|
||||
async def persist_workspace(self) -> Any:
|
||||
raise AssertionError("persist_workspace() should not be called")
|
||||
|
||||
async def hydrate_workspace(self, data: Any) -> None:
|
||||
_ = data
|
||||
raise AssertionError("hydrate_workspace() should not be called")
|
||||
|
||||
|
||||
class _IdentityCapability(Capability):
|
||||
type: str = "identity"
|
||||
setting: str
|
||||
@@ -315,8 +267,8 @@ class TestRunState:
|
||||
|
||||
trace_state = FalsyTraceState(trace_id="trace_falsy")
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_final_output_message("done")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_final_output_message("done")])
|
||||
result = await Runner.run(Agent(name="test", model=model), "input")
|
||||
result._trace_state = trace_state
|
||||
|
||||
@@ -324,8 +276,8 @@ class TestRunState:
|
||||
assert isinstance(restored, FalsyTraceState)
|
||||
assert restored.trace_id == "trace_falsy"
|
||||
|
||||
streaming_model = FakeModel()
|
||||
streaming_model.set_next_output([get_final_output_message("done")])
|
||||
streaming_model = ScriptedModel()
|
||||
streaming_model.enqueue([get_final_output_message("done")])
|
||||
streaming_result = Runner.run_streamed(
|
||||
Agent(name="streaming-test", model=streaming_model),
|
||||
"input",
|
||||
@@ -625,13 +577,21 @@ class TestRunState:
|
||||
|
||||
first_alpha_capability = _IdentityCapability(setting="alpha")
|
||||
first_beta_capability = _IdentityCapability(setting="beta")
|
||||
first_alpha_capability.bind(_IdentitySandboxSession("/workspace/first-alpha"))
|
||||
first_beta_capability.bind(_IdentitySandboxSession("/workspace/first-beta"))
|
||||
first_alpha_capability.bind(
|
||||
scripted_sandbox_session(manifest=Manifest(root="/workspace/first-alpha"))
|
||||
)
|
||||
first_beta_capability.bind(
|
||||
scripted_sandbox_session(manifest=Manifest(root="/workspace/first-beta"))
|
||||
)
|
||||
|
||||
second_alpha_capability = _IdentityCapability(setting="alpha")
|
||||
second_beta_capability = _IdentityCapability(setting="beta")
|
||||
second_alpha_capability.bind(_IdentitySandboxSession("/workspace/second-alpha"))
|
||||
second_beta_capability.bind(_IdentitySandboxSession("/workspace/second-beta"))
|
||||
second_alpha_capability.bind(
|
||||
scripted_sandbox_session(manifest=Manifest(root="/workspace/second-alpha"))
|
||||
)
|
||||
second_beta_capability.bind(
|
||||
scripted_sandbox_session(manifest=Manifest(root="/workspace/second-beta"))
|
||||
)
|
||||
|
||||
first_alpha_signature = _capability_identity_signature(first_alpha_capability)
|
||||
first_beta_signature = _capability_identity_signature(first_beta_capability)
|
||||
@@ -707,8 +667,8 @@ class TestRunState:
|
||||
def approval_tool() -> str:
|
||||
return "approved"
|
||||
|
||||
first_model = FakeModel()
|
||||
second_model = FakeModel()
|
||||
first_model = ScriptedModel()
|
||||
second_model = ScriptedModel()
|
||||
first = Agent(name="duplicate", model=first_model)
|
||||
second = Agent(
|
||||
name="duplicate",
|
||||
@@ -719,8 +679,8 @@ class TestRunState:
|
||||
first.handoffs = [second]
|
||||
second.handoffs = [first]
|
||||
|
||||
first_model.add_multiple_turn_outputs([[get_handoff_tool_call(second)]])
|
||||
second_model.add_multiple_turn_outputs(
|
||||
first_model.extend([[get_handoff_tool_call(second)]])
|
||||
second_model.extend(
|
||||
[[get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")]]
|
||||
)
|
||||
|
||||
@@ -1930,7 +1890,7 @@ class TestRunState:
|
||||
|
||||
probe_agent = Agent(
|
||||
name="ApprovalProbeAgent",
|
||||
model=FakeModel(initial_output=[get_text_message("done")]),
|
||||
model=ScriptedModel(steps=[[get_text_message("done")]]),
|
||||
)
|
||||
await Runner.run(
|
||||
probe_agent,
|
||||
@@ -4452,109 +4412,52 @@ class TestDeserializeHelpers:
|
||||
return True
|
||||
return False
|
||||
|
||||
class ResumeAwareToolModel(Model):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
tool_name: str,
|
||||
tool_arguments: str,
|
||||
final_text: str,
|
||||
call_prefix: str,
|
||||
preceding_tool_name: str | None = None,
|
||||
) -> None:
|
||||
self.tool_name = tool_name
|
||||
self.tool_arguments = tool_arguments
|
||||
self.final_text = final_text
|
||||
self.call_prefix = call_prefix
|
||||
self.preceding_tool_name = preceding_tool_name
|
||||
self.call_count = 0
|
||||
def _make_resume_aware_tool_model(
|
||||
*,
|
||||
tool_name: str,
|
||||
tool_arguments: str,
|
||||
final_text: str,
|
||||
call_prefix: str,
|
||||
preceding_tool_name: str | None = None,
|
||||
) -> ScriptedModel:
|
||||
tool_call_count = 0
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Any],
|
||||
output_schema: Any,
|
||||
handoffs: list[Any],
|
||||
tracing: Any,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: Any | None,
|
||||
) -> ModelResponse:
|
||||
del (
|
||||
system_instructions,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
tracing,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
prompt,
|
||||
)
|
||||
if _has_function_call_output(input):
|
||||
def _respond(call: ModelCall) -> ModelResponse:
|
||||
nonlocal tool_call_count
|
||||
if _has_function_call_output(call.input):
|
||||
return ModelResponse(
|
||||
output=[get_text_message(self.final_text)],
|
||||
output=[get_text_message(final_text)],
|
||||
usage=Usage(),
|
||||
response_id=f"{self.call_prefix}-done",
|
||||
response_id=f"{call_prefix}-done",
|
||||
)
|
||||
|
||||
self.call_count += 1
|
||||
tool_call_count += 1
|
||||
output: list[TResponseOutputItem] = []
|
||||
if self.preceding_tool_name is not None:
|
||||
if preceding_tool_name is not None:
|
||||
output.append(
|
||||
ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
name=self.preceding_tool_name,
|
||||
call_id=f"{self.call_prefix}-preceding-{self.call_count}",
|
||||
name=preceding_tool_name,
|
||||
call_id=f"{call_prefix}-preceding-{tool_call_count}",
|
||||
arguments="{}",
|
||||
)
|
||||
)
|
||||
output.append(
|
||||
ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
name=self.tool_name,
|
||||
call_id=f"{self.call_prefix}-{id(self)}-{self.call_count}",
|
||||
arguments=self.tool_arguments,
|
||||
name=tool_name,
|
||||
call_id=f"{call_prefix}-{id(model)}-{tool_call_count}",
|
||||
arguments=tool_arguments,
|
||||
)
|
||||
)
|
||||
return ModelResponse(
|
||||
output=output,
|
||||
usage=Usage(),
|
||||
response_id=f"{self.call_prefix}-call-{self.call_count}",
|
||||
response_id=f"{call_prefix}-call-{tool_call_count}",
|
||||
)
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Any],
|
||||
output_schema: Any,
|
||||
handoffs: list[Any],
|
||||
tracing: Any,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: Any | None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
del (
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
tracing,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
prompt,
|
||||
)
|
||||
if False:
|
||||
yield cast(TResponseStreamEvent, {})
|
||||
raise RuntimeError("Streaming is not supported in this test.")
|
||||
model = ScriptedModel(ModelStep.respond(_respond) for _ in range(3))
|
||||
return model
|
||||
|
||||
tool_calls: list[str] = []
|
||||
|
||||
@@ -4563,7 +4466,7 @@ class TestDeserializeHelpers:
|
||||
tool_calls.append(text)
|
||||
return f"approved:{text}"
|
||||
|
||||
inner_model = ResumeAwareToolModel(
|
||||
inner_model = _make_resume_aware_tool_model(
|
||||
tool_name="inner_sensitive_tool",
|
||||
tool_arguments=json.dumps({"text": "hello"}),
|
||||
final_text="inner-complete",
|
||||
@@ -4575,7 +4478,7 @@ class TestDeserializeHelpers:
|
||||
tool_name="inner_agent_tool",
|
||||
tool_description="Inner agent tool",
|
||||
)
|
||||
outer_model = ResumeAwareToolModel(
|
||||
outer_model = _make_resume_aware_tool_model(
|
||||
tool_name="inner_agent_tool",
|
||||
tool_arguments=json.dumps({"input": "hello"}),
|
||||
final_text="outer-complete",
|
||||
@@ -4628,6 +4531,8 @@ class TestDeserializeHelpers:
|
||||
assert resumed_result_two.final_output == "outer-complete"
|
||||
assert resumed_result_two.interruptions == []
|
||||
assert tool_calls == (["hello", "hello"] if approve_nested_tool else [])
|
||||
inner_model.assert_complete()
|
||||
outer_model.assert_complete()
|
||||
|
||||
async def test_json_decode_error_handling(self):
|
||||
"""Test that invalid JSON raises appropriate error."""
|
||||
@@ -4667,18 +4572,18 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_from_run_state(self):
|
||||
"""Test resuming a run from a RunState."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
# First run - create a state
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input")
|
||||
|
||||
# Create RunState from result
|
||||
state = result1.to_state()
|
||||
|
||||
# Resume from state
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = await Runner.run(agent, state)
|
||||
|
||||
assert result2.final_output == "Second response"
|
||||
@@ -4686,16 +4591,16 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_from_run_state_does_not_mutate_source_result(self):
|
||||
"""Resuming from a state must not append to the raw_responses already returned."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input")
|
||||
assert len(result1.raw_responses) == 1
|
||||
|
||||
state = result1.to_state()
|
||||
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = await Runner.run(agent, state)
|
||||
|
||||
# The second run accumulates on top of the first, but the RunResult that was
|
||||
@@ -4707,15 +4612,15 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_does_not_append_to_the_state_it_resumed_from(self):
|
||||
"""A resumed run must not accumulate its responses into the caller's checkpoint."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input")
|
||||
state = result1.to_state()
|
||||
serialized_before = state.to_json()["model_responses"]
|
||||
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = await Runner.run(agent, state)
|
||||
assert len(result2.raw_responses) == 2
|
||||
|
||||
@@ -4725,22 +4630,22 @@ class TestRunStateResumption:
|
||||
assert state.to_json()["model_responses"] == serialized_before
|
||||
|
||||
# Re-running the same checkpoint therefore replays only its own history.
|
||||
model.set_next_output([get_text_message("Third response")])
|
||||
model.enqueue([get_text_message("Third response")])
|
||||
result3 = await Runner.run(agent, state)
|
||||
assert len(result3.raw_responses) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_resume_does_not_append_to_the_state_it_resumed_from(self):
|
||||
"""A streamed resume must not accumulate its items into the caller's checkpoint."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input")
|
||||
state = result1.to_state()
|
||||
serialized_before = state.to_json()["session_items"]
|
||||
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = Runner.run_streamed(agent, state)
|
||||
async for _ in result2.stream_events():
|
||||
pass
|
||||
@@ -4750,7 +4655,7 @@ class TestRunStateResumption:
|
||||
assert state.to_json()["session_items"] == serialized_before
|
||||
|
||||
# Without this, the abandoned attempt's message leaks into the replayed history.
|
||||
model.set_next_output([get_text_message("Third response")])
|
||||
model.enqueue([get_text_message("Third response")])
|
||||
result3 = Runner.run_streamed(agent, state)
|
||||
async for _ in result3.stream_events():
|
||||
pass
|
||||
@@ -4760,10 +4665,10 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resumed_max_turns_handler_does_not_append_to_state_items(self):
|
||||
"""A resumed run that trips max turns must not append to the state's items."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input", max_turns=1)
|
||||
state = result1.to_state()
|
||||
serialized_before = state.to_json()["generated_items"]
|
||||
@@ -4780,15 +4685,15 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_fresh_runs_still_report_their_own_history(self):
|
||||
"""Boundary: a run that starts without a state is unaffected by the copies."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input")
|
||||
assert len(result1.raw_responses) == 1
|
||||
assert len(result1.new_items) == 1
|
||||
|
||||
model.set_next_output([get_text_message("Streamed response")])
|
||||
model.enqueue([get_text_message("Streamed response")])
|
||||
result2 = Runner.run_streamed(agent, "Second input")
|
||||
async for _ in result2.stream_events():
|
||||
pass
|
||||
@@ -4798,12 +4703,12 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_from_run_state_with_context(self):
|
||||
"""Test resuming a run from a RunState with context override."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
# First run with context
|
||||
context1 = {"key": "value1"}
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input", context=context1)
|
||||
|
||||
# Create RunState from result
|
||||
@@ -4811,7 +4716,7 @@ class TestRunStateResumption:
|
||||
|
||||
# Resume from state with different context (should use new context)
|
||||
context2 = {"key": "value2"}
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = await Runner.run(agent, state, context=context2)
|
||||
|
||||
# New context should be used.
|
||||
@@ -4823,18 +4728,18 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_from_run_state_with_conversation_id(self):
|
||||
"""Test resuming a run from a RunState with conversation_id."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
# First run
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input", conversation_id="conv123")
|
||||
|
||||
# Create RunState from result
|
||||
state = result1.to_state()
|
||||
|
||||
# Resume from state with conversation_id
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = await Runner.run(agent, state, conversation_id="conv123")
|
||||
|
||||
assert result2.final_output == "Second response"
|
||||
@@ -4842,18 +4747,18 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_from_run_state_with_previous_response_id(self):
|
||||
"""Test resuming a run from a RunState with previous_response_id."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
# First run
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input", previous_response_id="resp123")
|
||||
|
||||
# Create RunState from result
|
||||
state = result1.to_state()
|
||||
|
||||
# Resume from state with previous_response_id
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = await Runner.run(agent, state, previous_response_id="resp123")
|
||||
|
||||
assert result2.final_output == "Second response"
|
||||
@@ -4861,7 +4766,7 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_from_run_state_with_interruption(self):
|
||||
"""Test resuming a run from a RunState with an interruption."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
async def tool_func() -> str:
|
||||
return "tool_result"
|
||||
@@ -4875,7 +4780,8 @@ class TestRunStateResumption:
|
||||
)
|
||||
|
||||
# First run - create an interruption
|
||||
model.set_next_output([get_function_tool_call("test_tool", "{}")])
|
||||
model.enqueue([get_function_tool_call("test_tool", "{}")])
|
||||
model.enqueue([])
|
||||
result1 = await Runner.run(agent, "First input")
|
||||
|
||||
# Create RunState from result
|
||||
@@ -4886,7 +4792,7 @@ class TestRunStateResumption:
|
||||
state.approve(state.get_interruptions()[0])
|
||||
|
||||
# Resume from state - should execute approved tools
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = await Runner.run(agent, state)
|
||||
|
||||
assert result2.final_output == "Second response"
|
||||
@@ -4894,18 +4800,18 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_from_run_state_streamed(self):
|
||||
"""Test resuming a run from a RunState using run_streamed."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
# First run
|
||||
model.set_next_output([get_text_message("First response")])
|
||||
model.enqueue([get_text_message("First response")])
|
||||
result1 = await Runner.run(agent, "First input")
|
||||
|
||||
# Create RunState from result
|
||||
state = result1.to_state()
|
||||
|
||||
# Resume from state using run_streamed
|
||||
model.set_next_output([get_text_message("Second response")])
|
||||
model.enqueue([get_text_message("Second response")])
|
||||
result2 = Runner.run_streamed(agent, state)
|
||||
|
||||
events = []
|
||||
@@ -4920,8 +4826,8 @@ class TestRunStateResumption:
|
||||
async def test_resume_from_run_state_streamed_uses_context_from_state(self):
|
||||
"""Test that streaming with RunState uses context from state."""
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
# Create a RunState with context
|
||||
@@ -4940,8 +4846,8 @@ class TestRunStateResumption:
|
||||
async def test_resume_from_run_state_streamed_with_context_override(self):
|
||||
"""Test that streaming uses provided context override when resuming."""
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
# Create a RunState with context
|
||||
@@ -4959,7 +4865,7 @@ class TestRunStateResumption:
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_result_streaming_to_state_with_interruptions(self):
|
||||
"""Test RunResultStreaming.to_state() sets _current_step with interruptions."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="TestAgent", model=model)
|
||||
|
||||
async def test_tool() -> str:
|
||||
@@ -4969,7 +4875,7 @@ class TestRunStateResumption:
|
||||
agent.tools = [tool]
|
||||
|
||||
# Create a run that will have interruptions
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("test_tool", json.dumps({}))],
|
||||
[get_text_message("done")],
|
||||
@@ -9549,7 +9455,7 @@ async def _interrupted_approval_state_with_tool_input(
|
||||
return text
|
||||
|
||||
model, agent = make_model_and_agent(tools=[needs_ok], name="agent")
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("needs_ok", json.dumps({"text": "one"}), call_id="1")],
|
||||
[get_final_output_message("done")],
|
||||
@@ -9647,17 +9553,17 @@ async def test_resume_nested_agent_as_tool_with_context_override() -> None:
|
||||
output_tokens=3,
|
||||
total_tokens=20,
|
||||
)
|
||||
nested_model = FakeModel()
|
||||
nested_model.set_hardcoded_usage(nested_turn_usage)
|
||||
nested_model = ScriptedModel()
|
||||
nested_model.set_default_usage(nested_turn_usage)
|
||||
nested_agent = Agent(name="nested", tools=[needs_ok], model=nested_model)
|
||||
nested_model.add_multiple_turn_outputs(
|
||||
nested_model.extend(
|
||||
[
|
||||
[get_function_tool_call("needs_ok", json.dumps({"text": "one"}), call_id="inner-1")],
|
||||
[get_final_output_message("nested-done")],
|
||||
]
|
||||
)
|
||||
|
||||
outer_model = FakeModel()
|
||||
outer_model = ScriptedModel()
|
||||
outer = Agent(
|
||||
name="outer",
|
||||
tools=[
|
||||
@@ -9669,7 +9575,7 @@ async def test_resume_nested_agent_as_tool_with_context_override() -> None:
|
||||
],
|
||||
model=outer_model,
|
||||
)
|
||||
outer_model.add_multiple_turn_outputs(
|
||||
outer_model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
|
||||
@@ -19,10 +19,11 @@ from agents.run_context import RunContextWrapper
|
||||
from agents.run_internal.oai_conversation import OpenAIServerConversationTracker
|
||||
from agents.run_internal.run_steps import NextStepInterruption, NextStepRunAgain
|
||||
from agents.run_state import CURRENT_SCHEMA_VERSION, RunState
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import Tool
|
||||
from agents.usage import Usage
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .model_test_helpers import get_exact_output_stream_step
|
||||
from .test_computer_tool_lifecycle import FakeComputer
|
||||
from .test_responses import get_function_tool_call, get_text_message
|
||||
from .utils.simple_session import SimpleListSession
|
||||
@@ -53,7 +54,7 @@ async def _make_after_turn_state(
|
||||
*,
|
||||
session: SimpleListSession | None = None,
|
||||
auto_previous_response_id: bool = False,
|
||||
) -> tuple[FakeModel, Agent[Any], RunState[Any], list[str]]:
|
||||
) -> tuple[ScriptedModel, Agent[Any], RunState[Any], list[str]]:
|
||||
calls: list[str] = []
|
||||
|
||||
@function_tool(name_override="record_destination")
|
||||
@@ -61,8 +62,8 @@ async def _make_after_turn_state(
|
||||
calls.append(destination)
|
||||
return f"recorded:{destination}"
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_destination",
|
||||
@@ -136,13 +137,13 @@ async def test_after_turn_resume_admits_input_after_tool_output_exactly_once() -
|
||||
session = SimpleListSession()
|
||||
model, agent, state, calls = await _make_after_turn_state(session=session)
|
||||
state.add_input("Change the destination to Tokyo")
|
||||
model.set_next_output([get_text_message("Updated")])
|
||||
model.enqueue([get_text_message("Updated")])
|
||||
|
||||
result = await Runner.run(agent, state, session=session)
|
||||
|
||||
assert result.final_output == "Updated"
|
||||
assert calls == ["Paris"]
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_item_type(item) for item in model_input] == [
|
||||
"user",
|
||||
"function_call",
|
||||
@@ -171,7 +172,7 @@ async def test_after_turn_resume_admits_input_after_tool_output_exactly_once() -
|
||||
async def test_streamed_resume_matches_pending_input_ordering() -> None:
|
||||
model, agent, state, calls = await _make_after_turn_state()
|
||||
state.add_input("Change the destination to Tokyo")
|
||||
model.set_next_output([get_text_message("Updated")])
|
||||
model.enqueue([get_text_message("Updated")])
|
||||
|
||||
result = Runner.run_streamed(agent, state)
|
||||
async for _ in result.stream_events():
|
||||
@@ -179,7 +180,7 @@ async def test_streamed_resume_matches_pending_input_ordering() -> None:
|
||||
|
||||
assert result.final_output == "Updated"
|
||||
assert calls == ["Paris"]
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_item_type(item) for item in model_input] == [
|
||||
"user",
|
||||
"function_call",
|
||||
@@ -199,14 +200,14 @@ async def test_streamed_resume_matches_pending_input_ordering() -> None:
|
||||
async def test_server_managed_resume_sends_pending_input_as_unsent_delta_once() -> None:
|
||||
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Change the destination to Tokyo")
|
||||
model.set_next_output([get_text_message("Updated")])
|
||||
model.enqueue([get_text_message("Updated")])
|
||||
|
||||
result = await Runner.run(agent, state)
|
||||
|
||||
assert result.final_output == "Updated"
|
||||
assert calls == ["Paris"]
|
||||
assert model.last_turn_args["previous_response_id"] == "resp-789"
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
assert model.calls[-1].previous_response_id == "resp-789"
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_item_type(item) for item in model_input] == ["function_call_output", "user"]
|
||||
assert [_message_text(item) for item in model_input].count(
|
||||
"Change the destination to Tokyo"
|
||||
@@ -243,7 +244,7 @@ async def test_server_managed_resume_sends_identical_late_input_in_later_occurre
|
||||
) -> None:
|
||||
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Repeat")
|
||||
model.set_next_output(
|
||||
model.enqueue(
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_destination",
|
||||
@@ -261,7 +262,7 @@ async def test_server_managed_resume_sends_identical_late_input_in_later_occurre
|
||||
state = await RunState.from_json(agent, first_resume.to_state().to_json())
|
||||
admitted_before = next(item for item in state._generated_items if isinstance(item, InputItem))
|
||||
state.add_input("Repeat")
|
||||
model.set_next_output([get_text_message("Done")])
|
||||
model.enqueue([get_text_message("Done")])
|
||||
|
||||
if streamed_second_resume:
|
||||
streamed_result = Runner.run_streamed(agent, state)
|
||||
@@ -274,7 +275,7 @@ async def test_server_managed_resume_sends_identical_late_input_in_later_occurre
|
||||
|
||||
assert final_output == "Done"
|
||||
assert calls == ["Paris", "Rome"]
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in model_input].count("Repeat") == 1
|
||||
admitted_after = [item for item in state._generated_items if isinstance(item, InputItem)]
|
||||
assert [item.input_id for item in admitted_after].count(admitted_before.input_id) == 1
|
||||
@@ -290,8 +291,8 @@ async def test_unresolved_approval_keeps_pending_input_until_tool_finishes() ->
|
||||
calls.append(value)
|
||||
return f"approved:{value}"
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[get_function_tool_call("protected_tool", '{"value":"one"}', call_id="call-protected")]
|
||||
)
|
||||
agent = Agent(name="assistant", model=model, tools=[protected_tool])
|
||||
@@ -305,12 +306,12 @@ async def test_unresolved_approval_keeps_pending_input_until_tool_finishes() ->
|
||||
assert _message_text(state.pending_input[0]) == "Late input"
|
||||
|
||||
state.approve(state.get_interruptions()[0])
|
||||
model.set_next_output([get_text_message("Done")])
|
||||
model.enqueue([get_text_message("Done")])
|
||||
resumed = await Runner.run(agent, state)
|
||||
|
||||
assert resumed.final_output == "Done"
|
||||
assert calls == ["one"]
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_item_type(item) for item in model_input][-2:] == ["function_call_output", "user"]
|
||||
assert _message_text(model_input[-1]) == "Late input"
|
||||
|
||||
@@ -324,8 +325,8 @@ async def test_streamed_after_turn_cancel_keeps_pending_input_for_next_resume()
|
||||
calls.append(value)
|
||||
return f"approved:{value}"
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output(
|
||||
model = ScriptedModel()
|
||||
model.enqueue(
|
||||
[get_function_tool_call("protected_tool", '{"value":"one"}', call_id="call-protected")]
|
||||
)
|
||||
agent = Agent(name="assistant", model=model, tools=[protected_tool])
|
||||
@@ -342,10 +343,10 @@ async def test_streamed_after_turn_cancel_keeps_pending_input_for_next_resume()
|
||||
assert calls == ["one"]
|
||||
assert _message_text(state.pending_input[0]) == "Late input"
|
||||
|
||||
model.set_next_output([get_text_message("Done")])
|
||||
model.enqueue([get_text_message("Done")])
|
||||
result = await Runner.run(agent, state)
|
||||
assert result.final_output == "Done"
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in model_input].count("Late input") == 1
|
||||
|
||||
|
||||
@@ -367,13 +368,15 @@ async def test_interruption_without_guaranteed_next_model_rejects_input(
|
||||
def protected_tool(value: str) -> str:
|
||||
return value
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call(
|
||||
"protected_tool",
|
||||
'{"value":"one"}',
|
||||
call_id="call-protected-terminal",
|
||||
)
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call(
|
||||
"protected_tool",
|
||||
'{"value":"one"}',
|
||||
call_id="call-protected-terminal",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
agent = Agent(
|
||||
@@ -412,13 +415,13 @@ async def test_pending_input_guardrail_trip_keeps_input_recoverable() -> None:
|
||||
|
||||
agent.input_guardrails = [InputGuardrail(guardrail_function=trip_pending_input)]
|
||||
state.add_input("Unsafe late input")
|
||||
model.set_next_output([get_text_message("Must not run")])
|
||||
queued_outputs = len(model.turn_outputs)
|
||||
model.enqueue([get_text_message("Must not run")])
|
||||
queued_outputs = model.remaining_steps
|
||||
|
||||
with pytest.raises(InputGuardrailTripwireTriggered):
|
||||
await Runner.run(agent, state)
|
||||
|
||||
assert len(model.turn_outputs) == queued_outputs
|
||||
assert model.remaining_steps == queued_outputs
|
||||
assert [[_message_text(item) for item in batch] for batch in guarded_inputs] == [
|
||||
["Unsafe late input"]
|
||||
]
|
||||
@@ -453,7 +456,7 @@ async def test_pending_input_runs_agent_and_run_config_guardrails_on_only_pendin
|
||||
input_guardrails=[InputGuardrail(guardrail_function=inspect_config_input)]
|
||||
)
|
||||
state.add_input("Guard only this")
|
||||
model.set_next_output([get_text_message("Done")])
|
||||
model.enqueue([get_text_message("Done")])
|
||||
|
||||
result = await Runner.run(agent, state, run_config=run_config)
|
||||
|
||||
@@ -494,7 +497,7 @@ async def test_guardrail_retry_persists_successful_turn_with_session(
|
||||
await Runner.run(agent, state, session=session)
|
||||
|
||||
should_trip = False
|
||||
model.set_next_output([get_text_message("Recovered")])
|
||||
model.enqueue([get_text_message("Recovered")])
|
||||
if streamed_retry:
|
||||
streamed_result = Runner.run_streamed(agent, state, session=session)
|
||||
async for _event in streamed_result.stream_events():
|
||||
@@ -519,7 +522,7 @@ async def test_guardrail_retry_persists_successful_turn_with_session(
|
||||
async def test_failed_model_request_does_not_duplicate_admitted_input_on_resume() -> None:
|
||||
model, agent, state, _calls = await _make_after_turn_state()
|
||||
state.add_input("Late input")
|
||||
model.set_next_output(RuntimeError("model failed"))
|
||||
model.enqueue(RuntimeError("model failed"))
|
||||
|
||||
with pytest.raises(RuntimeError, match="model failed"):
|
||||
await Runner.run(agent, state)
|
||||
@@ -534,10 +537,10 @@ async def test_failed_model_request_does_not_duplicate_admitted_input_on_resume(
|
||||
next(item.input_id for item in state._generated_items if isinstance(item, InputItem))
|
||||
== admitted_input_id
|
||||
)
|
||||
model.set_next_output([get_text_message("Recovered")])
|
||||
model.enqueue([get_text_message("Recovered")])
|
||||
result = await Runner.run(agent, state)
|
||||
assert result.final_output == "Recovered"
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in model_input].count("Late input") == 1
|
||||
|
||||
|
||||
@@ -546,7 +549,7 @@ async def test_failed_model_request_with_session_persists_admitted_input_once()
|
||||
session = SimpleListSession()
|
||||
model, agent, state, _calls = await _make_after_turn_state(session=session)
|
||||
state.add_input("Late input")
|
||||
model.set_next_output(RuntimeError("model failed"))
|
||||
model.enqueue(RuntimeError("model failed"))
|
||||
|
||||
with pytest.raises(RuntimeError, match="model failed"):
|
||||
await Runner.run(agent, state, session=session)
|
||||
@@ -555,10 +558,10 @@ async def test_failed_model_request_with_session_persists_admitted_input_once()
|
||||
assert [_message_text(item) for item in await session.get_items()].count("Late input") == 1
|
||||
|
||||
state = await RunState.from_json(agent, state.to_json())
|
||||
model.set_next_output([get_text_message("Recovered")])
|
||||
model.enqueue([get_text_message("Recovered")])
|
||||
result = await Runner.run(agent, state, session=session)
|
||||
assert result.final_output == "Recovered"
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in model_input].count("Late input") == 1
|
||||
assert [_message_text(item) for item in await session.get_items()].count("Late input") == 1
|
||||
|
||||
@@ -567,17 +570,17 @@ async def test_failed_model_request_with_session_persists_admitted_input_once()
|
||||
async def test_failed_server_managed_request_keeps_pending_input_for_retry() -> None:
|
||||
model, agent, state, _calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Late input")
|
||||
model.set_next_output(RuntimeError("model failed"))
|
||||
model.enqueue(RuntimeError("model failed"))
|
||||
|
||||
with pytest.raises(RuntimeError, match="model failed"):
|
||||
await Runner.run(agent, state)
|
||||
|
||||
assert _message_text(state.pending_input[0]) == "Late input"
|
||||
state = await RunState.from_json(agent, state.to_json())
|
||||
model.set_next_output([get_text_message("Recovered")])
|
||||
model.enqueue([get_text_message("Recovered")])
|
||||
result = await Runner.run(agent, state)
|
||||
assert result.final_output == "Recovered"
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in model_input].count("Late input") == 1
|
||||
assert state.pending_input == []
|
||||
|
||||
@@ -586,7 +589,7 @@ async def test_failed_server_managed_request_keeps_pending_input_for_retry() ->
|
||||
async def test_server_filter_omission_remains_pending_for_later_nonstream_turn() -> None:
|
||||
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Late input")
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -616,7 +619,7 @@ async def test_server_filter_omission_remains_pending_for_later_nonstream_turn()
|
||||
|
||||
assert result.final_output == "Done"
|
||||
assert calls == ["Paris", "Rome"]
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in model_input].count("Late input") == 1
|
||||
assert state.pending_input == []
|
||||
|
||||
@@ -625,7 +628,7 @@ async def test_server_filter_omission_remains_pending_for_later_nonstream_turn()
|
||||
async def test_server_filter_omission_survives_streamed_state_round_trip() -> None:
|
||||
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Late input")
|
||||
model.set_next_output(
|
||||
model.enqueue(
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_destination",
|
||||
@@ -651,11 +654,11 @@ async def test_server_filter_omission_survives_streamed_state_round_trip() -> No
|
||||
assert [_message_text(item) for item in state.pending_input] == ["Late input"]
|
||||
assert not any(isinstance(item, InputItem) for item in state._generated_items)
|
||||
|
||||
model.set_next_output([get_text_message("Done")])
|
||||
model.enqueue([get_text_message("Done")])
|
||||
result = await Runner.run(agent, state)
|
||||
assert result.final_output == "Done"
|
||||
assert calls == ["Paris", "Rome"]
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in model_input].count("Late input") == 1
|
||||
|
||||
|
||||
@@ -664,7 +667,7 @@ async def test_server_filter_omission_survives_streamed_state_round_trip() -> No
|
||||
async def test_server_filter_reconstructed_pending_rewrite_is_rejected(streamed: bool) -> None:
|
||||
model, agent, state, _calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Late input")
|
||||
model.set_next_output([get_text_message("Done")])
|
||||
model.enqueue([get_text_message("Done")])
|
||||
|
||||
def reconstruct_pending(data: CallModelData[Any]) -> ModelInputData:
|
||||
rewritten = [
|
||||
@@ -678,7 +681,7 @@ async def test_server_filter_reconstructed_pending_rewrite_is_rejected(streamed:
|
||||
instructions=data.model_data.instructions,
|
||||
)
|
||||
|
||||
queued_outputs = len(model.turn_outputs)
|
||||
queued_outputs = model.remaining_steps
|
||||
run_config = RunConfig(call_model_input_filter=reconstruct_pending)
|
||||
if streamed:
|
||||
failed = Runner.run_streamed(agent, state, run_config=run_config)
|
||||
@@ -689,7 +692,7 @@ async def test_server_filter_reconstructed_pending_rewrite_is_rejected(streamed:
|
||||
with pytest.raises(UserError, match="cannot safely associate"):
|
||||
await Runner.run(agent, state, run_config=run_config)
|
||||
|
||||
assert len(model.turn_outputs) == queued_outputs
|
||||
assert model.remaining_steps == queued_outputs
|
||||
assert [_message_text(item) for item in state.pending_input] == ["Late input"]
|
||||
|
||||
|
||||
@@ -697,7 +700,7 @@ async def test_server_filter_reconstructed_pending_rewrite_is_rejected(streamed:
|
||||
async def test_server_filter_in_place_pending_rewrite_preserves_occurrence() -> None:
|
||||
model, agent, state, _calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Late input")
|
||||
model.set_next_output([get_text_message("Done")])
|
||||
model.enqueue([get_text_message("Done")])
|
||||
|
||||
def rewrite_pending_in_place(data: CallModelData[Any]) -> ModelInputData:
|
||||
for item in data.model_data.input:
|
||||
@@ -712,7 +715,7 @@ async def test_server_filter_in_place_pending_rewrite_preserves_occurrence() ->
|
||||
)
|
||||
|
||||
assert result.final_output == "Done"
|
||||
model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in model_input].count("Filtered late input") == 1
|
||||
assert state.pending_input == []
|
||||
|
||||
@@ -747,7 +750,7 @@ async def test_server_response_acceptance_commits_before_hook_failure(
|
||||
agent_hooks = CountAgentResponseHook()
|
||||
agent.hooks = agent_hooks
|
||||
state.add_input("Late input")
|
||||
model.set_next_output([get_text_message("Accepted")])
|
||||
model.enqueue([get_text_message("Accepted")])
|
||||
|
||||
if streamed_failure:
|
||||
failed = Runner.run_streamed(agent, state, hooks=FailAfterResponse())
|
||||
@@ -758,7 +761,7 @@ async def test_server_response_acceptance_commits_before_hook_failure(
|
||||
with pytest.raises(RuntimeError, match="after response"):
|
||||
await Runner.run(agent, state, hooks=FailAfterResponse())
|
||||
|
||||
accepted_model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
accepted_model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in accepted_model_input].count("Late input") == 1
|
||||
assert state.pending_input == []
|
||||
assert isinstance(state._current_step, NextStepInterruption)
|
||||
@@ -766,12 +769,12 @@ async def test_server_response_acceptance_commits_before_hook_failure(
|
||||
assert state._current_step.llm_end_hooks_started
|
||||
assert agent_hooks.call_count == 1
|
||||
state = await RunState.from_json(agent, state.to_json())
|
||||
queued_outputs = len(model.turn_outputs)
|
||||
queued_outputs = model.remaining_steps
|
||||
|
||||
recovered = await Runner.run(agent, state)
|
||||
assert recovered.final_output == "Accepted"
|
||||
assert agent_hooks.call_count == 1
|
||||
assert len(model.turn_outputs) == queued_outputs
|
||||
assert model.remaining_steps == queued_outputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -781,7 +784,7 @@ async def test_server_acceptance_commits_before_invocation_validation_failure(
|
||||
) -> None:
|
||||
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Late input")
|
||||
model.set_next_output(
|
||||
model.enqueue(
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_destination",
|
||||
@@ -800,7 +803,7 @@ async def test_server_acceptance_commits_before_invocation_validation_failure(
|
||||
with pytest.raises(ModelBehaviorError, match="completed tool call ID"):
|
||||
await Runner.run(agent, state)
|
||||
|
||||
accepted_model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
accepted_model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in accepted_model_input].count("Late input") == 1
|
||||
assert state.pending_input == []
|
||||
assert isinstance(state._current_step, NextStepInterruption)
|
||||
@@ -809,10 +812,10 @@ async def test_server_acceptance_commits_before_invocation_validation_failure(
|
||||
assert calls == ["Paris"]
|
||||
|
||||
state = await RunState.from_json(agent, state.to_json())
|
||||
queued_outputs = len(model.turn_outputs)
|
||||
queued_outputs = model.remaining_steps
|
||||
with pytest.raises(UserError, match="accepted model response could not be processed"):
|
||||
await Runner.run(agent, state)
|
||||
assert len(model.turn_outputs) == queued_outputs
|
||||
assert model.remaining_steps == queued_outputs
|
||||
assert calls == ["Paris"]
|
||||
|
||||
|
||||
@@ -845,18 +848,17 @@ async def test_server_accepted_computer_start_hook_failure_is_not_replayed(
|
||||
model, agent, state, _calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
agent.tools = [ComputerTool(computer=RecordingComputer())]
|
||||
state.add_input("Late input")
|
||||
model.set_next_output(
|
||||
[
|
||||
ResponseComputerToolCall(
|
||||
id="computer-item",
|
||||
type="computer_call",
|
||||
action=ActionScreenshot(type="screenshot"),
|
||||
call_id="computer-call",
|
||||
pending_safety_checks=[],
|
||||
status="completed",
|
||||
)
|
||||
]
|
||||
)
|
||||
output = [
|
||||
ResponseComputerToolCall(
|
||||
id="computer-item",
|
||||
type="computer_call",
|
||||
action=ActionScreenshot(type="screenshot"),
|
||||
call_id="computer-call",
|
||||
pending_safety_checks=[],
|
||||
status="completed",
|
||||
)
|
||||
]
|
||||
model.enqueue(get_exact_output_stream_step(output) if streamed_failure else output)
|
||||
hooks = FailComputerStart()
|
||||
|
||||
if streamed_failure:
|
||||
@@ -909,7 +911,7 @@ async def test_server_accepted_tool_side_effect_failure_is_safe(
|
||||
|
||||
model, agent, state, calls = await _make_after_turn_state(auto_previous_response_id=True)
|
||||
state.add_input("Late input")
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -947,13 +949,13 @@ async def test_server_accepted_tool_side_effect_failure_is_safe(
|
||||
recovered = await Runner.run(agent, state)
|
||||
assert recovered.final_output == "Recovered"
|
||||
assert calls == ["Paris", "Rome"]
|
||||
retry_model_input = cast(list[TResponseInputItem], model.last_turn_args["input"])
|
||||
retry_model_input = cast(list[TResponseInputItem], model.calls[-1].input)
|
||||
assert [_message_text(item) for item in retry_model_input].count("Late input") == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_terminal_state_rejects_pending_input_without_mutation() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("Done")])
|
||||
model = ScriptedModel(steps=[[get_text_message("Done")]])
|
||||
agent = Agent(name="assistant", model=model)
|
||||
result = await Runner.run(agent, "Initial request")
|
||||
state = result.to_state()
|
||||
|
||||
@@ -15,6 +15,7 @@ from agents.run_internal.run_steps import (
|
||||
SingleStepResult,
|
||||
)
|
||||
from agents.run_state import RunState
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool_guardrails import (
|
||||
AllowBehavior,
|
||||
ToolGuardrailFunctionOutput,
|
||||
@@ -24,12 +25,11 @@ from agents.tool_guardrails import (
|
||||
ToolOutputGuardrailResult,
|
||||
)
|
||||
from agents.usage import Usage
|
||||
from tests.fake_model import FakeModel
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_resume_preserves_guardrail_results(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
agent = Agent(name="agent", model=FakeModel())
|
||||
agent = Agent(name="agent", model=ScriptedModel())
|
||||
context_wrapper: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
|
||||
|
||||
input_guardrail: InputGuardrail[Any] = InputGuardrail(
|
||||
@@ -163,7 +163,7 @@ async def test_runner_resume_preserves_guardrail_results_on_reinterruption(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A resumed run that interrupts again must keep the tool guardrail results it carried in."""
|
||||
agent = Agent(name="agent", model=FakeModel())
|
||||
agent = Agent(name="agent", model=ScriptedModel())
|
||||
context_wrapper: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
|
||||
|
||||
tool_input_guardrail: ToolInputGuardrail[Any] = ToolInputGuardrail(
|
||||
|
||||
@@ -8,9 +8,9 @@ from openai.types.responses.response_usage import InputTokensDetails, OutputToke
|
||||
from agents import Agent, Runner, Tool, Usage
|
||||
from agents.items import ToolApprovalItem
|
||||
from agents.result import RunResult, RunResultStreaming
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.usage import serialize_usage
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_function_tool, get_function_tool_call, get_text_message
|
||||
from .testing_processor import SPAN_PROCESSOR_TESTING, fetch_normalized_spans
|
||||
from .utils.simple_session import SimpleListSession
|
||||
@@ -97,12 +97,12 @@ async def _run(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streamed", [False, True])
|
||||
async def test_fake_model_records_every_model_visible_request_field(streamed: bool) -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("READY")])
|
||||
async def test_scripted_model_records_every_model_visible_request_field(streamed: bool) -> None:
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("READY")])
|
||||
await _run(Agent(name="request-contract-agent", model=model), streamed=streamed)
|
||||
|
||||
assert set(model.last_turn_args) == {
|
||||
assert set(model.calls[-1].__dataclass_fields__) == {
|
||||
"system_instructions",
|
||||
"input",
|
||||
"model_settings",
|
||||
@@ -113,6 +113,7 @@ async def test_fake_model_records_every_model_visible_request_field(streamed: bo
|
||||
"previous_response_id",
|
||||
"conversation_id",
|
||||
"prompt",
|
||||
"streamed",
|
||||
}
|
||||
|
||||
|
||||
@@ -121,11 +122,11 @@ async def test_streamed_and_nonstreamed_runs_have_matching_semantics(scenario: s
|
||||
projections: list[dict[str, Any]] = []
|
||||
for streamed in (False, True):
|
||||
SPAN_PROCESSOR_TESTING.clear()
|
||||
model = FakeModel(tracing_enabled=True)
|
||||
model.set_hardcoded_usage(_detailed_usage())
|
||||
model = ScriptedModel(emit_traces=True)
|
||||
model.set_default_usage(_detailed_usage())
|
||||
tools: list[Tool] = []
|
||||
if scenario == "function-tool":
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("release_check", "{}", call_id="call-release")],
|
||||
[get_text_message("READY")],
|
||||
@@ -133,7 +134,7 @@ async def test_streamed_and_nonstreamed_runs_have_matching_semantics(scenario: s
|
||||
)
|
||||
tools = [get_function_tool("release_check", "checked")]
|
||||
else:
|
||||
model.set_next_output([get_text_message("READY")])
|
||||
model.enqueue([get_text_message("READY")])
|
||||
agent = Agent(name="symmetry-agent", model=model, tools=tools)
|
||||
session = SimpleListSession(session_id=f"{scenario}-{streamed}")
|
||||
result = await _run(agent, streamed=streamed, session=session)
|
||||
@@ -155,8 +156,8 @@ async def test_streamed_and_nonstreamed_runs_have_matching_semantics(scenario: s
|
||||
async def test_streamed_and_nonstreamed_runs_raise_the_same_exception_class() -> None:
|
||||
exception_classes: list[type[BaseException]] = []
|
||||
for streamed in (False, True):
|
||||
model = FakeModel()
|
||||
model.set_next_output(RuntimeError("release contract failure"))
|
||||
model = ScriptedModel()
|
||||
model.enqueue(RuntimeError("release contract failure"))
|
||||
agent = Agent(name="symmetry-agent", model=model)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
@@ -170,9 +171,9 @@ async def test_approval_resume_cross_modes_have_matching_semantics() -> None:
|
||||
projections: list[dict[str, Any]] = []
|
||||
for start_streamed, resume_streamed in ((True, False), (False, True)):
|
||||
SPAN_PROCESSOR_TESTING.clear()
|
||||
model = FakeModel(tracing_enabled=True)
|
||||
model.set_hardcoded_usage(_detailed_usage())
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel(emit_traces=True)
|
||||
model.set_default_usage(_detailed_usage())
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("release_check", "{}", call_id="call-release")],
|
||||
[get_text_message("READY")],
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,559 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import RunConfig, Runner
|
||||
from agents.sandbox import ExecResult, Manifest, SandboxAgent
|
||||
from agents.sandbox.capabilities import Shell
|
||||
from agents.sandbox.files import FileEntry
|
||||
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
|
||||
from agents.sandbox.session.pty_types import PtyExecUpdate
|
||||
from agents.testing import (
|
||||
InvalidSandboxStep,
|
||||
SandboxCall,
|
||||
SandboxCallMatcherError,
|
||||
ScriptedModel,
|
||||
UnconsumedSandboxSteps,
|
||||
UnexpectedSandboxCall,
|
||||
assistant_message,
|
||||
function_call,
|
||||
scripted_sandbox_session,
|
||||
)
|
||||
|
||||
|
||||
class _CallableBytesIO(io.BytesIO):
|
||||
def __call__(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _CallableBufferedReader(io.BufferedReader):
|
||||
def __call__(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def test_scripted_sandbox_exposes_only_configured_scriptable_methods() -> None:
|
||||
session = scripted_sandbox_session(
|
||||
[{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)}]
|
||||
)
|
||||
|
||||
assert isinstance(session, BaseSandboxSession)
|
||||
assert hasattr(session, "exec")
|
||||
assert not hasattr(session, "read")
|
||||
assert not hasattr(session, "apply_patch")
|
||||
assert not hasattr(session, "pty_exec_start")
|
||||
assert "exec" in dir(session)
|
||||
assert "read" not in dir(session)
|
||||
assert "apply_patch" not in dir(session)
|
||||
assert "pty_exec_start" not in dir(session)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_cleanup_does_not_advertise_pty_termination() -> None:
|
||||
session = scripted_sandbox_session()
|
||||
|
||||
assert not hasattr(session, "pty_terminate_all")
|
||||
assert "pty_terminate_all" not in dir(session)
|
||||
|
||||
await session.aclose()
|
||||
|
||||
async with scripted_sandbox_session() as context_session:
|
||||
assert await context_session.running() is True
|
||||
|
||||
assert await context_session.running() is False
|
||||
|
||||
|
||||
def test_scripted_sandbox_snapshots_manifest_and_derives_pty_support() -> None:
|
||||
manifest = Manifest(root="/configured")
|
||||
session = scripted_sandbox_session(
|
||||
[{"method": "pty_exec_start", "result": None}],
|
||||
manifest=manifest,
|
||||
)
|
||||
manifest.root = "/mutated"
|
||||
|
||||
assert session.state.manifest.root == "/configured"
|
||||
assert session.supports_pty() is True
|
||||
|
||||
pty_session = scripted_sandbox_session(
|
||||
[
|
||||
{"method": "pty_exec_start", "result": None},
|
||||
{"method": "pty_write_stdin", "result": None},
|
||||
]
|
||||
)
|
||||
assert pty_session.supports_pty() is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("configured_method", "missing_method"),
|
||||
[
|
||||
("pty_exec_start", "pty_write_stdin"),
|
||||
("pty_write_stdin", "pty_exec_start"),
|
||||
],
|
||||
)
|
||||
async def test_scripted_sandbox_exposes_pty_methods_as_one_capability(
|
||||
configured_method: str,
|
||||
missing_method: str,
|
||||
) -> None:
|
||||
session = scripted_sandbox_session([{"method": configured_method, "result": None}])
|
||||
|
||||
assert session.supports_pty() is True
|
||||
assert hasattr(session, "pty_exec_start")
|
||||
assert hasattr(session, "pty_write_stdin")
|
||||
assert "pty_exec_start" in dir(session)
|
||||
assert "pty_write_stdin" in dir(session)
|
||||
|
||||
with pytest.raises(UnexpectedSandboxCall) as exc_info:
|
||||
if missing_method == "pty_exec_start":
|
||||
await session.pty_exec_start("pwd")
|
||||
else:
|
||||
await session.pty_write_stdin(session_id=1, chars="")
|
||||
|
||||
assert exc_info.value.actual_method == missing_method
|
||||
assert exc_info.value.expected_method == configured_method
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_supports_capability_method_inventory() -> None:
|
||||
read_result = io.BytesIO(b"contents")
|
||||
list_result: list[FileEntry] = []
|
||||
pty_start_result = PtyExecUpdate(
|
||||
process_id=123,
|
||||
output=b"started",
|
||||
exit_code=None,
|
||||
original_token_count=None,
|
||||
)
|
||||
pty_write_result = PtyExecUpdate(
|
||||
process_id=None,
|
||||
output=b"done",
|
||||
exit_code=0,
|
||||
original_token_count=None,
|
||||
)
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{"method": "read", "result": read_result},
|
||||
{"method": "write", "result": None},
|
||||
{"method": "ls", "result": list_result},
|
||||
{"method": "mkdir", "result": None},
|
||||
{"method": "rm", "result": None},
|
||||
{"method": "apply_patch", "result": "Done!"},
|
||||
{"method": "pty_exec_start", "result": pty_start_result},
|
||||
{"method": "pty_write_stdin", "result": pty_write_result},
|
||||
]
|
||||
)
|
||||
stream = io.BytesIO(b"payload")
|
||||
|
||||
returned_read_result = cast(io.BytesIO, await session.read(Path("in.txt")))
|
||||
assert returned_read_result is not read_result
|
||||
assert returned_read_result.getvalue() == b"contents"
|
||||
await session.write(Path("out.txt"), stream)
|
||||
assert await session.ls(".") == []
|
||||
await session.mkdir("new", parents=True)
|
||||
await session.rm("old", recursive=True)
|
||||
assert await session.apply_patch({"type": "delete_file", "path": "old.txt"}) == "Done!"
|
||||
assert (await session.pty_exec_start("sh", tty=True)).process_id == 123
|
||||
assert (await session.pty_write_stdin(session_id=123, chars="exit")).exit_code == 0
|
||||
|
||||
assert [call.method for call in session.calls] == [
|
||||
"read",
|
||||
"write",
|
||||
"ls",
|
||||
"mkdir",
|
||||
"rm",
|
||||
"apply_patch",
|
||||
"pty_exec_start",
|
||||
"pty_write_stdin",
|
||||
]
|
||||
recorded_stream = cast(io.BytesIO, session.calls[1].args[1])
|
||||
assert recorded_stream is not stream
|
||||
assert recorded_stream.getvalue() == b"payload"
|
||||
assert recorded_stream.tell() == 0
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "original_content"),
|
||||
[(io.BytesIO(b"before"), b"before"), (io.StringIO("before"), "before")],
|
||||
)
|
||||
async def test_scripted_sandbox_snapshots_supported_stream_results(
|
||||
configured: io.BytesIO | io.StringIO,
|
||||
original_content: bytes | str,
|
||||
) -> None:
|
||||
configured.seek(2)
|
||||
session = scripted_sandbox_session([{"method": "read", "result": configured}])
|
||||
configured.seek(0)
|
||||
configured.write(b"after" if isinstance(configured, io.BytesIO) else "after")
|
||||
configured.close()
|
||||
|
||||
result = cast(io.BytesIO | io.StringIO, await session.read(Path("input.txt")))
|
||||
|
||||
assert result is not configured
|
||||
assert result.tell() == 2
|
||||
assert result.getvalue() == original_content
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_snapshots_supported_stream_call_arguments() -> None:
|
||||
session = scripted_sandbox_session([{"method": "write", "result": None}])
|
||||
source = io.BytesIO(b"before")
|
||||
source.seek(3)
|
||||
|
||||
await session.write(Path("output.bin"), source)
|
||||
source.seek(0)
|
||||
source.write(b"after")
|
||||
source.close()
|
||||
|
||||
recorded = cast(io.BytesIO, session.calls[0].args[1])
|
||||
assert recorded is not source
|
||||
assert recorded.tell() == 3
|
||||
assert recorded.getvalue() == b"before"
|
||||
recorded.write(b"changed")
|
||||
retained = cast(io.BytesIO, session.calls[0].args[1])
|
||||
assert retained.tell() == 3
|
||||
assert retained.getvalue() == b"before"
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_snapshots_callable_supported_streams() -> None:
|
||||
result_source = _CallableBytesIO(b"result")
|
||||
call_source = _CallableBytesIO(b"call")
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{"method": "read", "result": result_source},
|
||||
{"method": "write", "result": None},
|
||||
]
|
||||
)
|
||||
|
||||
result = cast(io.BytesIO, await session.read(Path("input.bin")))
|
||||
await session.write(Path("output.bin"), call_source)
|
||||
|
||||
assert result is not result_source
|
||||
assert result.getvalue() == b"result"
|
||||
recorded = cast(io.BytesIO, session.calls[1].args[1])
|
||||
assert recorded is not call_source
|
||||
assert recorded.getvalue() == b"call"
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
def test_scripted_sandbox_rejects_unsupported_or_closed_stream_results() -> None:
|
||||
with io.BufferedReader(io.BytesIO(b"payload")) as unsupported:
|
||||
with pytest.raises(InvalidSandboxStep) as unsupported_info:
|
||||
scripted_sandbox_session([{"method": "read", "result": unsupported}])
|
||||
assert unsupported_info.value.reason == "invalid_outcome"
|
||||
|
||||
closed = io.BytesIO(b"payload")
|
||||
closed.close()
|
||||
with pytest.raises(InvalidSandboxStep) as closed_info:
|
||||
scripted_sandbox_session([{"method": "read", "result": closed}])
|
||||
assert closed_info.value.reason == "invalid_outcome"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_rejects_unsupported_stream_call_before_commit() -> None:
|
||||
session = scripted_sandbox_session([{"method": "write", "result": None}])
|
||||
|
||||
with io.BufferedReader(io.BytesIO(b"payload")) as unsupported:
|
||||
with pytest.raises(TypeError, match="support only io.BytesIO and io.StringIO"):
|
||||
await session.write(Path("output.bin"), unsupported)
|
||||
|
||||
assert session.calls == ()
|
||||
assert session.remaining_steps == 1
|
||||
|
||||
|
||||
def test_scripted_sandbox_rejects_callable_unsupported_stream_result() -> None:
|
||||
with _CallableBufferedReader(io.BytesIO(b"payload")) as unsupported:
|
||||
with pytest.raises(InvalidSandboxStep) as exc_info:
|
||||
scripted_sandbox_session([{"method": "read", "result": unsupported}])
|
||||
|
||||
assert exc_info.value.reason == "invalid_outcome"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stream_factory",
|
||||
[
|
||||
lambda: _CallableBytesIO(b"payload"),
|
||||
lambda: _CallableBufferedReader(io.BytesIO(b"payload")),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("field", "reason"),
|
||||
[("match", "invalid_matcher"), ("responder", "invalid_outcome")],
|
||||
)
|
||||
def test_scripted_sandbox_rejects_callable_stream_matchers_and_responders(
|
||||
stream_factory: Any,
|
||||
field: str,
|
||||
reason: str,
|
||||
) -> None:
|
||||
stream = cast(io.IOBase, stream_factory())
|
||||
step: dict[str, Any] = {"method": "exec", field: stream}
|
||||
if field == "match":
|
||||
step["result"] = ExecResult(stdout=b"", stderr=b"", exit_code=0)
|
||||
|
||||
try:
|
||||
with pytest.raises(InvalidSandboxStep) as exc_info:
|
||||
scripted_sandbox_session([step])
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
assert exc_info.value.reason == reason
|
||||
assert exc_info.value.input_index == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_rejects_callable_unsupported_stream_call_before_commit() -> None:
|
||||
session = scripted_sandbox_session([{"method": "write", "result": None}])
|
||||
|
||||
with _CallableBufferedReader(io.BytesIO(b"payload")) as unsupported:
|
||||
with pytest.raises(TypeError, match="support only io.BytesIO and io.StringIO"):
|
||||
await session.write(Path("output.bin"), unsupported)
|
||||
|
||||
assert session.calls == ()
|
||||
assert session.remaining_steps == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_snapshots_static_results_when_queued() -> None:
|
||||
configured = ExecResult(stdout=b"before", stderr=b"", exit_code=0)
|
||||
session = scripted_sandbox_session([{"method": "exec", "result": configured}])
|
||||
configured.stdout = b"after"
|
||||
|
||||
result = await session.exec("pwd")
|
||||
|
||||
assert result.stdout == b"before"
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_records_detached_fifo_calls() -> None:
|
||||
source_operations: list[dict[str, object]] = [{"type": "delete_file", "path": "before.txt"}]
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{"method": "apply_patch", "result": "Done!"},
|
||||
{"method": "exec", "result": ExecResult(stdout=b"ok", stderr=b"", exit_code=0)},
|
||||
]
|
||||
)
|
||||
|
||||
assert await session.apply_patch(cast(Any, source_operations)) == "Done!"
|
||||
source_operations[0]["path"] = "after.txt"
|
||||
result = await session.exec("pwd", shell=False)
|
||||
|
||||
assert result.stdout == b"ok"
|
||||
assert session.remaining_steps == 0
|
||||
session.assert_complete()
|
||||
assert session.calls[0].method == "apply_patch"
|
||||
assert session.calls[0].args[0] == [{"type": "delete_file", "path": "before.txt"}]
|
||||
assert session.calls[1] == SandboxCall(
|
||||
call_index=1,
|
||||
method="exec",
|
||||
args=("pwd",),
|
||||
kwargs=MappingProxyType({"timeout": None, "shell": False, "user": None}),
|
||||
)
|
||||
|
||||
returned_operations = cast(list[dict[str, object]], session.calls[0].args[0])
|
||||
returned_operations[0]["path"] = "mutated.txt"
|
||||
assert session.calls[0].args[0] == [{"type": "delete_file", "path": "before.txt"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_snapshot_failure_does_not_commit_call() -> None:
|
||||
class Uncopyable:
|
||||
def __deepcopy__(self, memo: dict[int, object]) -> object:
|
||||
_ = memo
|
||||
raise RuntimeError("cannot snapshot")
|
||||
|
||||
session = scripted_sandbox_session(
|
||||
[{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)}]
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="cannot snapshot"):
|
||||
await session.exec(cast(Any, Uncopyable()))
|
||||
|
||||
assert session.calls == ()
|
||||
assert session.remaining_steps == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_supports_matchers_responders_and_errors() -> None:
|
||||
injected_error = RuntimeError("sandbox unavailable")
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{
|
||||
"method": "exec",
|
||||
"match": lambda call: call.args == ("pwd",),
|
||||
"responder": lambda call: ExecResult(
|
||||
stdout=f"call {len(call.args)}".encode(), stderr=b"", exit_code=0
|
||||
),
|
||||
},
|
||||
{"method": "exec", "error": injected_error},
|
||||
]
|
||||
)
|
||||
|
||||
assert (await session.exec("pwd")).stdout == b"call 1"
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
await session.exec("next")
|
||||
assert exc_info.value is injected_error
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_reports_structured_payload_free_failures() -> None:
|
||||
mismatch = scripted_sandbox_session(
|
||||
[
|
||||
{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)},
|
||||
{"method": "read", "result": None},
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(UnexpectedSandboxCall) as mismatch_info:
|
||||
await mismatch.read(Path("/secret/payload.txt"))
|
||||
mismatch_error = mismatch_info.value
|
||||
assert mismatch_error.call_index == 0
|
||||
assert "call #1" in str(mismatch_error)
|
||||
assert mismatch_error.actual_method == "read"
|
||||
assert mismatch_error.expected_method == "exec"
|
||||
assert mismatch_error.remaining_steps == 2
|
||||
assert mismatch.remaining_steps == 2
|
||||
assert "/secret/payload.txt" not in str(mismatch_error)
|
||||
await mismatch.exec("retry")
|
||||
assert mismatch.remaining_steps == 1
|
||||
|
||||
rejected = scripted_sandbox_session(
|
||||
[
|
||||
{
|
||||
"method": "exec",
|
||||
"match": lambda call: call.args == ("expected",),
|
||||
"result": ExecResult(stdout=b"", stderr=b"", exit_code=0),
|
||||
}
|
||||
]
|
||||
)
|
||||
with pytest.raises(SandboxCallMatcherError) as rejected_info:
|
||||
await rejected.exec("secret command")
|
||||
assert rejected_info.value.call_index == 0
|
||||
assert "call #1" in str(rejected_info.value)
|
||||
assert rejected_info.value.method == "exec"
|
||||
assert "secret command" not in str(rejected_info.value)
|
||||
assert rejected.remaining_steps == 1
|
||||
await rejected.exec("expected")
|
||||
rejected.assert_complete()
|
||||
|
||||
extra = scripted_sandbox_session(
|
||||
[{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)}]
|
||||
)
|
||||
await extra.exec("first")
|
||||
assert hasattr(extra, "exec")
|
||||
with pytest.raises(UnexpectedSandboxCall) as extra_info:
|
||||
await extra.exec("second secret")
|
||||
assert extra_info.value.call_index == 1
|
||||
assert "call #2" in str(extra_info.value)
|
||||
assert extra_info.value.expected_method is None
|
||||
assert extra_info.value.remaining_steps == 0
|
||||
assert "second secret" not in str(extra_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_retains_step_after_matcher_exception() -> None:
|
||||
matcher_error = RuntimeError("matcher failed")
|
||||
matcher_calls = 0
|
||||
|
||||
def match(_call: SandboxCall) -> bool:
|
||||
nonlocal matcher_calls
|
||||
matcher_calls += 1
|
||||
if matcher_calls == 1:
|
||||
raise matcher_error
|
||||
return True
|
||||
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{
|
||||
"method": "exec",
|
||||
"match": match,
|
||||
"result": ExecResult(stdout=b"ok", stderr=b"", exit_code=0),
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
await session.exec("first")
|
||||
|
||||
assert exc_info.value is matcher_error
|
||||
assert session.remaining_steps == 1
|
||||
assert (await session.exec("retry")).stdout == b"ok"
|
||||
session.assert_complete()
|
||||
|
||||
|
||||
def test_scripted_sandbox_reports_unconsumed_steps() -> None:
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{"method": "exec", "result": ExecResult(stdout=b"", stderr=b"", exit_code=0)},
|
||||
{"method": "read", "result": None},
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(UnconsumedSandboxSteps) as exc_info:
|
||||
session.assert_complete()
|
||||
assert exc_info.value.remaining_steps == 2
|
||||
assert exc_info.value.pending_methods == ("exec", "read")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("step", "reason"),
|
||||
[
|
||||
("not a mapping", "invalid_input"),
|
||||
({"method": "exec", "matc": lambda _call: True, "result": None}, "invalid_input"),
|
||||
({"method": "unknown", "result": None}, "unknown_method"),
|
||||
({"method": "exec", "match": "no", "result": None}, "invalid_matcher"),
|
||||
({"method": "exec"}, "invalid_outcome"),
|
||||
({"method": "exec", "result": None, "error": RuntimeError()}, "invalid_outcome"),
|
||||
({"method": "exec", "responder": "no"}, "invalid_outcome"),
|
||||
({"method": "exec", "error": "no"}, "invalid_outcome"),
|
||||
],
|
||||
)
|
||||
def test_scripted_sandbox_validates_steps_before_use(step: object, reason: str) -> None:
|
||||
with pytest.raises(InvalidSandboxStep) as exc_info:
|
||||
scripted_sandbox_session(cast(Any, [step]))
|
||||
assert exc_info.value.reason == reason
|
||||
assert exc_info.value.input_index == 0
|
||||
assert "step #1" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_sandbox_drives_black_box_sandbox_agent_workflow() -> None:
|
||||
session = scripted_sandbox_session(
|
||||
[
|
||||
{
|
||||
"method": "exec",
|
||||
"match": lambda call: call.args == ("pwd",),
|
||||
"result": ExecResult(stdout=b"/workspace\n", stderr=b"", exit_code=0),
|
||||
}
|
||||
]
|
||||
)
|
||||
model = ScriptedModel(
|
||||
[
|
||||
[function_call("exec_command", {"cmd": "pwd"}, call_id="call_1")],
|
||||
[assistant_message("The workspace is /workspace.")],
|
||||
]
|
||||
)
|
||||
agent = SandboxAgent(
|
||||
name="Test agent",
|
||||
model=model,
|
||||
capabilities=[Shell()],
|
||||
)
|
||||
|
||||
result = await Runner.run(
|
||||
agent,
|
||||
"Where am I?",
|
||||
run_config=RunConfig(sandbox={"session": session}),
|
||||
)
|
||||
|
||||
assert result.final_output == "The workspace is /workspace."
|
||||
assert len(session.calls) == 1
|
||||
assert len(model.calls) == 2
|
||||
session.assert_complete()
|
||||
model.assert_complete()
|
||||
@@ -28,9 +28,10 @@ from agents.run_internal.run_loop import get_new_response, run_single_turn_strea
|
||||
from agents.run_internal.run_steps import NextStepInterruption
|
||||
from agents.run_internal.tool_use_tracker import AgentToolUseTracker
|
||||
from agents.stream_events import RunItemStreamEvent
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.usage import Usage
|
||||
from tests.model_test_helpers import get_exact_output_stream_step
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_text_message
|
||||
|
||||
|
||||
@@ -815,8 +816,8 @@ def test_prepare_input_does_not_resend_reasoning_item_after_marking_omitted_id_a
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_new_response_marks_filtered_input_as_sent() -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("ok")])
|
||||
agent = Agent(name="test", model=model)
|
||||
tracker = OpenAIServerConversationTracker(conversation_id="conv4", previous_response_id=None)
|
||||
context_wrapper: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
|
||||
@@ -848,15 +849,15 @@ async def test_get_new_response_marks_filtered_input_as_sent() -> None:
|
||||
None,
|
||||
)
|
||||
|
||||
assert model.last_turn_args["input"] == [item_1]
|
||||
assert model.calls[-1].input == [item_1]
|
||||
assert any(item is item_1 for item in tracker.sent_items)
|
||||
assert all(item is not item_2 for item in tracker.sent_items)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_single_turn_streamed_marks_filtered_input_as_sent() -> None:
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("ok")])
|
||||
agent = Agent(name="test", model=model)
|
||||
tracker = OpenAIServerConversationTracker(conversation_id="conv6", previous_response_id=None)
|
||||
context_wrapper: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
|
||||
@@ -903,13 +904,13 @@ async def test_run_single_turn_streamed_marks_filtered_input_as_sent() -> None:
|
||||
server_conversation_tracker=tracker,
|
||||
)
|
||||
|
||||
assert model.last_turn_args["input"] == [item_1]
|
||||
assert model.calls[-1].input == [item_1]
|
||||
assert tracker.remaining_initial_input == [item_2]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_single_turn_streamed_seeds_hosted_mcp_metadata_from_pre_step_items() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
mcp_call = McpCall(
|
||||
id="mcp_call_1",
|
||||
arguments="{}",
|
||||
@@ -918,7 +919,7 @@ async def test_run_single_turn_streamed_seeds_hosted_mcp_metadata_from_pre_step_
|
||||
type="mcp_call",
|
||||
status="completed",
|
||||
)
|
||||
model.set_next_output([mcp_call])
|
||||
model.enqueue(get_exact_output_stream_step([mcp_call]))
|
||||
agent = Agent(name="test", model=model)
|
||||
hosted_tool = HostedMCPTool(
|
||||
tool_config=cast(
|
||||
@@ -977,7 +978,7 @@ async def test_run_single_turn_streamed_seeds_hosted_mcp_metadata_from_pre_step_
|
||||
all_tools=[hosted_tool],
|
||||
)
|
||||
|
||||
assert model.last_turn_args["input"] == [item_1]
|
||||
assert model.calls[-1].input == [item_1]
|
||||
|
||||
tool_call_events: list[ToolCallItem] = []
|
||||
while not streamed_result._event_queue.empty():
|
||||
|
||||
@@ -6,8 +6,8 @@ from agents.agent import Agent
|
||||
from agents.exceptions import ModelBehaviorError
|
||||
from agents.items import ToolCallOutputItem
|
||||
from agents.run_internal import run_loop
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import ShellCallOutcome, ShellCommandOutput
|
||||
from tests.fake_model import FakeModel
|
||||
|
||||
|
||||
def test_coerce_shell_call_reads_max_output_length() -> None:
|
||||
@@ -122,7 +122,7 @@ def test_serialize_shell_output_emits_canonical_outcome() -> None:
|
||||
|
||||
|
||||
def test_shell_rejection_payload_preserves_missing_exit_code() -> None:
|
||||
agent = Agent(name="tester", model=FakeModel())
|
||||
agent = Agent(name="tester", model=ScriptedModel())
|
||||
raw_item = {
|
||||
"type": "shell_call_output",
|
||||
"call_id": "call-1",
|
||||
@@ -148,7 +148,7 @@ def test_shell_rejection_payload_preserves_missing_exit_code() -> None:
|
||||
|
||||
|
||||
def test_shell_output_preserves_zero_exit_code() -> None:
|
||||
agent = Agent(name="tester", model=FakeModel())
|
||||
agent = Agent(name="tester", model=ScriptedModel())
|
||||
raw_item = {
|
||||
"type": "shell_call_output",
|
||||
"call_id": "call-2",
|
||||
|
||||
+38
-38
@@ -10,8 +10,8 @@ import pytest
|
||||
from agents import Agent, Runner, SQLiteSession
|
||||
from agents.agent_output import AgentOutputSchema
|
||||
from agents.stream_events import StreamEvent
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import (
|
||||
get_function_tool,
|
||||
get_function_tool_call,
|
||||
@@ -23,7 +23,7 @@ from .test_responses import (
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_completes_turn():
|
||||
"""Verify soft cancel waits for turn to complete."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel([[]])
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -44,7 +44,7 @@ async def test_soft_cancel_completes_turn():
|
||||
async def test_soft_cancel_vs_immediate():
|
||||
"""Compare soft cancel vs immediate cancel behavior."""
|
||||
# Immediate cancel
|
||||
model1 = FakeModel()
|
||||
model1 = ScriptedModel([[]])
|
||||
agent1 = Agent(name="A1", model=model1)
|
||||
result1 = Runner.run_streamed(agent1, input="Hello")
|
||||
immediate_events = []
|
||||
@@ -54,7 +54,7 @@ async def test_soft_cancel_vs_immediate():
|
||||
result1.cancel(mode="immediate")
|
||||
|
||||
# Soft cancel
|
||||
model2 = FakeModel()
|
||||
model2 = ScriptedModel([[]])
|
||||
agent2 = Agent(name="A2", model=model2)
|
||||
result2 = Runner.run_streamed(agent2, input="Hello")
|
||||
soft_events = []
|
||||
@@ -72,14 +72,14 @@ async def test_soft_cancel_vs_immediate():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_with_tool_calls():
|
||||
"""Verify tool calls execute before soft cancel stops."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
model=model,
|
||||
tools=[get_function_tool("calc", "42")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_text_message("Let me calculate"),
|
||||
@@ -109,7 +109,7 @@ async def test_soft_cancel_with_tool_calls():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_saves_session():
|
||||
"""Verify session is saved properly with soft cancel."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel([[], []])
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
session = SQLiteSession("test_soft_cancel_session")
|
||||
@@ -136,7 +136,7 @@ async def test_soft_cancel_saves_session():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_tracks_usage():
|
||||
"""Verify usage is tracked for completed turn."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel([[]])
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -145,7 +145,7 @@ async def test_soft_cancel_tracks_usage():
|
||||
if event.type == "raw_response_event":
|
||||
result.cancel(mode="after_turn")
|
||||
|
||||
# Usage should be tracked (FakeModel tracks requests even if tokens are 0)
|
||||
# Usage should be tracked (ScriptedModel tracks requests even if tokens are 0)
|
||||
assert result.context_wrapper.usage.requests > 0
|
||||
|
||||
|
||||
@@ -153,7 +153,7 @@ async def test_soft_cancel_tracks_usage():
|
||||
@pytest.mark.parametrize("consumer_suspensions", [0, 1, 3])
|
||||
async def test_soft_cancel_stops_next_turn(consumer_suspensions: int):
|
||||
"""Verify soft cancel prevents next turn from starting."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
model=model,
|
||||
@@ -161,7 +161,7 @@ async def test_soft_cancel_stops_next_turn(consumer_suspensions: int):
|
||||
)
|
||||
|
||||
# Set up multi-turn scenario
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool1", "{}")],
|
||||
[get_text_message("Turn 2")],
|
||||
@@ -188,13 +188,13 @@ async def test_soft_cancel_stops_next_turn(consumer_suspensions: int):
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_stops_next_turn_with_short_lived_anext_tasks():
|
||||
"""Per-event tasks must not acknowledge a turn before the caller handles its event."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
model=model,
|
||||
tools=[get_function_tool("tool1", "result1")],
|
||||
)
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool1", "{}")],
|
||||
[get_text_message("Turn 2")],
|
||||
@@ -218,8 +218,8 @@ async def test_soft_cancel_stops_next_turn_with_short_lived_anext_tasks():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_run_completes_without_an_event_consumer():
|
||||
"""Turn acknowledgement must not block a run whose events are not consumed."""
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool1", "{}")],
|
||||
[get_text_message("Turn 2")],
|
||||
@@ -242,8 +242,8 @@ async def test_streamed_run_completes_without_an_event_consumer():
|
||||
@pytest.mark.asyncio
|
||||
async def test_closing_stream_consumer_releases_turn_acknowledgement():
|
||||
"""Closing an iterator must not deadlock while a turn awaits its consumer."""
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool1", "{}")],
|
||||
[get_text_message("Turn 2")],
|
||||
@@ -271,8 +271,8 @@ async def test_closing_stream_consumer_releases_turn_acknowledgement():
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_stream_consumer_releases_turn_acknowledgement():
|
||||
"""Cancelling a consumer suspended after yield must release the completed turn."""
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool1", "{}")],
|
||||
[get_text_message("Turn 2")],
|
||||
@@ -315,8 +315,8 @@ async def test_cancelled_stream_consumer_releases_turn_acknowledgement():
|
||||
@pytest.mark.asyncio
|
||||
async def test_immediate_cancel_releases_turn_acknowledgement():
|
||||
"""Immediate cancellation must cancel a run waiting for streamed event acknowledgement."""
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool1", "{}")],
|
||||
[get_text_message("Turn 2")],
|
||||
@@ -342,7 +342,7 @@ async def test_immediate_cancel_releases_turn_acknowledgement():
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_mode_backward_compatibility():
|
||||
"""Verify default behavior unchanged."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -363,7 +363,7 @@ async def test_cancel_mode_backward_compatibility():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_idempotent():
|
||||
"""Verify calling cancel multiple times is safe."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel([[]])
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -382,7 +382,7 @@ async def test_soft_cancel_idempotent():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_before_streaming():
|
||||
"""Verify soft cancel before streaming starts."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -398,7 +398,7 @@ async def test_soft_cancel_before_streaming():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_mixed_modes():
|
||||
"""Verify changing cancel mode behaves correctly."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -418,7 +418,7 @@ async def test_soft_cancel_mixed_modes():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_explicit_immediate_mode():
|
||||
"""Test explicit immediate mode behaves same as default."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -439,7 +439,7 @@ async def test_soft_cancel_explicit_immediate_mode():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_with_multiple_tool_calls():
|
||||
"""Verify soft cancel works with multiple tool calls in one turn."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
model=model,
|
||||
@@ -450,7 +450,7 @@ async def test_soft_cancel_with_multiple_tool_calls():
|
||||
)
|
||||
|
||||
# Turn with multiple tool calls
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call("tool1", "{}", call_id="tool_1"),
|
||||
@@ -479,14 +479,14 @@ async def test_soft_cancel_with_multiple_tool_calls():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_preserves_state():
|
||||
"""Verify soft cancel preserves all result state correctly."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
model=model,
|
||||
tools=[get_function_tool("tool1", "result")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool1", "{}")],
|
||||
[get_text_message("Done")],
|
||||
@@ -509,7 +509,7 @@ async def test_soft_cancel_preserves_state():
|
||||
@pytest.mark.asyncio
|
||||
async def test_immediate_cancel_clears_queues():
|
||||
"""Verify immediate cancel clears queues as expected."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -528,7 +528,7 @@ async def test_immediate_cancel_clears_queues():
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_does_not_clear_queues_immediately():
|
||||
"""Verify soft cancel does NOT clear queues immediately."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="Assistant", model=model)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Hello")
|
||||
@@ -551,7 +551,7 @@ async def test_soft_cancel_with_handoff():
|
||||
"""Verify soft cancel after handoff saves the handoff turn."""
|
||||
from agents import Handoff
|
||||
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
# Create two agents with handoff
|
||||
agent2 = Agent(name="Agent2", model=model)
|
||||
@@ -574,7 +574,7 @@ async def test_soft_cancel_with_handoff():
|
||||
)
|
||||
|
||||
# Setup: Agent1 does handoff, Agent2 responds
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# Agent1's turn - triggers handoff
|
||||
[get_function_tool_call(Handoff.default_tool_name(agent2), "{}")],
|
||||
@@ -610,7 +610,7 @@ async def test_soft_cancel_waits_for_handoff_event_consumption_before_next_turn(
|
||||
"""A suspended handoff consumer can stop the run before the delegate model starts."""
|
||||
second_request_started = asyncio.Event()
|
||||
|
||||
class HandoffModel(FakeModel):
|
||||
class HandoffModel(ScriptedModel):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_count = 0
|
||||
@@ -625,7 +625,7 @@ async def test_soft_cancel_waits_for_handoff_event_consumption_before_next_turn(
|
||||
model = HandoffModel()
|
||||
delegate = Agent(name="Delegate", model=model, output_type=int)
|
||||
triage = Agent(name="Triage", model=model, handoffs=[delegate])
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_handoff_tool_call(delegate)],
|
||||
[get_text_message("Delegate response")],
|
||||
@@ -666,7 +666,7 @@ async def test_soft_cancel_waits_for_handoff_event_consumption_before_next_turn(
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_cancel_with_session_and_multiple_turns():
|
||||
"""Verify soft cancel with session across multiple turns."""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
model=model,
|
||||
@@ -677,7 +677,7 @@ async def test_soft_cancel_with_session_and_multiple_turns():
|
||||
await session.clear_session()
|
||||
|
||||
# Setup 3 turns
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("tool1", "{}", call_id="tool_1")],
|
||||
[get_function_tool_call("tool1", "{}", call_id="tool_2")],
|
||||
|
||||
+43
-34
@@ -51,10 +51,11 @@ from agents.items import (
|
||||
ToolSearchOutputItem,
|
||||
)
|
||||
from agents.run_internal.streaming import stream_step_items_to_queue, stream_step_result_to_queue
|
||||
from agents.testing import ScriptedModel
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .mcp.helpers import FakeMCPServer
|
||||
from .mcp.model_compat import Tool as MCPTool
|
||||
from .model_test_helpers import get_exact_output_stream_step
|
||||
from .test_responses import get_function_tool_call, get_handoff_tool_call, get_text_message
|
||||
|
||||
|
||||
@@ -88,14 +89,14 @@ async def foo() -> str:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_events_main():
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="Joker",
|
||||
model=model,
|
||||
tools=[foo],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[
|
||||
@@ -127,7 +128,7 @@ async def test_stream_events_main():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_events_tool_called_includes_local_mcp_title() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
server = FakeMCPServer(
|
||||
tools=[
|
||||
MCPTool(
|
||||
@@ -140,7 +141,7 @@ async def test_stream_events_tool_called_includes_local_mcp_title() -> None:
|
||||
)
|
||||
agent = Agent(name="MCPAgent", model=model, mcp_servers=[server])
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("search_docs", "{}")],
|
||||
[get_text_message("done")],
|
||||
@@ -287,11 +288,11 @@ async def test_stream_events_main_with_handoff():
|
||||
english_agent = Agent(
|
||||
name="EnglishAgent",
|
||||
instructions="You only speak English.",
|
||||
model=FakeModel(),
|
||||
model=ScriptedModel([[]]),
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_text_message("Hello"),
|
||||
@@ -341,14 +342,14 @@ async def test_complete_streaming_events():
|
||||
- Function call with arguments delta/done events
|
||||
- Message output with content_part and text delta/done events
|
||||
"""
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="TestAgent",
|
||||
model=model,
|
||||
tools=[foo],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_reasoning_item(),
|
||||
@@ -481,8 +482,8 @@ async def test_complete_streaming_events():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_call_event_preserves_order_before_later_reasoning_item() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call("foo", '{"arg": "value"}'),
|
||||
@@ -511,12 +512,14 @@ async def test_tool_call_event_preserves_order_before_later_reasoning_item() ->
|
||||
async def test_handoff_event_preserves_order_before_later_reasoning_item() -> None:
|
||||
english_agent = Agent(
|
||||
name="EnglishAgent",
|
||||
model=FakeModel(initial_output=[get_text_message("Done")]),
|
||||
model=ScriptedModel(steps=[[get_text_message("Done")]]),
|
||||
)
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_handoff_tool_call(english_agent),
|
||||
get_reasoning_item(),
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_handoff_tool_call(english_agent),
|
||||
get_reasoning_item(),
|
||||
]
|
||||
]
|
||||
)
|
||||
triage_agent = Agent(name="TriageAgent", model=model, handoffs=[english_agent])
|
||||
@@ -542,12 +545,14 @@ async def test_handoff_filter_copy_does_not_duplicate_streamed_model_items() ->
|
||||
|
||||
english_agent = Agent(
|
||||
name="EnglishAgent",
|
||||
model=FakeModel(initial_output=[get_text_message("Done")]),
|
||||
model=ScriptedModel(steps=[[get_text_message("Done")]]),
|
||||
)
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_text_message("Transferring"),
|
||||
get_handoff_tool_call(english_agent),
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_text_message("Transferring"),
|
||||
get_handoff_tool_call(english_agent),
|
||||
]
|
||||
]
|
||||
)
|
||||
triage_agent = Agent(
|
||||
@@ -572,7 +577,7 @@ async def test_handoff_filter_copy_does_not_duplicate_streamed_model_items() ->
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_events_emit_tool_search_items() -> None:
|
||||
model = FakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(name="ToolSearchAgent", model=model)
|
||||
tool_search_call = cast(
|
||||
ResponseOutputItem,
|
||||
@@ -616,8 +621,12 @@ async def test_stream_events_emit_tool_search_items() -> None:
|
||||
},
|
||||
),
|
||||
)
|
||||
model.add_multiple_turn_outputs(
|
||||
[[tool_search_call, tool_search_output, get_text_message("Done")]]
|
||||
model.extend(
|
||||
[
|
||||
get_exact_output_stream_step(
|
||||
[tool_search_call, tool_search_output, get_text_message("Done")]
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result = Runner.run_streamed(agent, input="Search for CRM order tools")
|
||||
@@ -641,10 +650,10 @@ async def test_stream_events_emit_tool_search_items() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_handoff_call_is_not_emitted_as_tool_called():
|
||||
"""A handoff call streams only as `handoff_requested`, never also as `tool_called`."""
|
||||
english_agent = Agent(name="EnglishAgent", model=FakeModel())
|
||||
english_agent = Agent(name="EnglishAgent", model=ScriptedModel([[]]))
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_handoff_tool_call(english_agent)],
|
||||
[get_text_message("Done")],
|
||||
@@ -673,10 +682,10 @@ async def test_streamed_handoff_call_is_not_emitted_as_tool_called():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_tool_call_alongside_handoff_still_emits_tool_called():
|
||||
"""A real tool call in the same turn as a handoff keeps its `tool_called` event."""
|
||||
english_agent = Agent(name="EnglishAgent", model=FakeModel())
|
||||
english_agent = Agent(name="EnglishAgent", model=ScriptedModel([[]]))
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call("foo", '{"a": "b"}', call_id="tool_call"),
|
||||
@@ -712,10 +721,10 @@ async def test_streamed_tool_call_alongside_handoff_still_emits_tool_called():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_handoff_item_events_match_new_items():
|
||||
"""Streamed run item events stay in sync with the items recorded on the result."""
|
||||
english_agent = Agent(name="EnglishAgent", model=FakeModel())
|
||||
english_agent = Agent(name="EnglishAgent", model=ScriptedModel([[]]))
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_text_message("Transferring"), get_handoff_tool_call(english_agent)],
|
||||
[get_text_message("Done")],
|
||||
|
||||
@@ -10,7 +10,7 @@ from openai.types.responses import ResponseCompletedEvent
|
||||
from agents import Agent, GuardrailFunctionOutput, InputGuardrail, RunContextWrapper, Runner
|
||||
from agents.exceptions import InputGuardrailTripwireTriggered
|
||||
from agents.items import TResponseInputItem
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
from tests.testing_processor import fetch_events, fetch_ordered_spans
|
||||
|
||||
@@ -49,8 +49,8 @@ async def test_input_guardrail_results_follow_completion_order():
|
||||
output_info={"delay": FAST_GUARDRAIL_DELAY}, tripwire_triggered=False
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Final response")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Final response")])
|
||||
|
||||
agent = Agent(
|
||||
name="TimingAgentOrder",
|
||||
@@ -81,8 +81,8 @@ async def test_run_streamed_input_guardrail_timing_is_consistent(guardrail_delay
|
||||
"""
|
||||
|
||||
# Arrange: Agent with a single text output and a delayed input guardrail
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Final response")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Final response")])
|
||||
|
||||
agent = Agent(
|
||||
name="TimingAgent",
|
||||
@@ -122,8 +122,8 @@ async def test_run_streamed_input_guardrail_sequences_match_between_fast_and_slo
|
||||
"""Run twice with fast vs slow input guardrail and compare event sequences exactly."""
|
||||
|
||||
async def run_once(delay: float) -> list[str]:
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Final response")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Final response")])
|
||||
agent = Agent(
|
||||
name="TimingAgent",
|
||||
model=model,
|
||||
@@ -148,8 +148,8 @@ async def test_run_streamed_input_guardrail_sequences_match_between_fast_and_slo
|
||||
async def test_run_streamed_input_guardrail_tripwire_raises(guardrail_delay: float):
|
||||
"""Guardrail tripwire must raise from stream_events regardless of timing."""
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("Final response")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("Final response")])
|
||||
|
||||
agent = Agent(
|
||||
name="TimingAgentTrip",
|
||||
@@ -173,11 +173,11 @@ async def test_run_streamed_input_guardrail_tripwire_raises(guardrail_delay: flo
|
||||
)
|
||||
|
||||
|
||||
class SlowCompleteFakeModel(FakeModel):
|
||||
"""A FakeModel that delays just before emitting ResponseCompletedEvent in streaming."""
|
||||
class SlowCompleteScriptedModel(ScriptedModel):
|
||||
"""A ScriptedModel that delays just before emitting ResponseCompletedEvent in streaming."""
|
||||
|
||||
def __init__(self, delay_seconds: float, tracing_enabled: bool = True):
|
||||
super().__init__(tracing_enabled=tracing_enabled)
|
||||
def __init__(self, delay_seconds: float, emit_traces: bool = True):
|
||||
super().__init__(emit_traces=emit_traces)
|
||||
self._delay_seconds = delay_seconds
|
||||
|
||||
async def stream_response(self, *args, **kwargs):
|
||||
@@ -206,8 +206,8 @@ def _iso(s: str | None) -> datetime:
|
||||
async def test_parent_span_and_trace_finish_after_slow_input_guardrail():
|
||||
"""Agent span and trace finish after guardrail when guardrail completes last."""
|
||||
|
||||
model = FakeModel(tracing_enabled=True)
|
||||
model.set_next_output([get_text_message("Final response")])
|
||||
model = ScriptedModel(emit_traces=True)
|
||||
model.enqueue([get_text_message("Final response")])
|
||||
agent = Agent(
|
||||
name="TimingAgentTrace",
|
||||
model=model,
|
||||
@@ -240,8 +240,8 @@ async def test_parent_span_and_trace_finish_after_slow_input_guardrail():
|
||||
async def test_parent_span_and_trace_finish_after_slow_model():
|
||||
"""Agent span and trace finish after model when model completes last."""
|
||||
|
||||
model = SlowCompleteFakeModel(delay_seconds=SLOW_GUARDRAIL_DELAY, tracing_enabled=True)
|
||||
model.set_next_output([get_text_message("Final response")])
|
||||
model = SlowCompleteScriptedModel(delay_seconds=SLOW_GUARDRAIL_DELAY, emit_traces=True)
|
||||
model.enqueue([get_text_message("Final response")])
|
||||
agent = Agent(
|
||||
name="TimingAgentTrace",
|
||||
model=model,
|
||||
|
||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import (
|
||||
@@ -13,75 +12,24 @@ from openai.types.responses import (
|
||||
)
|
||||
|
||||
from agents import Agent, Runner
|
||||
from agents.agent_output import AgentOutputSchemaBase
|
||||
from agents.handoffs import Handoff
|
||||
from agents.items import TResponseInputItem, TResponseOutputItem, TResponseStreamEvent
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
from agents.tool import Tool, function_tool
|
||||
from agents.items import TResponseOutputItem, TResponseStreamEvent
|
||||
from agents.testing import ModelStep, ScriptedModel
|
||||
from agents.tool import function_tool
|
||||
|
||||
from .fake_model import FakeModel, get_response_obj
|
||||
from .model_test_helpers import get_response_obj
|
||||
from .test_responses import get_final_output_message, get_function_tool_call
|
||||
|
||||
|
||||
class TerminalOutputStreamModel(FakeModel):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.terminal_turn_outputs: list[list[TResponseOutputItem]] = []
|
||||
|
||||
def add_terminal_turn_outputs(
|
||||
self,
|
||||
outputs: list[list[TResponseOutputItem]],
|
||||
) -> None:
|
||||
self.terminal_turn_outputs.extend(outputs)
|
||||
|
||||
def get_next_terminal_output(self) -> list[TResponseOutputItem]:
|
||||
if not self.terminal_turn_outputs:
|
||||
return []
|
||||
return self.terminal_turn_outputs.pop(0)
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None = None,
|
||||
conversation_id: str | None = None,
|
||||
prompt: Any | None = None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
turn_args = {
|
||||
"system_instructions": system_instructions,
|
||||
"input": input,
|
||||
"model_settings": model_settings,
|
||||
"tools": tools,
|
||||
"output_schema": output_schema,
|
||||
"previous_response_id": previous_response_id,
|
||||
"conversation_id": conversation_id,
|
||||
}
|
||||
|
||||
if self.first_turn_args is None:
|
||||
self.first_turn_args = turn_args.copy()
|
||||
|
||||
self.last_turn_args = turn_args
|
||||
streamed_output = self.get_next_output()
|
||||
if isinstance(streamed_output, Exception):
|
||||
raise streamed_output
|
||||
|
||||
terminal_response = get_response_obj(
|
||||
self.get_next_terminal_output(),
|
||||
usage=self.hardcoded_usage,
|
||||
)
|
||||
def _stream_step(
|
||||
streamed_output: list[TResponseOutputItem],
|
||||
terminal_output: list[TResponseOutputItem],
|
||||
) -> ModelStep:
|
||||
async def events(_call) -> AsyncIterator[TResponseStreamEvent]:
|
||||
terminal_response = get_response_obj(terminal_output)
|
||||
sequence_number = 0
|
||||
|
||||
yield ResponseCreatedEvent(
|
||||
type="response.created",
|
||||
response=terminal_response,
|
||||
sequence_number=sequence_number,
|
||||
type="response.created", response=terminal_response, sequence_number=sequence_number
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
@@ -107,6 +55,8 @@ class TerminalOutputStreamModel(FakeModel):
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
|
||||
return ModelStep.stream(events)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_runner_backfills_empty_terminal_output_before_step_resolution() -> None:
|
||||
@@ -117,22 +67,17 @@ async def test_streamed_runner_backfills_empty_terminal_output_before_step_resol
|
||||
return "tool_result"
|
||||
|
||||
tool = function_tool(test_tool, name_override="foo")
|
||||
model = TerminalOutputStreamModel()
|
||||
model = ScriptedModel(
|
||||
[
|
||||
_stream_step(
|
||||
[get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call-1")],
|
||||
[],
|
||||
),
|
||||
_stream_step([get_final_output_message("done")], [get_final_output_message("done")]),
|
||||
]
|
||||
)
|
||||
agent = Agent(name="test", model=model, tools=[tool])
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call-1")],
|
||||
[get_final_output_message("done")],
|
||||
]
|
||||
)
|
||||
model.add_terminal_turn_outputs(
|
||||
[
|
||||
[],
|
||||
[get_final_output_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
result = Runner.run_streamed(agent, input="test")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
@@ -151,20 +96,16 @@ async def test_streamed_runner_preserves_populated_terminal_output() -> None:
|
||||
return "tool_result"
|
||||
|
||||
tool = function_tool(test_tool, name_override="foo")
|
||||
model = TerminalOutputStreamModel()
|
||||
model = ScriptedModel(
|
||||
[
|
||||
_stream_step(
|
||||
[get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call-1")],
|
||||
[get_final_output_message("done")],
|
||||
)
|
||||
]
|
||||
)
|
||||
agent = Agent(name="test", model=model, tools=[tool])
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call-1")],
|
||||
]
|
||||
)
|
||||
model.add_terminal_turn_outputs(
|
||||
[
|
||||
[get_final_output_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
result = Runner.run_streamed(agent, input="test")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
@@ -188,25 +129,20 @@ async def test_streamed_runner_backfills_multiple_tool_calls_in_order() -> None:
|
||||
|
||||
foo = function_tool(foo_tool, name_override="foo")
|
||||
bar = function_tool(bar_tool, name_override="bar")
|
||||
model = TerminalOutputStreamModel()
|
||||
model = ScriptedModel(
|
||||
[
|
||||
_stream_step(
|
||||
[
|
||||
get_function_tool_call("foo", json.dumps({"a": "first"}), call_id="call-1"),
|
||||
get_function_tool_call("bar", json.dumps({"b": "second"}), call_id="call-2"),
|
||||
],
|
||||
[],
|
||||
),
|
||||
_stream_step([get_final_output_message("done")], [get_final_output_message("done")]),
|
||||
]
|
||||
)
|
||||
agent = Agent(name="test", model=model, tools=[foo, bar])
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[
|
||||
get_function_tool_call("foo", json.dumps({"a": "first"}), call_id="call-1"),
|
||||
get_function_tool_call("bar", json.dumps({"b": "second"}), call_id="call-2"),
|
||||
],
|
||||
[get_final_output_message("done")],
|
||||
]
|
||||
)
|
||||
model.add_terminal_turn_outputs(
|
||||
[
|
||||
[],
|
||||
[get_final_output_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
result = Runner.run_streamed(agent, input="test")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
|
||||
@@ -10,7 +10,7 @@ from agents.items import ToolCallOutputItem
|
||||
from agents.run import AgentRunner
|
||||
from agents.run_context import RunContextWrapper
|
||||
from agents.run_state import RunState
|
||||
from tests.fake_model import FakeModel
|
||||
from agents.testing import ScriptedModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
|
||||
@@ -20,8 +20,8 @@ async def test_run_streamed_resume_omits_tool_output_in_log_when_dont_log(
|
||||
) -> None:
|
||||
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True)
|
||||
|
||||
model = FakeModel()
|
||||
model.set_next_output([get_text_message("ok")])
|
||||
model = ScriptedModel()
|
||||
model.enqueue([get_text_message("ok")])
|
||||
agent = Agent(name="log-agent", model=model)
|
||||
context_wrapper: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
|
||||
state = RunState(
|
||||
|
||||
@@ -7,7 +7,7 @@ were emitted with empty arguments during streaming (Issue #1629).
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, cast
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import (
|
||||
@@ -18,116 +18,54 @@ from openai.types.responses import (
|
||||
)
|
||||
|
||||
from agents import Agent, Runner, function_tool
|
||||
from agents.agent_output import AgentOutputSchemaBase
|
||||
from agents.handoffs import Handoff
|
||||
from agents.items import TResponseInputItem, TResponseOutputItem, TResponseStreamEvent
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import Model, ModelTracing
|
||||
from agents.items import TResponseOutputItem, TResponseStreamEvent
|
||||
from agents.stream_events import RunItemStreamEvent
|
||||
from agents.tool import Tool
|
||||
from agents.tracing import generation_span
|
||||
from agents.testing import ModelStep, ScriptedModel
|
||||
from tests.model_test_helpers import get_response_obj
|
||||
|
||||
from .fake_model import get_response_obj
|
||||
from .test_responses import get_function_tool_call
|
||||
|
||||
|
||||
class StreamingFakeModel(Model):
|
||||
"""A fake model that actually emits streaming events to test our streaming fix."""
|
||||
def _split_argument_step(output: list[TResponseOutputItem]) -> ModelStep:
|
||||
async def events(_call) -> AsyncIterator[TResponseStreamEvent]:
|
||||
sequence_number = 0
|
||||
|
||||
def __init__(self):
|
||||
self.turn_outputs: list[list[TResponseOutputItem]] = []
|
||||
self.last_turn_args: dict[str, Any] = {}
|
||||
# Emit each output item with proper streaming events.
|
||||
for item in output:
|
||||
if isinstance(item, ResponseFunctionToolCall):
|
||||
# First emit an added event with empty arguments, as the API does before deltas.
|
||||
empty_args_item = ResponseFunctionToolCall(
|
||||
id=item.id,
|
||||
call_id=item.call_id,
|
||||
type=item.type,
|
||||
name=item.name,
|
||||
arguments="",
|
||||
)
|
||||
|
||||
def set_next_output(self, output: list[TResponseOutputItem]):
|
||||
self.turn_outputs.append(output)
|
||||
yield ResponseOutputItemAddedEvent(
|
||||
item=empty_args_item,
|
||||
output_index=0,
|
||||
type="response.output_item.added",
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
def get_next_output(self) -> list[TResponseOutputItem]:
|
||||
if not self.turn_outputs:
|
||||
return []
|
||||
return self.turn_outputs.pop(0)
|
||||
# Then emit the completed item with its final arguments.
|
||||
yield ResponseOutputItemDoneEvent(
|
||||
item=item,
|
||||
output_index=0,
|
||||
type="response.output_item.done",
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: Any | None,
|
||||
):
|
||||
raise NotImplementedError("Use stream_response instead")
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=get_response_obj(output),
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None = None,
|
||||
conversation_id: str | None = None,
|
||||
prompt: Any | None = None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
"""Stream events that simulate real OpenAI streaming behavior for tool calls."""
|
||||
self.last_turn_args = {
|
||||
"system_instructions": system_instructions,
|
||||
"input": input,
|
||||
"model_settings": model_settings,
|
||||
"tools": tools,
|
||||
"output_schema": output_schema,
|
||||
"previous_response_id": previous_response_id,
|
||||
"conversation_id": conversation_id,
|
||||
}
|
||||
|
||||
with generation_span(disabled=True) as _:
|
||||
output = self.get_next_output()
|
||||
|
||||
sequence_number = 0
|
||||
|
||||
# Emit each output item with proper streaming events
|
||||
for item in output:
|
||||
if isinstance(item, ResponseFunctionToolCall):
|
||||
# First: emit ResponseOutputItemAddedEvent with EMPTY arguments
|
||||
# (this simulates the real streaming behavior that was causing the bug)
|
||||
empty_args_item = ResponseFunctionToolCall(
|
||||
id=item.id,
|
||||
call_id=item.call_id,
|
||||
type=item.type,
|
||||
name=item.name,
|
||||
arguments="", # EMPTY - this is the bug condition!
|
||||
)
|
||||
|
||||
yield ResponseOutputItemAddedEvent(
|
||||
item=empty_args_item,
|
||||
output_index=0,
|
||||
type="response.output_item.added",
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
# Then: emit ResponseOutputItemDoneEvent with COMPLETE arguments
|
||||
yield ResponseOutputItemDoneEvent(
|
||||
item=item, # This has the complete arguments
|
||||
output_index=0,
|
||||
type="response.output_item.done",
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
# Finally: emit completion
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=get_response_obj(output),
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
return ModelStep.stream(events)
|
||||
|
||||
|
||||
@function_tool
|
||||
@@ -146,7 +84,7 @@ def format_message(name: str, message: str, urgent: bool = False) -> str:
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_tool_call_arguments_not_empty():
|
||||
"""Test that tool_called events contain non-empty arguments during streaming."""
|
||||
model = StreamingFakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="TestAgent",
|
||||
model=model,
|
||||
@@ -155,12 +93,14 @@ async def test_streaming_tool_call_arguments_not_empty():
|
||||
|
||||
# Set up a tool call with arguments
|
||||
expected_arguments = '{"a": 5, "b": 3}'
|
||||
model.set_next_output(
|
||||
[
|
||||
get_function_tool_call("calculate_sum", expected_arguments, "call_123"),
|
||||
]
|
||||
model.enqueue(
|
||||
_split_argument_step(
|
||||
[
|
||||
get_function_tool_call("calculate_sum", expected_arguments, "call_123"),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
model.enqueue([])
|
||||
result = Runner.run_streamed(agent, input="Add 5 and 3")
|
||||
|
||||
tool_called_events = []
|
||||
@@ -212,7 +152,7 @@ async def test_streaming_tool_call_arguments_not_empty():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_tool_call_arguments_complex():
|
||||
"""Test streaming tool calls with complex arguments including strings and booleans."""
|
||||
model = StreamingFakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="TestAgent",
|
||||
model=model,
|
||||
@@ -223,12 +163,14 @@ async def test_streaming_tool_call_arguments_complex():
|
||||
expected_arguments = (
|
||||
'{"name": "Alice", "message": "Your meeting is starting soon", "urgent": true}'
|
||||
)
|
||||
model.set_next_output(
|
||||
[
|
||||
get_function_tool_call("format_message", expected_arguments, "call_456"),
|
||||
]
|
||||
model.enqueue(
|
||||
_split_argument_step(
|
||||
[
|
||||
get_function_tool_call("format_message", expected_arguments, "call_456"),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
model.enqueue([])
|
||||
result = Runner.run_streamed(agent, input="Format a message for Alice")
|
||||
|
||||
tool_called_events = []
|
||||
@@ -265,7 +207,7 @@ async def test_streaming_tool_call_arguments_complex():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_multiple_tool_calls_arguments():
|
||||
"""Test that multiple tool calls in streaming all have proper arguments."""
|
||||
model = StreamingFakeModel()
|
||||
model = ScriptedModel()
|
||||
agent = Agent(
|
||||
name="TestAgent",
|
||||
model=model,
|
||||
@@ -273,15 +215,17 @@ async def test_streaming_multiple_tool_calls_arguments():
|
||||
)
|
||||
|
||||
# Set up multiple tool calls
|
||||
model.set_next_output(
|
||||
[
|
||||
get_function_tool_call("calculate_sum", '{"a": 10, "b": 20}', "call_1"),
|
||||
get_function_tool_call(
|
||||
"format_message", '{"name": "Bob", "message": "Test"}', "call_2"
|
||||
),
|
||||
]
|
||||
model.enqueue(
|
||||
_split_argument_step(
|
||||
[
|
||||
get_function_tool_call("calculate_sum", '{"a": 10, "b": 20}', "call_1"),
|
||||
get_function_tool_call(
|
||||
"format_message", '{"name": "Bob", "message": "Test"}', "call_2"
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
model.enqueue([])
|
||||
result = Runner.run_streamed(agent, input="Do some calculations")
|
||||
|
||||
tool_called_events = []
|
||||
@@ -324,7 +268,7 @@ async def test_streaming_multiple_tool_calls_arguments():
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_tool_call_with_empty_arguments():
|
||||
"""Test that tool calls with legitimately empty arguments still work correctly."""
|
||||
model = StreamingFakeModel()
|
||||
model = ScriptedModel()
|
||||
|
||||
@function_tool
|
||||
def get_current_time() -> str:
|
||||
@@ -338,12 +282,14 @@ async def test_streaming_tool_call_with_empty_arguments():
|
||||
)
|
||||
|
||||
# Tool call with empty arguments (legitimate case)
|
||||
model.set_next_output(
|
||||
[
|
||||
get_function_tool_call("get_current_time", "{}", "call_time"),
|
||||
]
|
||||
model.enqueue(
|
||||
_split_argument_step(
|
||||
[
|
||||
get_function_tool_call("get_current_time", "{}", "call_time"),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
model.enqueue([])
|
||||
result = Runner.run_streamed(agent, input="What time is it?")
|
||||
|
||||
tool_called_events = []
|
||||
|
||||
@@ -46,7 +46,7 @@ from agents._tool_invocation import (
|
||||
)
|
||||
from agents.editor import ApplyPatchOperation, ApplyPatchResult
|
||||
from agents.exceptions import ModelBehaviorError, UserError
|
||||
from agents.items import ModelResponse, ToolApprovalItem
|
||||
from agents.items import ModelResponse, ToolApprovalItem, TResponseOutputItem
|
||||
from agents.lifecycle import RunHooks
|
||||
from agents.models.interface import Model, ModelProvider
|
||||
from agents.run_context import RunContextWrapper
|
||||
@@ -59,9 +59,10 @@ from agents.run_internal.tool_execution import (
|
||||
from agents.run_internal.tool_planning import _collect_runs_by_approval
|
||||
from agents.run_state import RunState
|
||||
from agents.stream_events import RunItemStreamEvent
|
||||
from agents.testing import ScriptedModel
|
||||
from agents.tool import Tool
|
||||
from agents.tool_context import ToolContext
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.model_test_helpers import get_exact_output_stream_step
|
||||
from tests.test_computer_tool_lifecycle import FakeComputer
|
||||
from tests.test_responses import get_function_tool_call, get_handoff_tool_call, get_text_message
|
||||
from tests.utils.hitl import make_apply_patch_dict, make_shell_call, make_state_with_interruptions
|
||||
@@ -205,8 +206,8 @@ async def test_completed_apply_patch_fallback_run_state_round_trip(fallback_type
|
||||
call_id="patch_0",
|
||||
arguments=json.dumps(operation),
|
||||
)
|
||||
model = FakeModel(initial_output=[call])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = ScriptedModel(steps=[[call]])
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="agent", model=model, tools=[tool])
|
||||
|
||||
result = await Runner.run(agent, "update the file")
|
||||
@@ -236,10 +237,8 @@ async def test_streamed_function_apply_patch_replay_emits_one_tool_called_event(
|
||||
call_id="patch_0",
|
||||
arguments=json.dumps(operation),
|
||||
)
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
[[call], [call.model_copy(deep=True)], [get_text_message("done")]]
|
||||
)
|
||||
model = ScriptedModel()
|
||||
model.extend([[call], [call.model_copy(deep=True)], [get_text_message("done")]])
|
||||
agent = Agent(
|
||||
name="agent",
|
||||
model=model,
|
||||
@@ -305,8 +304,8 @@ async def test_custom_named_shell_sticky_approval_applies_to_fresh_call_ids() ->
|
||||
|
||||
first_call = make_shell_call("call_0", commands=["echo first"])
|
||||
second_call = make_shell_call("call_1", commands=["echo second"])
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[first_call], [second_call], [get_text_message("done")]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[first_call], [second_call], [get_text_message("done")]])
|
||||
tool = ShellTool(
|
||||
executor=execute,
|
||||
name="safe_shell",
|
||||
@@ -337,7 +336,7 @@ async def test_custom_named_shell_replacement_rejects_completed_call_id_replay()
|
||||
return "ok"
|
||||
|
||||
call = make_shell_call("call_0", commands=["echo safe"])
|
||||
model = FakeModel(initial_output=[call])
|
||||
model = ScriptedModel(steps=[[call]])
|
||||
original_tool = ShellTool(
|
||||
executor=run_first,
|
||||
name="safe_shell",
|
||||
@@ -348,7 +347,7 @@ async def test_custom_named_shell_replacement_rejects_completed_call_id_replay()
|
||||
first = await Runner.run(agent, "run command")
|
||||
state = first.to_state()
|
||||
state.approve(first.interruptions[0])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
completed = await Runner.run(agent, state)
|
||||
|
||||
agent.tools = [
|
||||
@@ -358,7 +357,7 @@ async def test_custom_named_shell_replacement_rejects_completed_call_id_replay()
|
||||
needs_approval=False,
|
||||
)
|
||||
]
|
||||
model.set_next_output([cast(Any, dict(cast(dict[str, Any], call)))])
|
||||
model.enqueue([cast(Any, dict(cast(dict[str, Any], call)))])
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="unique call ID"):
|
||||
await Runner.run(agent, completed.to_state())
|
||||
@@ -376,8 +375,8 @@ async def test_schema_1_13_custom_named_shell_skips_exact_completed_replay() ->
|
||||
return "ok"
|
||||
|
||||
call = make_shell_call("legacy-shell", commands=["echo safe"])
|
||||
model = FakeModel(initial_output=[call])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = ScriptedModel(steps=[[call]])
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(
|
||||
name="agent",
|
||||
model=model,
|
||||
@@ -401,7 +400,7 @@ async def test_schema_1_13_custom_named_shell_skips_exact_completed_replay() ->
|
||||
restored_record = restored._context._tool_invocations["legacy-shell"]
|
||||
assert restored_record.approval_scope == expected_identity[2]
|
||||
assert restored_record.fingerprint == expected_identity[3]
|
||||
model.add_multiple_turn_outputs(
|
||||
model.extend(
|
||||
[
|
||||
[call],
|
||||
[get_text_message("done again")],
|
||||
@@ -499,8 +498,8 @@ def _build_scenario(
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -549,14 +548,16 @@ async def test_empty_custom_tool_call_id_fails_before_approval_or_execution() ->
|
||||
needs_approval=True,
|
||||
on_approval=approve,
|
||||
)
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
name=tool.name,
|
||||
call_id="",
|
||||
input="changed",
|
||||
)
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
name=tool.name,
|
||||
call_id="",
|
||||
input="changed",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
agent = Agent(name="agent", model=model, tools=[tool])
|
||||
@@ -576,15 +577,17 @@ async def test_empty_unresolved_function_call_id_fails_before_error_formatter()
|
||||
formatter_calls.append(args.tool_name)
|
||||
return "error"
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
ResponseFunctionToolCall(
|
||||
id="item_0",
|
||||
type="function_call",
|
||||
name="missing",
|
||||
arguments="{}",
|
||||
call_id="",
|
||||
)
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
ResponseFunctionToolCall(
|
||||
id="item_0",
|
||||
type="function_call",
|
||||
name="missing",
|
||||
arguments="{}",
|
||||
call_id="",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
agent = Agent(name="agent", model=model)
|
||||
@@ -627,8 +630,8 @@ async def test_bound_call_id_with_missing_arguments_fails_before_error_formatter
|
||||
name="missing",
|
||||
call_id="shared",
|
||||
)
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[valid_call], [malformed_replacement]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[valid_call], [malformed_replacement]])
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="unique call ID"):
|
||||
@@ -654,15 +657,17 @@ async def test_empty_handoff_call_id_fails_before_handoff_callback() -> None:
|
||||
tool_name_override="route",
|
||||
on_handoff=lambda _context: handoff_calls.append("route"),
|
||||
)
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
ResponseFunctionToolCall(
|
||||
id="item_0",
|
||||
type="function_call",
|
||||
name="route",
|
||||
arguments="{}",
|
||||
call_id="",
|
||||
)
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
ResponseFunctionToolCall(
|
||||
id="item_0",
|
||||
type="function_call",
|
||||
name="route",
|
||||
arguments="{}",
|
||||
call_id="",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
agent = Agent(name="agent", model=model, handoffs=[route])
|
||||
@@ -689,8 +694,8 @@ async def test_failed_handoff_hook_does_not_commit_output_or_repeat_callback() -
|
||||
hook_calls.append(to_agent.name)
|
||||
raise RuntimeError("handoff hook failed")
|
||||
|
||||
model = FakeModel(initial_output=[call])
|
||||
model.set_next_output([call.model_copy(deep=True)])
|
||||
model = ScriptedModel(steps=[[call]])
|
||||
model.enqueue([call.model_copy(deep=True)])
|
||||
agent = Agent(name="source", model=model, handoffs=[target])
|
||||
context = RunContextWrapper(context=None)
|
||||
hooks = FailingHooks()
|
||||
@@ -763,8 +768,8 @@ async def test_non_approval_identical_siblings_execute_once(
|
||||
'{"value":"safe"}',
|
||||
call_id="call_0",
|
||||
)
|
||||
model = FakeModel(initial_output=[duplicate, duplicate.model_copy(deep=True)])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = ScriptedModel(steps=[[duplicate, duplicate.model_copy(deep=True)]])
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
|
||||
result = await _run(
|
||||
@@ -795,8 +800,8 @@ async def test_non_approval_completed_replay_does_not_execute_again(
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
|
||||
[get_function_tool_call("record_value", '{ "value" : "safe" }', call_id="call_0")],
|
||||
@@ -838,8 +843,8 @@ async def test_non_approval_changed_completed_call_id_fails_before_execution(
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
|
||||
[get_function_tool_call("record_value", '{"value":"changed"}', call_id="call_0")],
|
||||
@@ -877,28 +882,32 @@ async def test_nested_agent_tool_approval_allows_outer_call_id_collision(
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
inner_model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call(
|
||||
"inner_sensitive_tool",
|
||||
'{"value":"safe"}',
|
||||
call_id="shared",
|
||||
)
|
||||
inner_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call(
|
||||
"inner_sensitive_tool",
|
||||
'{"value":"safe"}',
|
||||
call_id="shared",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
inner_model.set_next_output([get_text_message("inner done")])
|
||||
inner_model.enqueue([get_text_message("inner done")])
|
||||
inner_agent = Agent(name="inner", model=inner_model, tools=[inner_sensitive_tool])
|
||||
|
||||
outer_model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call(
|
||||
"nested_agent",
|
||||
'{"input":"hello"}',
|
||||
call_id="shared",
|
||||
)
|
||||
outer_model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call(
|
||||
"nested_agent",
|
||||
'{"input":"hello"}',
|
||||
call_id="shared",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
outer_model.set_next_output([get_text_message("outer done")])
|
||||
outer_model.enqueue([get_text_message("outer done")])
|
||||
outer_agent = Agent(
|
||||
name="outer",
|
||||
model=outer_model,
|
||||
@@ -936,8 +945,8 @@ async def test_parent_approval_does_not_authorize_independent_nested_run(
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
inner_model = FakeModel()
|
||||
inner_model.add_multiple_turn_outputs(
|
||||
inner_model = ScriptedModel()
|
||||
inner_model.extend(
|
||||
[
|
||||
[get_function_tool_call("sensitive", '{"value":"same"}', call_id="shared")],
|
||||
[get_text_message("inner done")],
|
||||
@@ -945,8 +954,8 @@ async def test_parent_approval_does_not_authorize_independent_nested_run(
|
||||
)
|
||||
inner_agent = Agent(name="inner", model=inner_model, tools=[sensitive])
|
||||
|
||||
outer_model = FakeModel()
|
||||
outer_model.add_multiple_turn_outputs(
|
||||
outer_model = ScriptedModel()
|
||||
outer_model.extend(
|
||||
[
|
||||
[get_function_tool_call("sensitive", '{"value":"same"}', call_id="shared")],
|
||||
[
|
||||
@@ -1016,13 +1025,15 @@ async def test_streamed_validated_items_precede_llm_end_hook_failure() -> None:
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call(
|
||||
"record_value",
|
||||
'{"value":"safe"}',
|
||||
call_id="call_0",
|
||||
)
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_value",
|
||||
'{"value":"safe"}',
|
||||
call_id="call_0",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
@@ -1048,8 +1059,8 @@ async def test_non_approval_failed_tool_body_does_not_reexecute() -> None:
|
||||
raise RuntimeError("failed after side effect")
|
||||
|
||||
call = get_function_tool_call("perform_side_effect", "{}", call_id="call_0")
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[call], [call.model_copy(deep=True)]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[call], [call.model_copy(deep=True)]])
|
||||
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
|
||||
context = RunContextWrapper(context=None)
|
||||
|
||||
@@ -1082,8 +1093,8 @@ async def test_non_approval_custom_tool_identical_siblings_execute_once() -> Non
|
||||
call_id="call_0",
|
||||
input="safe",
|
||||
)
|
||||
model = FakeModel(initial_output=[duplicate, duplicate.model_copy(deep=True)])
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model = ScriptedModel(steps=[[duplicate, duplicate.model_copy(deep=True)]])
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="agent", model=model, tools=[tool])
|
||||
|
||||
result = await Runner.run(agent, "edit text")
|
||||
@@ -1107,10 +1118,12 @@ async def test_changed_same_id_siblings_fail_before_approval_callback(
|
||||
async def record_value(value: str) -> str:
|
||||
return value
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
|
||||
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
|
||||
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
|
||||
]
|
||||
]
|
||||
)
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
@@ -1140,7 +1153,9 @@ async def test_response_processing_error_still_invokes_llm_end(
|
||||
self.llm_end_calls += 1
|
||||
|
||||
hooks = CountingHooks()
|
||||
model = FakeModel(initial_output=[make_shell_call("call_0", commands=["echo safe"])])
|
||||
output: list[TResponseOutputItem] = [make_shell_call("call_0", commands=["echo safe"])]
|
||||
step = get_exact_output_stream_step(output) if mode == "streamed" else output
|
||||
model = ScriptedModel(steps=[step])
|
||||
agent = Agent(name="agent", model=model)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="without a shell tool"):
|
||||
@@ -1171,19 +1186,21 @@ async def test_processing_error_with_changed_call_id_still_suppresses_llm_end(
|
||||
async def record_value(value: str) -> str:
|
||||
return value
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
outputs: list[list[TResponseOutputItem]] = [
|
||||
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="shared")],
|
||||
[
|
||||
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="shared")],
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_value",
|
||||
'{"value":"changed"}',
|
||||
call_id="shared",
|
||||
),
|
||||
make_shell_call("shell_0", commands=["echo safe"]),
|
||||
],
|
||||
]
|
||||
get_function_tool_call(
|
||||
"record_value",
|
||||
'{"value":"changed"}',
|
||||
call_id="shared",
|
||||
),
|
||||
make_shell_call("shell_0", commands=["echo safe"]),
|
||||
],
|
||||
]
|
||||
model = ScriptedModel(
|
||||
[get_exact_output_stream_step(output) for output in outputs]
|
||||
if mode == "streamed"
|
||||
else outputs
|
||||
)
|
||||
hooks = CountingHooks()
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
@@ -1223,7 +1240,9 @@ async def test_processing_error_with_repeated_uncanonical_id_suppresses_llm_end(
|
||||
call_id="shared",
|
||||
)
|
||||
hooks = CountingHooks()
|
||||
model = FakeModel(initial_output=[first_call, second_call])
|
||||
output: list[TResponseOutputItem] = [first_call, second_call]
|
||||
step = get_exact_output_stream_step(output) if mode == "streamed" else output
|
||||
model = ScriptedModel(steps=[step])
|
||||
agent = Agent(name="agent", model=model)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="one response"):
|
||||
@@ -1244,10 +1263,12 @@ async def test_changed_same_id_siblings_fail_before_non_approval_execution(
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
|
||||
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
|
||||
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
|
||||
]
|
||||
]
|
||||
)
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
@@ -1295,9 +1316,11 @@ async def test_changed_computer_safety_checks_fail_before_same_response_effects(
|
||||
],
|
||||
}
|
||||
)
|
||||
output: list[TResponseOutputItem] = [first_call, changed_call]
|
||||
step = get_exact_output_stream_step(output) if mode == "streamed" else output
|
||||
agent = Agent(
|
||||
name="computer-agent",
|
||||
model=FakeModel(initial_output=[first_call, changed_call]),
|
||||
model=ScriptedModel(steps=[step]),
|
||||
tools=[tool],
|
||||
)
|
||||
|
||||
@@ -1345,8 +1368,12 @@ async def test_changed_computer_safety_checks_fail_before_completed_replay_effec
|
||||
],
|
||||
}
|
||||
)
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[first_call], [changed_call]])
|
||||
outputs: list[list[TResponseOutputItem]] = [[first_call], [changed_call]]
|
||||
model = ScriptedModel(
|
||||
[get_exact_output_stream_step(output) for output in outputs]
|
||||
if mode == "streamed"
|
||||
else outputs
|
||||
)
|
||||
agent = Agent(name="computer-agent", model=model, tools=[tool])
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="completed tool call ID"):
|
||||
@@ -1389,8 +1416,8 @@ async def test_computer_hook_failure_does_not_repeat_side_effect() -> None:
|
||||
pending_safety_checks=[],
|
||||
status="completed",
|
||||
)
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[call], [call.model_copy(deep=True)]])
|
||||
model = ScriptedModel()
|
||||
model.extend([[call], [call.model_copy(deep=True)]])
|
||||
agent = Agent(name="computer-agent", model=model, tools=[tool])
|
||||
context = RunContextWrapper(context=None)
|
||||
hooks = FailOnceHooks()
|
||||
@@ -1413,8 +1440,8 @@ async def test_exact_replay_drops_tied_reasoning_item() -> None:
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
|
||||
[
|
||||
@@ -1437,7 +1464,7 @@ async def test_exact_replay_drops_tied_reasoning_item() -> None:
|
||||
|
||||
assert resumed.final_output == "done"
|
||||
assert executed == ["safe"]
|
||||
model_input = model.last_turn_args["input"]
|
||||
model_input = model.calls[-1].input
|
||||
assert isinstance(model_input, list)
|
||||
assert not any(item.get("id") == "rs_replay" for item in model_input)
|
||||
assert (
|
||||
@@ -1458,8 +1485,8 @@ async def test_streamed_exact_replay_does_not_emit_tied_reasoning_item() -> None
|
||||
executed.append(value)
|
||||
return value
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
|
||||
[
|
||||
@@ -1539,9 +1566,7 @@ async def test_failed_output_guardrail_does_not_reexecute_approved_call() -> Non
|
||||
executed.append("ran")
|
||||
return "sensitive"
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[get_function_tool_call("record_value", "{}", call_id="call_0")]
|
||||
)
|
||||
model = ScriptedModel(steps=[[get_function_tool_call("record_value", "{}", call_id="call_0")]])
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
first = await Runner.run(agent, "record a value")
|
||||
state = first.to_state()
|
||||
@@ -1566,8 +1591,8 @@ async def test_failed_approved_tool_body_does_not_reexecute() -> None:
|
||||
attempts.append("ran")
|
||||
raise RuntimeError("failed after side effect")
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]
|
||||
model = ScriptedModel(
|
||||
steps=[[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]]
|
||||
)
|
||||
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
|
||||
first = await Runner.run(agent, "run it")
|
||||
@@ -1596,8 +1621,8 @@ async def test_cancelled_approved_tool_body_does_not_reexecute() -> None:
|
||||
await keep_running.wait()
|
||||
return "done"
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]
|
||||
model = ScriptedModel(
|
||||
steps=[[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]]
|
||||
)
|
||||
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
|
||||
first = await Runner.run(agent, "run it")
|
||||
@@ -1621,7 +1646,7 @@ async def test_failed_approved_agent_tool_start_does_not_reexecute() -> None:
|
||||
hook_calls: list[str] = []
|
||||
inner_agent = Agent(
|
||||
name="inner",
|
||||
model=FakeModel(initial_output=[get_text_message("inner done")]),
|
||||
model=ScriptedModel(steps=[[get_text_message("inner done")]]),
|
||||
)
|
||||
agent_tool = inner_agent.as_tool(
|
||||
tool_name="delegate",
|
||||
@@ -1640,8 +1665,8 @@ async def test_failed_approved_agent_tool_start_does_not_reexecute() -> None:
|
||||
hook_calls.append("ran")
|
||||
raise RuntimeError("failed after side effect")
|
||||
|
||||
outer_model = FakeModel(
|
||||
initial_output=[get_function_tool_call("delegate", '{"input":"hi"}', call_id="call_0")]
|
||||
outer_model = ScriptedModel(
|
||||
steps=[[get_function_tool_call("delegate", '{"input":"hi"}', call_id="call_0")]]
|
||||
)
|
||||
outer_agent = Agent(name="outer", model=outer_model, tools=[agent_tool])
|
||||
first = await Runner.run(outer_agent, "delegate")
|
||||
@@ -1690,13 +1715,15 @@ async def test_failed_parallel_tool_end_hook_checkpoints_outputs_in_model_order(
|
||||
self.failed = True
|
||||
raise RuntimeError("second end hook failed")
|
||||
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call("first_tool", "{}", call_id="call_first"),
|
||||
get_function_tool_call("second_tool", "{}", call_id="call_second"),
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call("first_tool", "{}", call_id="call_first"),
|
||||
get_function_tool_call("second_tool", "{}", call_id="call_second"),
|
||||
]
|
||||
]
|
||||
)
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="agent", model=model, tools=[first_tool, second_tool])
|
||||
hooks = FailSecondHookOnce()
|
||||
|
||||
@@ -1712,7 +1739,7 @@ async def test_failed_parallel_tool_end_hook_checkpoints_outputs_in_model_order(
|
||||
|
||||
assert resumed.final_output == "done"
|
||||
assert sorted(executed) == ["first", "second"]
|
||||
model_input = model.last_turn_args["input"]
|
||||
model_input = model.calls[-1].input
|
||||
assert isinstance(model_input, list)
|
||||
output_call_ids = [
|
||||
item["call_id"]
|
||||
@@ -1739,8 +1766,8 @@ async def test_identical_approval_bound_siblings_execute_once(
|
||||
json.dumps({"value": "safe"}),
|
||||
call_id="call-duplicate",
|
||||
)
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[[duplicated_call, duplicated_call.model_copy(deep=True)], [get_text_message("done")]]
|
||||
)
|
||||
run_config = RunConfig(model_provider=_ScriptedProvider(model))
|
||||
@@ -1829,13 +1856,15 @@ async def test_serialized_sticky_identical_siblings_execute_once() -> None:
|
||||
'{"value":1}',
|
||||
call_id="call_0",
|
||||
)
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
duplicate,
|
||||
duplicate.model_copy(deep=True),
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
duplicate,
|
||||
duplicate.model_copy(deep=True),
|
||||
]
|
||||
]
|
||||
)
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
|
||||
first = await Runner.run(agent, "record a value")
|
||||
@@ -2061,8 +2090,8 @@ async def test_changed_tool_under_approved_call_id_fails_before_second_tool_star
|
||||
executed.append(f"second:{value}")
|
||||
return value
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -2116,8 +2145,8 @@ async def test_approved_call_id_reused_for_another_invocation_type_fails_before_
|
||||
on_invoke_tool=invoke_custom,
|
||||
format={"type": "text"},
|
||||
)
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -2172,8 +2201,8 @@ async def test_changed_missing_tool_under_approved_call_id_fails_before_sibling_
|
||||
executed.append("sibling")
|
||||
return "sibling"
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -2224,8 +2253,8 @@ def _build_serialized_replay_scenario(
|
||||
executed.append(f"gate:{value}")
|
||||
return value
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
model = ScriptedModel()
|
||||
model.extend(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
@@ -2316,8 +2345,8 @@ async def test_serialized_completed_approval_skips_exact_replay(
|
||||
provider = run_config.model_provider
|
||||
assert isinstance(provider, _ScriptedProvider)
|
||||
model = provider.model
|
||||
assert isinstance(model, FakeModel)
|
||||
model_input = model.last_turn_args["input"]
|
||||
assert isinstance(model, ScriptedModel)
|
||||
model_input = model.calls[-1].input
|
||||
assert isinstance(model_input, list)
|
||||
for call_id in ("call_0", "call_1"):
|
||||
calls = [
|
||||
@@ -2368,16 +2397,18 @@ async def test_legacy_schema_historical_sticky_call_id_is_not_reexecuted(
|
||||
call_id="call_0",
|
||||
),
|
||||
)
|
||||
model = FakeModel(
|
||||
initial_output=[
|
||||
get_function_tool_call(
|
||||
"record_value",
|
||||
json.dumps({"value": replay_value}),
|
||||
call_id="call_0",
|
||||
)
|
||||
model = ScriptedModel(
|
||||
steps=[
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_value",
|
||||
json.dumps({"value": replay_value}),
|
||||
call_id="call_0",
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
model.set_next_output([get_text_message("done")])
|
||||
model.enqueue([get_text_message("done")])
|
||||
agent = Agent(name="agent", model=model, tools=[record_value])
|
||||
context: RunContextWrapper[Any] = RunContextWrapper(context=None)
|
||||
context.approve_tool(
|
||||
@@ -2738,7 +2769,7 @@ async def test_unbindable_mcp_callback_request_requires_manual_reapproval() -> N
|
||||
)
|
||||
agent = Agent(
|
||||
name="mcp-approval-agent",
|
||||
model=FakeModel(initial_output=[request]),
|
||||
model=ScriptedModel(steps=[[request]]),
|
||||
tools=[mcp_tool],
|
||||
)
|
||||
|
||||
@@ -2746,7 +2777,8 @@ async def test_unbindable_mcp_callback_request_requires_manual_reapproval() -> N
|
||||
|
||||
assert callback_calls == 0
|
||||
assert len(result.interruptions) == 1
|
||||
assert result.interruptions[0].raw_item is request
|
||||
assert result.interruptions[0].raw_item == request
|
||||
assert result.interruptions[0].raw_item is not request
|
||||
|
||||
|
||||
@pytest.mark.parametrize("always_approve", [False, True], ids=["per-call", "sticky"])
|
||||
@@ -2828,10 +2860,8 @@ async def test_runner_omits_completed_mcp_approval_request_replay(
|
||||
server_label="test_server",
|
||||
arguments='{"limit":1,"query":"safe"}',
|
||||
)
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
[[first_request], [replayed_request], [get_text_message("done")]]
|
||||
)
|
||||
model = ScriptedModel()
|
||||
model.extend([[first_request], [replayed_request], [get_text_message("done")]])
|
||||
agent = Agent(name="mcp-approval-agent", model=model, tools=[mcp_tool])
|
||||
|
||||
first = await Runner.run(agent, "lookup")
|
||||
@@ -2845,7 +2875,7 @@ async def test_runner_omits_completed_mcp_approval_request_replay(
|
||||
|
||||
assert result.final_output == "done"
|
||||
assert callback_calls == int(with_callback)
|
||||
model_input = model.last_turn_args["input"]
|
||||
model_input = model.calls[-1].input
|
||||
assert isinstance(model_input, list)
|
||||
replay_items = [
|
||||
item
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user