Files
openai--openai-agents-python/tests/test_agent_runner.py
T
2026-08-12 23:44:15 +09:00

7981 lines
259 KiB
Python

from __future__ import annotations
import asyncio
import json
import logging
import tempfile
import warnings
from collections.abc import Callable
from pathlib import Path
from typing import Any, cast
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from openai import APIConnectionError, BadRequestError, NotFoundError
from openai.types.responses import ResponseFunctionToolCall
from openai.types.responses.response_output_item import McpApprovalRequest
from openai.types.responses.response_output_text import AnnotationFileCitation, ResponseOutputText
from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary
from openai.types.responses.tool_param import Mcp
from typing_extensions import TypedDict
import agents._debug as _debug
from agents import (
Agent,
AgentOutputSchema,
GuardrailFunctionOutput,
Handoff,
HandoffInputData,
InputGuardrail,
InputGuardrailTripwireTriggered,
ModelBehaviorError,
ModelRetryAdvice,
ModelRetrySettings,
ModelSettings,
OpenAIConversationsSession,
OutputGuardrail,
OutputGuardrailTripwireTriggered,
RetryDecision,
RetryPolicyContext,
RunConfig,
RunContextWrapper,
Runner,
SQLiteSession,
ToolExecutionConfig,
ToolGuardrailFunctionOutput,
ToolInputGuardrailData,
ToolNameCollisionPolicy,
ToolTimeoutError,
UserError,
handoff,
retry_policies,
tool_input_guardrail,
tool_namespace,
)
from agents._tool_identity import resolve_tool_name_collisions
from agents.agent import ToolsToFinalOutputResult
from agents.computer import Computer
from agents.items import (
HandoffOutputItem,
ModelResponse,
ReasoningItem,
RunItem,
ToolApprovalItem,
ToolCallItem,
ToolCallOutputItem,
TResponseInputItem,
)
from agents.lifecycle import RunHooks
from agents.memory import OpenAIResponsesCompactionSession, SessionSettings
from agents.models.fake_id import FAKE_RESPONSES_ID
from agents.run import AgentRunner, get_default_agent_runner, set_default_agent_runner
from agents.run_config import _default_trace_include_sensitive_data
from agents.run_internal.agent_bindings import bind_public_agent
from agents.run_internal.agent_runner_helpers import build_resumed_stream_debug_extra
from agents.run_internal.items import (
TOOL_CALL_SESSION_DESCRIPTION_KEY,
TOOL_CALL_SESSION_TITLE_KEY,
drop_orphan_function_calls,
ensure_input_item_format,
fingerprint_input_item,
normalize_input_items_for_api,
normalize_resumed_input,
)
from agents.run_internal.oai_conversation import OpenAIServerConversationTracker
from agents.run_internal.run_loop import get_new_response
from agents.run_internal.run_steps import NextStepFinalOutput, SingleStepResult
from agents.run_internal.session_persistence import (
_collect_retry_owned_tail_serializations,
persist_session_items_for_guardrail_trip,
prepare_input_with_session,
rewind_session_items,
save_result_to_session,
wait_for_session_cleanup,
)
from agents.run_internal.tool_execution import execute_approved_tools
from agents.run_internal.tool_use_tracker import AgentToolUseTracker
from agents.run_state import RunState
from agents.tool import ComputerTool, FunctionToolResult, HostedMCPTool, ShellTool, function_tool
from agents.tool_context import ToolContext
from agents.usage import Usage
from .fake_model import FakeModel
from .test_responses import (
get_final_output_message,
get_function_tool,
get_function_tool_call,
get_handoff_tool_call,
get_text_input_item,
get_text_message,
)
from .utils.factories import make_run_state
from .utils.hitl import consume_stream, make_context_wrapper, make_model_and_agent, make_shell_call
from .utils.simple_session import (
CountingSession,
IdStrippingSession,
RewriteAwareSimpleSession,
ServerManagedSimpleSession,
SimpleListSession,
)
class _DummyRunItem:
def __init__(self, payload: dict[str, Any], item_type: str = "tool_call_output_item"):
self._payload = payload
self.type = item_type
def to_input_item(self) -> dict[str, Any]:
return self._payload
async def run_execute_approved_tools(
agent: Agent[Any],
approval_item: ToolApprovalItem,
*,
approve: bool | None,
run_config: RunConfig | None = None,
mutate_state: Callable[[RunState[Any, Agent[Any]], ToolApprovalItem], None] | None = None,
) -> list[RunItem]:
"""Execute approved tools with a consistent setup."""
context_wrapper: RunContextWrapper[Any] = make_context_wrapper()
state = make_run_state(
agent,
context=context_wrapper,
original_input="test",
max_turns=1,
)
if approve is True:
state.approve(approval_item)
elif approve is False:
state.reject(approval_item)
if mutate_state is not None:
mutate_state(state, approval_item)
generated_items: list[RunItem] = []
all_tools = await agent.get_all_tools(context_wrapper)
await execute_approved_tools(
agent=agent,
interruptions=[approval_item],
context_wrapper=context_wrapper,
generated_items=generated_items,
run_config=run_config or RunConfig(),
hooks=RunHooks(),
all_tools=all_tools,
)
return generated_items
async def _run_agent_with_optional_streaming(
agent: Agent[Any],
*,
input: str | list[TResponseInputItem] | RunState[Any, Agent[Any]],
streamed: bool,
**kwargs: Any,
):
if streamed:
result = Runner.run_streamed(agent, input=input, **kwargs)
async for _ in result.stream_events():
pass
return result
return await Runner.run(agent, input=input, **kwargs)
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.asyncio
async def test_persistent_hosted_mcp_approval_does_not_cross_servers(streamed: bool) -> None:
model = FakeModel()
server_a = HostedMCPTool(
tool_config=Mcp(
type="mcp",
server_label="server-a",
server_url="https://server-a.example/mcp",
)
)
server_b = HostedMCPTool(
tool_config=Mcp(
type="mcp",
server_label="server-b",
server_url="https://server-b.example/mcp",
)
)
model.add_multiple_turn_outputs(
[
[
McpApprovalRequest(
id="request-a",
type="mcp_approval_request",
arguments="{}",
name="lookup_account",
server_label="server-a",
)
],
[
McpApprovalRequest(
id="request-b",
type="mcp_approval_request",
arguments="{}",
name="lookup_account",
server_label="server-b",
)
],
]
)
agent = Agent(name="test", model=model, tools=[server_a, server_b])
first = await _run_agent_with_optional_streaming(agent, input="hello", streamed=streamed)
assert len(first.interruptions) == 1
assert first.interruptions[0].raw_item.server_label == "server-a"
state = first.to_state()
state.approve(first.interruptions[0], always_approve=True)
restored_state = await RunState.from_json(agent, state.to_json())
resumed = await _run_agent_with_optional_streaming(
agent,
input=restored_state,
streamed=streamed,
)
assert len(resumed.interruptions) == 1
assert resumed.interruptions[0].raw_item.server_label == "server-b"
@pytest.mark.parametrize("surface", ["agent_tool", "handoff", "mixed"])
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.parametrize("collision_policy", ["warn", "error"])
@pytest.mark.asyncio
async def test_run_reports_derived_agent_name_collisions_before_model_call(
surface: str,
streamed: bool,
collision_policy: ToolNameCollisionPolicy,
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
model = FakeModel(initial_output=[get_text_message("done")])
billing = Agent(name="Billing Agent")
normalized_billing = Agent(name="billing agent")
if surface == "agent_tool":
agent = Agent(
name="triage",
model=model,
tools=[
billing.as_tool(tool_name=None, tool_description="First billing agent"),
normalized_billing.as_tool(
tool_name=None,
tool_description="Second billing agent",
),
],
)
elif surface == "handoff":
agent = Agent(
name="triage",
model=model,
handoffs=[billing, normalized_billing],
)
else:
agent = Agent(
name="triage",
model=model,
tools=[
Agent(name="transfer to Billing Agent").as_tool(
tool_name=None,
tool_description="Billing tool",
)
],
handoffs=[billing],
)
run_config = RunConfig(tool_name_collision_policy=collision_policy)
if collision_policy == "error":
with pytest.raises(
UserError,
match="Ambiguous (agent tool|handoff|agent routing) configuration",
):
await _run_agent_with_optional_streaming(
agent,
input="Route this request",
streamed=streamed,
run_config=run_config,
)
assert model.first_turn_args is None
assert not model.last_turn_args
else:
with caplog.at_level("WARNING", logger="openai.agents"):
await _run_agent_with_optional_streaming(
agent,
input="Route this request",
streamed=streamed,
run_config=run_config,
)
assert model.first_turn_args is not None
collision_messages = [
message for message in caplog.messages if message.startswith("Ambiguous ")
]
assert len(collision_messages) == 1
assert "Pass an explicit" in collision_messages[0]
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.asyncio
async def test_run_warns_and_keeps_last_duplicate_function_tool(
streamed: bool,
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
calls: list[str] = []
@function_tool(name_override="lookup")
def first_lookup() -> str:
calls.append("first")
return "first"
@function_tool(name_override="lookup")
def second_lookup() -> str:
calls.append("second")
return "second"
model = FakeModel(initial_output=[get_function_tool_call("lookup", "{}")])
model.set_next_output([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[first_lookup, second_lookup])
with caplog.at_level("WARNING", logger="openai.agents"):
await _run_agent_with_optional_streaming(
agent,
input="Look this up",
streamed=streamed,
)
assert calls == ["second"]
assert model.first_turn_args is not None
assert model.first_turn_args["tools"] == [second_lookup]
collision_messages = [
message for message in caplog.messages if message.startswith("Ambiguous ")
]
assert len(collision_messages) == 2
assert all(
message
== (
"Ambiguous function tool configuration: the tool name `lookup` is used by multiple "
"tools. Assign a unique routed name to every colliding function tool with "
"`name_override=`, `tool_name=`, or a namespace."
)
for message in collision_messages
)
def test_collision_warning_redacts_tool_data(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
secret_tool_name = "tenant_secret_tool_token"
@function_tool(name_override=secret_tool_name)
def first_tool() -> str:
return "first"
@function_tool(name_override=secret_tool_name)
def second_tool() -> str:
return "second"
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True)
with caplog.at_level("WARNING", logger="openai.agents"):
resolved_tools, resolved_handoffs = resolve_tool_name_collisions(
[first_tool, second_tool],
collision_policy="warn",
)
assert resolved_tools == [second_tool]
assert resolved_handoffs == []
assert len(caplog.records) == 1
record = caplog.records[0]
assert record.msg == (
"Tool name collision detected. Assign unique routed tool names or enable tool data "
"logging for details."
)
assert record.args == ()
assert record.exc_info is None
assert record.exc_text is None
assert all(
secret_tool_name not in value
for value in record.__dict__.values()
if isinstance(value, str)
)
assert secret_tool_name not in logging.Formatter().format(record)
def test_collision_warning_preserves_tool_diagnostics_when_enabled(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
tool_name = "diagnostic_tool_name"
@function_tool(name_override=tool_name)
def first_tool() -> str:
return "first"
@function_tool(name_override=tool_name)
def second_tool() -> str:
return "second"
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
with caplog.at_level("WARNING", logger="openai.agents"):
resolve_tool_name_collisions(
[first_tool, second_tool],
collision_policy="warn",
)
assert len(caplog.records) == 1
record = caplog.records[0]
assert record.msg == "%s"
assert isinstance(record.args, tuple)
assert len(record.args) == 1
assert isinstance(record.args[0], str)
assert tool_name in record.args[0]
assert tool_name in logging.Formatter().format(record)
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.asyncio
async def test_run_rejects_duplicate_function_tools_in_error_mode(streamed: bool) -> None:
@function_tool(name_override="lookup")
def first_lookup() -> str:
return "first"
@function_tool(name_override="lookup")
def second_lookup() -> str:
return "second"
model = FakeModel(initial_output=[get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[first_lookup, second_lookup])
with pytest.raises(
UserError,
match="the tool name `lookup` is used by multiple tools",
):
await _run_agent_with_optional_streaming(
agent,
input="Look this up",
streamed=streamed,
run_config=RunConfig(tool_name_collision_policy="error"),
)
assert model.first_turn_args is None
@pytest.mark.asyncio
async def test_run_warns_once_for_repeated_source_agent_name(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
model = FakeModel(initial_output=[get_text_message("done")])
agent = Agent(
name="orchestrator",
model=model,
tools=[
Agent(name="Refund").as_tool(tool_name=None, tool_description="First refund agent"),
Agent(name="refund").as_tool(tool_name=None, tool_description="Second refund agent"),
Agent(name="refund").as_tool(tool_name=None, tool_description="Third refund agent"),
],
)
with caplog.at_level("WARNING", logger="openai.agents"):
await Runner.run(agent, "Route this request")
collision_messages = [
message for message in caplog.messages if message.startswith("Ambiguous ")
]
assert collision_messages == [
"Ambiguous function tool configuration: the tool name `refund` is used by multiple "
"tools. Assign a unique routed name to every colliding function tool with "
"`name_override=`, `tool_name=`, or a namespace."
]
assert model.first_turn_args is not None
assert model.first_turn_args["tools"] == [agent.tools[-1]]
def test_multiway_mixed_collision_reports_every_owner_must_be_unique(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
@function_tool(name_override="route")
def first_route() -> str:
return "first"
@function_tool(name_override="route")
def second_route() -> str:
return "second"
route_handoff = handoff(Agent(name="Billing"), tool_name_override="route")
with caplog.at_level("WARNING", logger="openai.agents"):
resolved_tools, resolved_handoffs = resolve_tool_name_collisions(
[first_route, second_route],
[route_handoff],
collision_policy="warn",
)
assert resolved_tools == []
assert resolved_handoffs == [route_handoff]
assert caplog.messages == [
"Ambiguous tool routing configuration: the tool name `route` is used by both a function "
"tool and a handoff. Assign a unique routed name to every colliding function tool and "
"handoff with `name_override=`, `tool_name=`, `tool_name_override=`, or a namespace."
]
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.asyncio
async def test_handoff_enablement_uses_initialized_turn_context(streamed: bool) -> None:
model = FakeModel()
target = Agent(name="target", model=model)
model.add_multiple_turn_outputs(
[
[get_handoff_tool_call(target)],
[get_text_message("done")],
]
)
observed_context: list[tuple[list[TResponseInputItem], dict[str, bool]]] = []
class InitializeContextHooks(RunHooks[dict[str, bool]]):
async def on_agent_start(self, context, agent) -> None:
if agent.name == "source":
context.context["hook_initialized"] = True
def dynamic_prompt(data):
data.context.context["prompt_initialized"] = True
return {"id": "prompt-id"}
def handoff_is_enabled(context: RunContextWrapper[dict[str, bool]], agent: Agent[Any]) -> bool:
observed_context.append((list(context.turn_input), dict(context.context)))
return (
agent.name == "source"
and context.turn_input == [{"content": "current turn", "role": "user"}]
and context.context.get("hook_initialized") is True
and context.context.get("prompt_initialized") is True
)
source = Agent(
name="source",
model=model,
prompt=dynamic_prompt,
handoffs=[handoff(target, is_enabled=handoff_is_enabled)],
)
hooks = InitializeContextHooks()
if streamed:
result = Runner.run_streamed(
source,
"current turn",
context={},
hooks=hooks,
)
async for _ in result.stream_events():
pass
else:
await Runner.run(
source,
"current turn",
context={},
hooks=hooks,
)
assert observed_context == [
(
[{"content": "current turn", "role": "user"}],
{"hook_initialized": True, "prompt_initialized": True},
)
]
def test_set_default_agent_runner_roundtrip():
runner = AgentRunner()
set_default_agent_runner(runner)
assert get_default_agent_runner() is runner
# Reset to ensure other tests are unaffected.
set_default_agent_runner(None)
assert isinstance(get_default_agent_runner(), AgentRunner)
def test_set_default_agent_runner_preserves_falsey_runner():
class FalseyRunner(AgentRunner):
def __bool__(self) -> bool:
return False
original_runner = get_default_agent_runner()
runner = FalseyRunner()
try:
set_default_agent_runner(runner)
assert get_default_agent_runner() is runner
finally:
set_default_agent_runner(original_runner)
def test_resumed_stream_debug_extra_preserves_falsy_current_agent() -> None:
class FalsyAgent(Agent[Any]):
def __bool__(self) -> bool:
return False
agent: Agent[Any] = FalsyAgent(name="falsy")
state: RunState[None] = RunState(
context=RunContextWrapper(context=None),
original_input="input",
starting_agent=agent,
max_turns=1,
)
extra = build_resumed_stream_debug_extra(state, include_tool_output=False)
assert extra["current_agent"] == "falsy"
def test_run_streamed_preserves_legacy_positional_previous_response_id():
captured: dict[str, Any] = {}
class DummyRunner:
def run_streamed(self, starting_agent: Any, input: Any, **kwargs: Any):
captured.update(kwargs)
return object()
original_runner = get_default_agent_runner()
set_default_agent_runner(cast(Any, DummyRunner()))
try:
Runner.run_streamed(
cast(Any, None),
"hello",
None,
10,
None,
None,
"resp-legacy",
)
finally:
set_default_agent_runner(original_runner)
assert captured["previous_response_id"] == "resp-legacy"
assert captured["error_handlers"] is None
def test_default_trace_include_sensitive_data_env(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("OPENAI_AGENTS_TRACE_INCLUDE_SENSITIVE_DATA", "false")
assert _default_trace_include_sensitive_data() is False
monkeypatch.setenv("OPENAI_AGENTS_TRACE_INCLUDE_SENSITIVE_DATA", "TRUE")
assert _default_trace_include_sensitive_data() is True
def test_run_config_defaults_nested_handoff_history_opt_in():
assert RunConfig().nest_handoff_history is False
def testdrop_orphan_function_calls_removes_orphans():
items: list[TResponseInputItem] = [
cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_orphan",
"name": "tool_one",
"arguments": "{}",
},
),
cast(TResponseInputItem, {"type": "message", "role": "user", "content": "hello"}),
cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_keep",
"name": "tool_keep",
"arguments": "{}",
},
),
cast(
TResponseInputItem,
{"type": "function_call_output", "call_id": "call_keep", "output": "done"},
),
cast(TResponseInputItem, {"type": "shell_call", "call_id": "shell_orphan"}),
cast(TResponseInputItem, {"type": "shell_call", "call_id": "shell_keep"}),
cast(
TResponseInputItem,
{"type": "shell_call_output", "call_id": "shell_keep", "output": []},
),
cast(TResponseInputItem, {"type": "apply_patch_call", "call_id": "patch_orphan"}),
cast(TResponseInputItem, {"type": "apply_patch_call", "call_id": "patch_keep"}),
cast(
TResponseInputItem,
{"type": "apply_patch_call_output", "call_id": "patch_keep", "output": "done"},
),
cast(TResponseInputItem, {"type": "computer_call", "call_id": "computer_orphan"}),
cast(TResponseInputItem, {"type": "computer_call", "call_id": "computer_keep"}),
cast(
TResponseInputItem,
{"type": "computer_call_output", "call_id": "computer_keep", "output": {}},
),
cast(TResponseInputItem, {"type": "local_shell_call", "call_id": "local_shell_orphan"}),
cast(TResponseInputItem, {"type": "local_shell_call", "call_id": "local_shell_keep"}),
cast(
TResponseInputItem,
{
"type": "local_shell_call_output",
"call_id": "local_shell_keep",
"output": {"stdout": "", "stderr": "", "outcome": {}},
},
),
]
filtered = drop_orphan_function_calls(items)
orphan_call_ids = {
"call_orphan",
"shell_orphan",
"patch_orphan",
"computer_orphan",
"local_shell_orphan",
}
for entry in filtered:
if isinstance(entry, dict):
assert entry.get("call_id") not in orphan_call_ids
def _has_call(call_type: str, call_id: str) -> bool:
return any(
isinstance(entry, dict)
and entry.get("type") == call_type
and entry.get("call_id") == call_id
for entry in filtered
)
assert _has_call("function_call", "call_keep")
assert _has_call("shell_call", "shell_keep")
assert _has_call("apply_patch_call", "patch_keep")
assert _has_call("computer_call", "computer_keep")
assert _has_call("local_shell_call", "local_shell_keep")
def test_normalize_resumed_input_drops_orphan_function_calls():
raw_input: list[TResponseInputItem] = [
cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "orphan_call",
"name": "tool_orphan",
"arguments": "{}",
},
),
cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "paired_call",
"name": "tool_paired",
"arguments": "{}",
},
),
cast(
TResponseInputItem,
{"type": "function_call_output", "call_id": "paired_call", "output": "ok"},
),
]
normalized = normalize_resumed_input(raw_input)
assert isinstance(normalized, list)
call_ids = [
cast(dict[str, Any], item).get("call_id")
for item in normalized
if isinstance(item, dict) and item.get("type") == "function_call"
]
assert "orphan_call" not in call_ids
assert "paired_call" in call_ids
def test_normalize_resumed_input_drops_orphan_tool_search_calls():
raw_input: list[TResponseInputItem] = [
cast(
TResponseInputItem,
{
"type": "tool_search_call",
"call_id": "orphan_search",
"arguments": {"query": "orphan"},
"execution": "server",
"status": "completed",
},
),
cast(
TResponseInputItem,
{
"type": "tool_search_call",
"call_id": "paired_search",
"arguments": {"query": "paired"},
"execution": "server",
"status": "completed",
},
),
cast(
TResponseInputItem,
{
"type": "tool_search_output",
"call_id": "paired_search",
"execution": "server",
"status": "completed",
"tools": [],
},
),
]
normalized = normalize_resumed_input(raw_input)
assert isinstance(normalized, list)
call_ids = [
cast(dict[str, Any], item).get("call_id")
for item in normalized
if isinstance(item, dict) and item.get("type") == "tool_search_call"
]
assert "orphan_search" not in call_ids
assert "paired_search" in call_ids
def test_normalize_resumed_input_preserves_hosted_tool_search_pair_without_call_ids():
raw_input: list[TResponseInputItem] = [
cast(
TResponseInputItem,
{
"type": "tool_search_call",
"call_id": None,
"arguments": {"query": "paired"},
"execution": "server",
"status": "completed",
},
),
cast(
TResponseInputItem,
{
"type": "tool_search_output",
"call_id": None,
"execution": "server",
"status": "completed",
"tools": [],
},
),
]
normalized = normalize_resumed_input(raw_input)
assert isinstance(normalized, list)
assert [cast(dict[str, Any], item)["type"] for item in normalized] == [
"tool_search_call",
"tool_search_output",
]
def test_normalize_resumed_input_matches_latest_anonymous_tool_search_call():
raw_input: list[TResponseInputItem] = [
cast(
TResponseInputItem,
{
"type": "tool_search_call",
"call_id": None,
"arguments": {"query": "orphan"},
"execution": "server",
"status": "completed",
},
),
cast(
TResponseInputItem,
{
"type": "tool_search_call",
"call_id": None,
"arguments": {"query": "paired"},
"execution": "server",
"status": "completed",
},
),
cast(
TResponseInputItem,
{
"type": "tool_search_output",
"call_id": None,
"execution": "server",
"status": "completed",
"tools": [],
},
),
]
normalized = normalize_resumed_input(raw_input)
assert isinstance(normalized, list)
assert [cast(dict[str, Any], item)["type"] for item in normalized] == [
"tool_search_call",
"tool_search_output",
]
assert cast(dict[str, Any], normalized[0])["arguments"] == {"query": "paired"}
def testnormalize_input_items_for_api_preserves_provider_data():
items: list[TResponseInputItem] = [
cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_norm",
"status": "completed",
"output": "out",
"provider_data": {"trace": "keep"},
},
),
cast(
TResponseInputItem,
{
"type": "message",
"role": "user",
"content": "hi",
"provider_data": {"trace": "remove"},
},
),
]
normalized = normalize_input_items_for_api(items)
first = cast(dict[str, Any], normalized[0])
second = cast(dict[str, Any], normalized[1])
assert first["type"] == "function_call_output"
assert first["call_id"] == "call_norm"
assert first["provider_data"] == {"trace": "keep"}
assert second["role"] == "user"
assert second["provider_data"] == {"trace": "remove"}
def test_fingerprint_input_item_returns_none_when_model_dump_fails():
class _BrokenModelDump:
def model_dump(self, *_args: Any, **_kwargs: Any) -> dict[str, Any]:
raise RuntimeError("model_dump failed")
assert fingerprint_input_item(_BrokenModelDump()) is None
def test_server_conversation_tracker_tracks_previous_response_id():
tracker = OpenAIServerConversationTracker(conversation_id=None, previous_response_id="resp_a")
response = ModelResponse(
output=[get_text_message("hello")],
usage=Usage(),
response_id="resp_b",
)
tracker.track_server_items(response)
assert tracker.previous_response_id == "resp_b"
assert len(tracker.server_items) == 1
def _as_message(item: Any) -> dict[str, Any]:
assert isinstance(item, dict)
role = item.get("role")
assert isinstance(role, str)
assert role in {"assistant", "user", "system", "developer"}
return cast(dict[str, Any], item)
def _input_message_text(item: Any) -> str:
message = _as_message(item)
content = message.get("content")
if isinstance(content, str):
return content
assert isinstance(content, list)
texts: list[str] = []
for part in content:
assert isinstance(part, dict)
text = part.get("text")
if isinstance(text, str):
texts.append(text)
return "".join(texts)
def _find_reasoning_input_item(
items: str | list[TResponseInputItem] | Any,
) -> dict[str, Any] | None:
if not isinstance(items, list):
return None
for item in items:
if isinstance(item, dict) and item.get("type") == "reasoning":
return cast(dict[str, Any], item)
return None
@pytest.mark.asyncio
async def test_simple_first_run():
model = FakeModel()
agent = Agent(
name="test",
model=model,
)
model.set_next_output([get_text_message("first")])
result = await Runner.run(agent, input="test")
assert result.input == "test"
assert len(result.new_items) == 1, "exactly one item should be generated"
assert result.final_output == "first"
assert len(result.raw_responses) == 1, "exactly one model response should be generated"
assert result.raw_responses[0].output == [get_text_message("first")]
assert result.last_agent == agent
assert len(result.to_input_list()) == 2, "should have original input and generated item"
model.set_next_output([get_text_message("second")])
result = await Runner.run(
agent, input=[get_text_input_item("message"), get_text_input_item("another_message")]
)
assert len(result.new_items) == 1, "exactly one item should be generated"
assert result.final_output == "second"
assert len(result.raw_responses) == 1, "exactly one model response should be generated"
assert len(result.to_input_list()) == 3, "should have original input and generated item"
@pytest.mark.asyncio
async def test_subsequent_runs():
model = FakeModel()
agent = Agent(
name="test",
model=model,
)
model.set_next_output([get_text_message("third")])
result = await Runner.run(agent, input="test")
assert result.input == "test"
assert len(result.new_items) == 1, "exactly one item should be generated"
assert len(result.to_input_list()) == 2, "should have original input and generated item"
model.set_next_output([get_text_message("fourth")])
result = await Runner.run(agent, input=result.to_input_list())
assert len(result.input) == 2, f"should have previous input but got {result.input}"
assert len(result.new_items) == 1, "exactly one item should be generated"
assert result.final_output == "fourth"
assert len(result.raw_responses) == 1, "exactly one model response should be generated"
assert result.raw_responses[0].output == [get_text_message("fourth")]
assert result.last_agent == agent
assert len(result.to_input_list()) == 3, "should have original input and generated items"
@pytest.mark.asyncio
async def test_tool_call_runs():
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("foo", json.dumps({"a": "b"}))],
# Second turn: text message
[get_text_message("done")],
]
)
result = await Runner.run(agent, input="user_message")
assert result.final_output == "done"
assert len(result.raw_responses) == 2, (
"should have two responses: the first which produces a tool call, and the second which"
"handles the tool result"
)
assert len(result.to_input_list()) == 5, (
"should have five inputs: the original input, the message, the tool call, the tool result "
"and the done message"
)
@pytest.mark.asyncio
async def test_parallel_tool_call_with_cancelled_sibling_reaches_final_output() -> None:
async def _ok_tool() -> str:
return "ok"
async def _cancel_tool() -> str:
raise asyncio.CancelledError("tool-cancelled")
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[
function_tool(_ok_tool, name_override="ok_tool"),
function_tool(_cancel_tool, name_override="cancel_tool"),
],
)
model.add_multiple_turn_outputs(
[
[
get_function_tool_call("ok_tool", "{}", call_id="call_ok"),
get_function_tool_call("cancel_tool", "{}", call_id="call_cancel"),
],
[get_text_message("final answer")],
]
)
result = await Runner.run(agent, input="user_message")
assert result.final_output == "final answer"
assert len(result.raw_responses) == 2
second_turn_input = cast(list[dict[str, Any]], model.last_turn_args["input"])
tool_outputs = [
item for item in second_turn_input if item.get("type") == "function_call_output"
]
assert tool_outputs == [
{"call_id": "call_ok", "output": "ok", "type": "function_call_output"},
{
"call_id": "call_cancel",
"output": (
"An error occurred while running the tool. Please try again. Error: tool-cancelled"
),
"type": "function_call_output",
},
]
@pytest.mark.asyncio
async def test_single_tool_call_with_cancelled_tool_reaches_final_output() -> None:
async def _cancel_tool() -> str:
raise asyncio.CancelledError("tool-cancelled")
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[function_tool(_cancel_tool, name_override="cancel_tool")],
)
model.add_multiple_turn_outputs(
[
[get_function_tool_call("cancel_tool", "{}", call_id="call_cancel")],
[get_text_message("final answer")],
]
)
result = await Runner.run(agent, input="user_message")
assert result.final_output == "final answer"
assert len(result.raw_responses) == 2
second_turn_input = cast(list[dict[str, Any]], model.last_turn_args["input"])
tool_outputs = [
item for item in second_turn_input if item.get("type") == "function_call_output"
]
assert tool_outputs == [
{
"call_id": "call_cancel",
"output": (
"An error occurred while running the tool. Please try again. Error: tool-cancelled"
),
"type": "function_call_output",
},
]
@pytest.mark.asyncio
async def test_reasoning_item_id_policy_omits_follow_up_reasoning_ids() -> None:
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "tool_result")],
)
model.add_multiple_turn_outputs(
[
[
ResponseReasoningItem(
id="rs_first",
type="reasoning",
summary=[Summary(text="Thinking...", type="summary_text")],
),
get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call_first"),
],
[get_text_message("done")],
]
)
result = await Runner.run(
agent,
input="hello",
run_config=RunConfig(reasoning_item_id_policy="omit"),
)
assert result.final_output == "done"
second_request_reasoning = _find_reasoning_input_item(model.last_turn_args.get("input"))
assert second_request_reasoning is not None
assert "id" not in second_request_reasoning
history_reasoning = _find_reasoning_input_item(result.to_input_list())
assert history_reasoning is not None
assert "id" not in history_reasoning
@pytest.mark.asyncio
async def test_call_model_input_filter_can_reintroduce_reasoning_ids() -> None:
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "tool_result")],
)
model.add_multiple_turn_outputs(
[
[
ResponseReasoningItem(
id="rs_filter",
type="reasoning",
summary=[Summary(text="Thinking...", type="summary_text")],
),
get_function_tool_call("foo", json.dumps({"a": "b"}), call_id="call_filter"),
],
[get_text_message("done")],
]
)
def reintroduce_reasoning_id(data: Any) -> Any:
updated_input: list[TResponseInputItem] = []
for item in data.model_data.input:
if isinstance(item, dict) and item.get("type") == "reasoning" and "id" not in item:
updated_input.append(cast(TResponseInputItem, {**item, "id": "rs_reintroduced"}))
else:
updated_input.append(item)
data.model_data.input = updated_input
return data.model_data
result = await Runner.run(
agent,
input="hello",
run_config=RunConfig(
reasoning_item_id_policy="omit",
call_model_input_filter=reintroduce_reasoning_id,
),
)
assert result.final_output == "done"
second_request_reasoning = _find_reasoning_input_item(model.last_turn_args.get("input"))
assert second_request_reasoning is not None
assert second_request_reasoning.get("id") == "rs_reintroduced"
history_reasoning = _find_reasoning_input_item(result.to_input_list())
assert history_reasoning is not None
assert "id" not in history_reasoning
class _RevokedReasoningIdModel(FakeModel):
"""FakeModel that 404s like the Responses API when a revoked reasoning ID is replayed."""
def __init__(self) -> None:
super().__init__()
self.revoked_reasoning_ids: set[str] = set()
async def get_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
*args: Any,
**kwargs: Any,
) -> ModelResponse:
if isinstance(input, list):
for item in input:
if not isinstance(item, dict) or item.get("type") != "reasoning":
continue
item_id = item.get("id")
if item_id in self.revoked_reasoning_ids:
message = f"Item with id '{item_id}' not found."
body = {"error": {"message": message, "type": "invalid_request_error"}}
raise NotFoundError(
message,
response=httpx.Response(
404,
request=httpx.Request("POST", "https://api.openai.com/v1/responses"),
json=body,
),
body=body,
)
return await super().get_response(system_instructions, input, *args, **kwargs)
@pytest.mark.asyncio
async def test_omit_policy_strips_reasoning_ids_already_stored_in_the_session() -> None:
"""Adopting `omit` must also cover reasoning IDs a session recorded before it was set.
Reproduces https://github.com/openai/openai-agents-python/issues/2020: a triage agent hands
off, its empty-summary reasoning item is persisted to the session, the server later drops the
item, and every later turn of that conversation fails with
`404 Item with id 'rs_...' not found`.
"""
model = _RevokedReasoningIdModel()
specialist = Agent(name="specialist", model=model)
triage = Agent(name="triage", model=model, handoffs=[specialist])
session = SQLiteSession("issue-2020")
# Turn 1 predates the mitigation, so the session records the reasoning ID.
model.add_multiple_turn_outputs(
[
[
ResponseReasoningItem(id="rs_triage", type="reasoning", summary=[]),
get_handoff_tool_call(specialist),
],
[get_text_message("handled")],
]
)
first = await Runner.run(triage, input="hello", session=session)
assert first.final_output == "handled"
stored_reasoning = _find_reasoning_input_item(await session.get_items())
assert stored_reasoning is not None
assert stored_reasoning.get("id") == "rs_triage"
# The server no longer resolves that reasoning item.
model.revoked_reasoning_ids.add("rs_triage")
# Turn 2 opts into the documented mitigation for this failure.
model.add_multiple_turn_outputs([[get_text_message("done")]])
second = await Runner.run(
triage,
input="anything else?",
session=session,
run_config=RunConfig(reasoning_item_id_policy="omit"),
)
assert second.final_output == "done"
replayed_reasoning = _find_reasoning_input_item(model.last_turn_args.get("input"))
assert replayed_reasoning is not None
assert "id" not in replayed_reasoning
@pytest.mark.asyncio
async def test_resumed_run_uses_serialized_reasoning_item_id_policy() -> None:
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "ok"
agent = Agent(
name="test",
model=model,
tools=[approval_tool],
)
model.add_multiple_turn_outputs(
[
[
ResponseReasoningItem(
id="rs_resume",
type="reasoning",
summary=[Summary(text="Thinking...", type="summary_text")],
),
get_function_tool_call(
"approval_tool",
json.dumps({}),
call_id="call_resume",
),
],
[get_text_message("done")],
]
)
first_run = await Runner.run(
agent,
input="hello",
run_config=RunConfig(reasoning_item_id_policy="omit"),
)
assert len(first_run.interruptions) == 1
state = first_run.to_state()
state.approve(first_run.interruptions[0])
restored_state = await RunState.from_string(agent, state.to_string())
resumed = await Runner.run(agent, restored_state)
assert resumed.final_output == "done"
second_request_reasoning = _find_reasoning_input_item(model.last_turn_args.get("input"))
assert second_request_reasoning is not None
assert "id" not in second_request_reasoning
@pytest.mark.asyncio
async def test_pending_approval_skips_tool_input_guardrails_by_default() -> None:
model = FakeModel()
guardrail_runs = 0
@tool_input_guardrail
def count_guardrail(_data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
nonlocal guardrail_runs
guardrail_runs += 1
return ToolGuardrailFunctionOutput.allow()
@function_tool(
name_override="approval_tool",
needs_approval=True,
tool_input_guardrails=[count_guardrail],
)
def approval_tool() -> str:
return "ok"
agent = Agent(name="test", model=model, tools=[approval_tool])
model.set_next_output([get_function_tool_call("approval_tool", "{}", call_id="call_default")])
result = await Runner.run(agent, "hello")
assert len(result.interruptions) == 1
assert guardrail_runs == 0
assert result.tool_input_guardrail_results == []
@pytest.mark.asyncio
async def test_pre_approval_tool_input_guardrails_can_reject_before_pending_approval() -> None:
model = FakeModel()
executed = False
@tool_input_guardrail
def reject_guardrail(_data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.reject_content("blocked before approval")
@function_tool(
name_override="approval_tool",
needs_approval=True,
tool_input_guardrails=[reject_guardrail],
)
def approval_tool() -> str:
nonlocal executed
executed = True
return "ok"
agent = Agent(name="test", model=model, tools=[approval_tool])
model.add_multiple_turn_outputs(
[
[get_function_tool_call("approval_tool", "{}", call_id="call_reject")],
[get_text_message("done")],
]
)
result = await Runner.run(
agent,
"hello",
run_config=RunConfig(
tool_execution=ToolExecutionConfig(pre_approval_tool_input_guardrails=True)
),
)
assert result.final_output == "done"
assert result.interruptions == []
assert executed is False
assert len(result.tool_input_guardrail_results) == 1
assert any(
isinstance(item, ToolCallOutputItem) and item.output == "blocked before approval"
for item in result.new_items
)
@pytest.mark.asyncio
async def test_pre_approval_tool_input_guardrails_rerun_after_resume() -> None:
model = FakeModel()
guardrail_runs = 0
executed = 0
@tool_input_guardrail
def count_guardrail(_data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
nonlocal guardrail_runs
guardrail_runs += 1
return ToolGuardrailFunctionOutput.allow()
@function_tool(
name_override="approval_tool",
needs_approval=True,
tool_input_guardrails=[count_guardrail],
)
def approval_tool() -> str:
nonlocal executed
executed += 1
return "ok"
agent = Agent(name="test", model=model, tools=[approval_tool])
model.add_multiple_turn_outputs(
[
[get_function_tool_call("approval_tool", "{}", call_id="call_resume")],
[get_text_message("done")],
]
)
run_config = RunConfig(
tool_execution=ToolExecutionConfig(pre_approval_tool_input_guardrails=True)
)
first = await Runner.run(agent, "hello", run_config=run_config)
assert len(first.interruptions) == 1
assert guardrail_runs == 1
assert executed == 0
assert len(first.tool_input_guardrail_results) == 1
state = first.to_state()
state.approve(first.interruptions[0])
restored_state = await RunState.from_string(agent, state.to_string())
resumed = await Runner.run(agent, restored_state, run_config=run_config)
assert resumed.final_output == "done"
assert guardrail_runs == 2
assert executed == 1
assert len(resumed.tool_input_guardrail_results) == 1
@pytest.mark.asyncio
async def test_tool_call_context_includes_current_agent() -> None:
model = FakeModel()
captured_contexts: list[ToolContext[Any]] = []
@function_tool(name_override="foo")
def foo(context: ToolContext[Any]) -> str:
captured_contexts.append(context)
return "tool_result"
agent = Agent(
name="test",
model=model,
tools=[foo],
)
model.add_multiple_turn_outputs(
[
[get_function_tool_call("foo", "{}")],
[get_text_message("done")],
]
)
result = await Runner.run(agent, input="user_message")
assert result.final_output == "done"
assert len(captured_contexts) == 1
assert captured_contexts[0].agent is agent
@pytest.mark.asyncio
async def test_handoffs():
model = FakeModel()
agent_1 = Agent(
name="test",
model=model,
)
agent_2 = Agent(
name="test",
model=model,
)
agent_3 = Agent(
name="test",
model=model,
handoffs=[agent_1, agent_2],
tools=[get_function_tool("some_function", "result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a tool call
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
# Second turn: a message and a handoff
[get_text_message("a_message"), get_handoff_tool_call(agent_1)],
# Third turn: text message
[get_text_message("done")],
]
)
result = await Runner.run(agent_3, input="user_message")
assert result.final_output == "done"
assert len(result.raw_responses) == 3, "should have three model responses"
assert len(result.to_input_list()) == 7, (
"should have 7 inputs: summary message, tool call, tool result, message, handoff, "
"handoff result, and done message"
)
assert result.last_agent == agent_1, "should have handed off to agent_1"
@pytest.mark.asyncio
async def test_nested_handoff_filters_model_input_but_preserves_session_items():
model = FakeModel()
delegate = Agent(
name="delegate",
model=model,
)
triage = Agent(
name="triage",
model=model,
handoffs=[delegate],
tools=[get_function_tool("some_function", "result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a tool call.
[get_function_tool_call("some_function", json.dumps({"a": "b"}))],
# Second turn: a message and a handoff.
[get_text_message("a_message"), get_handoff_tool_call(delegate)],
# Third turn: final message.
[get_text_message("done")],
]
)
model_input_types: list[list[str]] = []
def capture_model_input(data):
types: list[str] = []
for item in data.model_data.input:
if isinstance(item, dict):
item_type = item.get("type")
if isinstance(item_type, str):
types.append(item_type)
model_input_types.append(types)
return data.model_data
session = SimpleListSession()
result = await Runner.run(
triage,
input="user_message",
run_config=RunConfig(
nest_handoff_history=True,
call_model_input_filter=capture_model_input,
),
session=session,
)
assert result.final_output == "done"
assert len(model_input_types) >= 3
handoff_input_types = model_input_types[2]
assert "function_call" not in handoff_input_types
assert "function_call_output" not in handoff_input_types
assert any(isinstance(item, ToolCallOutputItem) for item in result.new_items)
assert any(isinstance(item, HandoffOutputItem) for item in result.new_items)
session_items = await session.get_items()
has_function_call_output = any(
isinstance(item, dict) and item.get("type") == "function_call_output"
for item in session_items
)
assert has_function_call_output
@pytest.mark.asyncio
async def test_nested_handoff_filters_reasoning_items_from_model_input():
model = FakeModel()
delegate = Agent(
name="delegate",
model=model,
)
triage = Agent(
name="triage",
model=model,
handoffs=[delegate],
)
model.add_multiple_turn_outputs(
[
[
ResponseReasoningItem(
id="reasoning_1",
type="reasoning",
summary=[Summary(text="Thinking about a handoff.", type="summary_text")],
),
get_handoff_tool_call(delegate),
],
[get_text_message("done")],
]
)
captured_inputs: list[list[dict[str, Any]]] = []
def capture_model_input(data):
if isinstance(data.model_data.input, list):
captured_inputs.append(
[item for item in data.model_data.input if isinstance(item, dict)]
)
return data.model_data
result = await Runner.run(
triage,
input="user_message",
run_config=RunConfig(
nest_handoff_history=True,
call_model_input_filter=capture_model_input,
),
)
assert result.final_output == "done"
assert len(captured_inputs) >= 2
handoff_input = captured_inputs[1]
handoff_input_types = [
item["type"] for item in handoff_input if isinstance(item.get("type"), str)
]
assert "reasoning" not in handoff_input_types
@pytest.mark.asyncio
async def test_resume_preserves_filtered_model_input_after_handoff():
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool() -> str:
return "ok"
delegate = Agent(
name="delegate",
model=model,
tools=[approval_tool],
)
triage = Agent(
name="triage",
model=model,
handoffs=[delegate],
tools=[get_function_tool("some_function", "result")],
)
model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"some_function", json.dumps({"a": "b"}), call_id="triage-call"
)
],
[get_text_message("a_message"), get_handoff_tool_call(delegate)],
[get_function_tool_call("approval_tool", json.dumps({}), call_id="delegate-call")],
[get_text_message("done")],
]
)
model_input_call_ids: list[set[str]] = []
model_input_output_call_ids: list[set[str]] = []
def capture_model_input(data):
call_ids: set[str] = set()
output_call_ids: set[str] = set()
for item in data.model_data.input:
if not isinstance(item, dict):
continue
item_type = item.get("type")
call_id = item.get("call_id")
if not isinstance(call_id, str):
continue
if item_type == "function_call":
call_ids.add(call_id)
elif item_type == "function_call_output":
output_call_ids.add(call_id)
model_input_call_ids.append(call_ids)
model_input_output_call_ids.append(output_call_ids)
return data.model_data
run_config = RunConfig(
nest_handoff_history=True,
call_model_input_filter=capture_model_input,
)
first = await Runner.run(triage, input="user_message", run_config=run_config)
assert first.interruptions
state = first.to_state()
state.approve(first.interruptions[0])
resumed = await Runner.run(triage, state, run_config=run_config)
last_call_ids = model_input_call_ids[-1]
last_output_call_ids = model_input_output_call_ids[-1]
assert "triage-call" not in last_call_ids
assert "triage-call" not in last_output_call_ids
assert "delegate-call" in last_call_ids
assert "delegate-call" in last_output_call_ids
assert resumed.final_output == "done"
@pytest.mark.asyncio
async def test_resumed_state_updates_agent_after_handoff() -> None:
model = FakeModel()
@function_tool(name_override="triage_tool", needs_approval=True)
def triage_tool() -> str:
return "ok"
@function_tool(name_override="delegate_tool", needs_approval=True)
def delegate_tool() -> str:
return "ok"
delegate = Agent(
name="delegate",
model=model,
tools=[delegate_tool],
)
triage = Agent(
name="triage",
model=model,
handoffs=[delegate],
tools=[triage_tool],
)
model.add_multiple_turn_outputs(
[
[get_function_tool_call("triage_tool", "{}", call_id="triage-1")],
[get_text_message("handoff"), get_handoff_tool_call(delegate)],
[get_function_tool_call("delegate_tool", "{}", call_id="delegate-1")],
]
)
first = await Runner.run(triage, input="user_message")
assert first.interruptions
state = first.to_state()
state.approve(first.interruptions[0])
second = await Runner.run(triage, state)
assert second.interruptions
assert any(item.tool_name == delegate_tool.name for item in second.interruptions), (
"handoff should switch approvals to the delegate agent"
)
assert state._current_agent is delegate
@pytest.mark.asyncio
async def test_resume_with_durable_override_rewrites_local_session_history() -> None:
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
return f"result:{test}"
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = RewriteAwareSimpleSession()
model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"approval_tool", json.dumps({"test": "foo"}), call_id="call-1"
)
],
[get_text_message("done")],
]
)
first = await Runner.run(agent, input="user_message", session=session)
assert first.interruptions
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
resumed = await Runner.run(agent, state, session=session)
assert resumed.final_output == "done"
assert cast(Any, resumed.raw_responses[0].output[0]).arguments == json.dumps({"test": "foo"})
saved_items = await session.get_items()
assert saved_items[1]["type"] == "function_call"
assert cast(dict[str, Any], saved_items[1])["arguments"] == json.dumps({"test": "bar"})
assert saved_items[2]["type"] == "function_call_output"
assert saved_items[2]["call_id"] == "call-1"
assert state.to_json()["approval_argument_override_modes"] == []
async def _run_approval_recovery_case(
agent: Agent[Any],
input: str | RunState[Any, Agent[Any]],
*,
streamed: bool,
run_config: RunConfig | None = None,
session: RewriteAwareSimpleSession | None = None,
previous_response_id: str | None = "resp-root",
) -> Any:
if streamed:
result = Runner.run_streamed(
agent,
input,
session=session,
previous_response_id=previous_response_id,
run_config=run_config,
)
await consume_stream(result)
return result
return await Runner.run(
agent,
input,
session=session,
previous_response_id=previous_response_id,
run_config=run_config,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_execution_only_override_failure_state_roundtrips(streamed: bool) -> None:
@function_tool(needs_approval=True, failure_error_function=None)
async def failing_tool(value: str) -> str:
raise RuntimeError(f"failed:{value}")
model = FakeModel()
model.set_next_output(
[
get_function_tool_call(
"failing_tool",
json.dumps({"value": "original"}),
call_id="call-failure",
)
]
)
agent = Agent(name="approval_agent", model=model, tools=[failing_tool])
first = await _run_approval_recovery_case(agent, "go", streamed=streamed)
state = first.to_state()
state.approve(
first.interruptions[0],
override_arguments={"value": "corrected"},
save_override_arguments=False,
)
with pytest.raises(UserError, match="Error running tool failing_tool: failed:corrected"):
await _run_approval_recovery_case(agent, state, streamed=streamed)
restored = await RunState.from_json(agent, state.to_json())
assert restored.to_json()["approval_argument_override_modes"] == []
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_execution_only_override_cancelled_state_roundtrips(streamed: bool) -> None:
started = asyncio.Event()
@function_tool(needs_approval=True, failure_error_function=None)
async def blocking_tool(value: str) -> str:
started.set()
await asyncio.Event().wait()
return value
model = FakeModel()
model.set_next_output(
[
get_function_tool_call(
"blocking_tool",
json.dumps({"value": "original"}),
call_id="call-cancelled",
)
]
)
agent = Agent(name="approval_agent", model=model, tools=[blocking_tool])
first = await _run_approval_recovery_case(agent, "go", streamed=streamed)
state = first.to_state()
state.approve(
first.interruptions[0],
override_arguments={"value": "corrected"},
save_override_arguments=False,
)
if streamed:
resumed = Runner.run_streamed(agent, state, previous_response_id="resp-root")
consumer = asyncio.create_task(consume_stream(resumed))
await started.wait()
resumed.cancel(mode="immediate")
await consumer
else:
task = asyncio.create_task(Runner.run(agent, state, previous_response_id="resp-root"))
await started.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
restored = await RunState.from_json(agent, state.to_json())
assert restored.to_json()["approval_argument_override_modes"] == []
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_execution_only_multi_call_partial_failure_preserves_unstarted_mode(
streamed: bool,
) -> None:
calls: list[tuple[str, str]] = []
@function_tool(needs_approval=True, failure_error_function=None)
async def failing_tool(value: str) -> str:
calls.append(("failing", value))
raise RuntimeError(f"failed:{value}")
@function_tool(needs_approval=True, failure_error_function=None)
async def pending_tool(value: str) -> str:
calls.append(("pending", value))
return value
model = FakeModel()
model.set_next_output(
[
get_function_tool_call(
"failing_tool",
json.dumps({"value": "original-failure"}),
call_id="call-failure",
),
get_function_tool_call(
"pending_tool",
json.dumps({"value": "original-pending"}),
call_id="call-pending",
),
]
)
agent = Agent(
name="approval_agent",
model=model,
tools=[failing_tool, pending_tool],
)
run_config = RunConfig(tool_execution=ToolExecutionConfig(max_function_tool_concurrency=1))
first = await _run_approval_recovery_case(
agent,
"go",
streamed=streamed,
run_config=run_config,
)
state = first.to_state()
for interruption in first.interruptions:
state.approve(
interruption,
override_arguments={
"value": (
"corrected-failure"
if interruption.tool_name == "failing_tool"
else "corrected-pending"
)
},
save_override_arguments=False,
)
with pytest.raises(UserError, match="Error running tool failing_tool: failed:corrected"):
await _run_approval_recovery_case(
agent,
state,
streamed=streamed,
run_config=run_config,
)
assert calls == [("failing", "corrected-failure")]
assert state.to_json()["approval_argument_override_modes"] == [
{"call_id": "call-pending", "mode": "execution_only"}
]
restored = await RunState.from_json(agent, state.to_json())
assert restored.to_json()["approval_argument_override_modes"] == [
{"call_id": "call-pending", "mode": "execution_only"}
]
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_durable_multi_call_partial_failure_preserves_unstarted_override(
streamed: bool,
) -> None:
calls: list[tuple[str, str]] = []
@function_tool(needs_approval=True, failure_error_function=None)
async def failing_tool(value: str) -> str:
calls.append(("failing", value))
raise RuntimeError(f"failed:{value}")
@function_tool(needs_approval=True, failure_error_function=None)
async def pending_tool(value: str) -> str:
calls.append(("pending", value))
return value
model = FakeModel()
model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"failing_tool",
json.dumps({"value": "original-failure"}),
call_id="call-failure",
),
get_function_tool_call(
"pending_tool",
json.dumps({"value": "original-pending"}),
call_id="call-pending",
),
],
[get_text_message("done")],
]
)
agent = Agent(
name="approval_agent",
model=model,
tools=[failing_tool, pending_tool],
)
session = RewriteAwareSimpleSession()
run_config = RunConfig(tool_execution=ToolExecutionConfig(max_function_tool_concurrency=1))
first = await _run_approval_recovery_case(
agent,
"go",
streamed=streamed,
run_config=run_config,
session=session,
previous_response_id=None,
)
state = first.to_state()
for interruption in first.interruptions:
state.approve(
interruption,
override_arguments={
"value": (
"corrected-failure"
if interruption.tool_name == "failing_tool"
else "corrected-pending"
)
},
)
with pytest.raises(UserError, match="Error running tool failing_tool: failed:corrected"):
await _run_approval_recovery_case(
agent,
state,
streamed=streamed,
run_config=run_config,
session=session,
previous_response_id=None,
)
assert calls == [("failing", "corrected-failure")]
serialized = state.to_json()
assert serialized["approval_argument_override_modes"] == [
{"call_id": "call-pending", "mode": "durable"}
]
assert [mutation["call_id"] for mutation in serialized["session_history_mutations"]] == [
"call-pending"
]
restored = await RunState.from_json(agent, serialized)
assert restored.to_json()["approval_argument_override_modes"] == [
{"call_id": "call-pending", "mode": "durable"}
]
assert [
mutation["call_id"] for mutation in restored.to_json()["session_history_mutations"]
] == ["call-pending"]
with pytest.raises(
ModelBehaviorError,
match="A tool call already executed, but its output was not committed",
):
await _run_approval_recovery_case(
agent,
restored,
streamed=streamed,
run_config=run_config,
session=session,
previous_response_id=None,
)
assert calls == [("failing", "corrected-failure")]
assert [
mutation["call_id"] for mutation in restored.to_json()["session_history_mutations"]
] == ["call-pending"]
class _TrackingRewriteSession(RewriteAwareSimpleSession):
def __init__(self) -> None:
super().__init__()
self.rewrite_calls = 0
async def apply_history_mutations(self, args: Any) -> bool:
self.rewrite_calls += 1
return await super().apply_history_mutations(args)
async def _build_nested_approval_override_state(
case_id: str,
) -> tuple[Agent[Any], RunState[Any, Agent[Any]], _TrackingRewriteSession, list[str]]:
calls: list[str] = []
@function_tool(needs_approval=True)
async def inner_tool(value: str) -> str:
calls.append(value)
return value
nested_model = FakeModel()
nested_agent = Agent(name="nested", model=nested_model, tools=[inner_tool])
nested_model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"inner_tool",
json.dumps({"value": "original"}),
call_id=f"inner-call-{case_id}",
)
],
[get_text_message("nested done")],
]
)
session = _TrackingRewriteSession()
run_config = RunConfig(tracing_disabled=True)
outer_model = FakeModel()
outer_agent = Agent(
name="outer",
model=outer_model,
tools=[
nested_agent.as_tool(
tool_name="nested_tool",
tool_description="Run the nested agent.",
needs_approval=True,
session=session,
run_config=run_config,
failure_error_function=None,
)
],
)
outer_model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"nested_tool",
json.dumps({"input": "go"}),
call_id=f"outer-call-{case_id}",
)
],
[get_text_message("done")],
]
)
first = await Runner.run(outer_agent, "start", run_config=run_config)
first_state = first.to_state()
first_state.approve(first.interruptions[0], always_approve=True)
second = await Runner.run(outer_agent, first_state, run_config=run_config)
assert second.interruptions
return outer_agent, second.to_state(), session, calls
@pytest.mark.asyncio
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "serialized"])
async def test_nested_durable_override_rewrites_local_session_history(
round_trip: bool,
) -> None:
outer_agent, state, session, calls = await _build_nested_approval_override_state(
f"durable-{round_trip}"
)
state.approve(state.get_interruptions()[0], override_arguments={"value": "corrected"})
if round_trip:
state = await RunState.from_json(outer_agent, state.to_json())
resumed = await Runner.run(
outer_agent,
state,
run_config=RunConfig(tracing_disabled=True),
)
assert resumed.final_output == "done"
assert resumed.interruptions == []
assert calls == ["corrected"]
assert session.rewrite_calls == 1
saved_calls = [
item
for item in await session.get_items()
if isinstance(item, dict) and item.get("type") == "function_call"
]
assert len(saved_calls) == 1
assert saved_calls[0]["arguments"] == json.dumps({"value": "corrected"})
@pytest.mark.asyncio
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "serialized"])
async def test_nested_execution_only_override_rejects_local_session_history(
round_trip: bool,
) -> None:
outer_agent, state, session, calls = await _build_nested_approval_override_state(
f"execution-only-{round_trip}"
)
state.approve(
state.get_interruptions()[0],
override_arguments={"value": "corrected"},
save_override_arguments=False,
)
if round_trip:
state = await RunState.from_json(outer_agent, state.to_json())
with pytest.raises(UserError, match="save_override_arguments=False is only supported"):
await Runner.run(
outer_agent,
state,
run_config=RunConfig(tracing_disabled=True),
)
assert calls == []
assert session.rewrite_calls == 0
@pytest.mark.asyncio
async def test_invalid_nested_override_preserves_cached_run_result() -> None:
from agents.agent_tool_state import peek_agent_tool_run_result
outer_agent, state, _session, calls = await _build_nested_approval_override_state(
"invalid-override"
)
assert state._last_processed_response is not None
outer_call = state._last_processed_response.functions[0].tool_call
pending_result = peek_agent_tool_run_result(
outer_call,
scope_id=state._agent_tool_state_scope_id,
)
assert pending_result is not None
with pytest.raises(UserError, match="cannot be used together with always_approve"):
state.approve(
state.get_interruptions()[0],
always_approve=True,
override_arguments={"value": "corrected"},
)
assert (
peek_agent_tool_run_result(
outer_call,
scope_id=state._agent_tool_state_scope_id,
)
is pending_result
)
assert calls == []
@pytest.mark.asyncio
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "serialized"])
async def test_nested_execution_only_sync_cancellation_state_roundtrips(
round_trip: bool,
) -> None:
from agents.agent_tool_state import peek_agent_tool_run_result
config = RunConfig(tracing_disabled=True)
case_id = "serialized" if round_trip else "live"
tool_started = asyncio.Event()
tool_stopped = asyncio.Event()
calls: list[str] = []
@function_tool(needs_approval=True, failure_error_function=None)
async def inner_tool(value: str) -> str:
calls.append(value)
tool_started.set()
try:
await asyncio.Event().wait()
finally:
tool_stopped.set()
return value
nested_model = FakeModel()
nested_agent = Agent(name="nested", model=nested_model, tools=[inner_tool])
nested_model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"inner_tool",
json.dumps({"value": "original"}),
call_id=f"inner-call-{case_id}",
)
],
[get_text_message("nested done")],
]
)
outer_model = FakeModel()
outer_agent = Agent(
name="outer",
model=outer_model,
tools=[
nested_agent.as_tool(
tool_name="nested_tool",
tool_description="Run the nested agent.",
needs_approval=True,
session=ServerManagedSimpleSession(),
run_config=config,
failure_error_function=None,
)
],
)
outer_model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"nested_tool",
json.dumps({"input": "go"}),
call_id=f"outer-call-{case_id}",
)
],
[get_text_message("outer done")],
]
)
first = await Runner.run(outer_agent, "go", run_config=config)
state = first.to_state()
state.approve(first.interruptions[0], always_approve=True)
second = await Runner.run(outer_agent, state, run_config=config)
state = second.to_state()
state.approve(
state.get_interruptions()[0],
override_arguments={"value": "corrected"},
save_override_arguments=False,
)
if round_trip:
state = await RunState.from_json(outer_agent, state.to_json())
resume_task = asyncio.create_task(Runner.run(outer_agent, state, run_config=config))
try:
await asyncio.wait_for(tool_started.wait(), timeout=2)
resume_task.cancel()
with pytest.raises(asyncio.CancelledError):
await resume_task
immediate_payload = state.to_json()
nested_payload = immediate_payload["last_processed_response"]["functions"][0][
"agent_run_state"
]
assert nested_payload["approval_argument_override_modes"] == []
inner_invocation = nested_payload["context"]["tool_invocations"][f"inner-call-{case_id}"]
assert inner_invocation["executed"] is True
assert inner_invocation["completed"] is False
restored = await RunState.from_json(outer_agent, immediate_payload)
assert restored._last_processed_response is not None
restored_outer_call = restored._last_processed_response.functions[0].tool_call
assert (
peek_agent_tool_run_result(
restored_outer_call,
scope_id=restored._agent_tool_state_scope_id,
)
is not None
)
finally:
if not resume_task.done():
resume_task.cancel()
await asyncio.gather(resume_task, return_exceptions=True)
await asyncio.wait_for(tool_stopped.wait(), timeout=2)
assert calls == ["corrected"]
@pytest.mark.asyncio
async def test_durable_override_rewrite_fails_before_tool_execution() -> None:
calls: list[str] = []
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
class FailingRewriteSession(RewriteAwareSimpleSession):
async def apply_history_mutations(self, _args: Any) -> bool:
raise RuntimeError("rewrite failed")
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = FailingRewriteSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
with pytest.raises(RuntimeError, match="rewrite failed"):
await Runner.run(agent, state, session=session)
assert calls == []
assert state._get_session_history_mutations()
@pytest.mark.asyncio
async def test_durable_override_missing_history_fails_before_tool_execution() -> None:
calls: list[str] = []
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = RewriteAwareSimpleSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
await session.clear_session()
with pytest.raises(ValueError, match="did not match the expected function call"):
await Runner.run(agent, state, session=session)
assert calls == []
assert state._get_session_history_mutations()
@pytest.mark.asyncio
async def test_durable_override_requires_rewrite_confirmation_before_tool_execution() -> None:
calls: list[str] = []
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
class UnconfirmedRewriteSession(RewriteAwareSimpleSession):
async def apply_history_mutations(self, _args: Any) -> bool:
return False
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = UnconfirmedRewriteSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
with pytest.raises(UserError, match="did not confirm that every target call was rewritten"):
await Runner.run(agent, state, session=session)
assert calls == []
assert state._get_session_history_mutations()
@pytest.mark.asyncio
async def test_malformed_durable_override_state_fails_before_tool_execution() -> None:
calls: list[str] = []
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = RewriteAwareSimpleSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
serialized = state.to_json()
serialized["session_history_mutations"][0]["replacement"] = None
with pytest.raises(UserError, match="invalid function-call replacement"):
await RunState.from_json(agent, serialized)
assert calls == []
@pytest.mark.asyncio
async def test_missing_durable_override_mutation_fails_before_tool_execution() -> None:
calls: list[str] = []
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = RewriteAwareSimpleSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
serialized = state.to_json()
serialized["session_history_mutations"] = []
with pytest.raises(UserError, match="is inconsistent with pending function calls"):
await RunState.from_json(agent, serialized)
assert calls == []
@pytest.mark.asyncio
async def test_mismatched_durable_override_state_fails_before_tool_execution() -> None:
calls: list[str] = []
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = RewriteAwareSimpleSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
serialized = state.to_json()
serialized["session_history_mutations"][0]["replacement"]["arguments"] = '{"test":"different"}'
with pytest.raises(UserError, match="does not match the pending function call"):
await RunState.from_json(agent, serialized)
assert calls == []
@pytest.mark.asyncio
async def test_processed_durable_override_mismatch_fails_before_session_rewrite() -> None:
calls: list[str] = []
class TrackingRewriteSession(RewriteAwareSimpleSession):
def __init__(self) -> None:
super().__init__()
self.rewrite_calls = 0
async def apply_history_mutations(self, args: Any) -> bool:
self.rewrite_calls += 1
return await super().apply_history_mutations(args)
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = TrackingRewriteSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
serialized = state.to_json()
serialized["last_processed_response"]["functions"][0]["tool_call"]["arguments"] = (
'{"test":"different"}'
)
with pytest.raises(UserError, match="does not match the pending function call"):
await RunState.from_json(agent, serialized)
assert session.rewrite_calls == 0
assert calls == []
@pytest.mark.asyncio
@pytest.mark.parametrize("remove_canonical_call", [False, True])
async def test_unmarked_execution_only_override_state_fails_before_tool_execution(
remove_canonical_call: bool,
) -> None:
calls: list[str] = []
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = SimpleListSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(
first.interruptions[0],
override_arguments={"test": "bar"},
save_override_arguments=False,
)
serialized = state.to_json()
serialized["approval_argument_override_modes"] = []
if remove_canonical_call:
serialized["generated_items"] = []
serialized["generated_session_item_indexes"] = []
with pytest.raises(UserError, match="is inconsistent with pending function calls"):
await RunState.from_json(agent, serialized)
assert calls == []
@pytest.mark.asyncio
async def test_resume_rejects_execution_only_override_without_server_managed_history() -> None:
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
return f"result:{test}"
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"approval_tool", json.dumps({"test": "foo"}), call_id="call-1"
)
],
[get_text_message("done")],
]
)
first = await Runner.run(agent, input="user_message")
assert first.interruptions
state = first.to_state()
state.approve(
first.interruptions[0],
override_arguments={"test": "bar"},
save_override_arguments=False,
)
with pytest.raises(UserError, match="save_override_arguments=False is only supported"):
await Runner.run(agent, state)
@pytest.mark.asyncio
async def test_resume_supports_execution_only_override_with_server_managed_session() -> None:
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
return f"result:{test}"
agent = Agent(
name="approval_agent",
model=model,
tools=[approval_tool],
tool_use_behavior="stop_on_first_tool",
)
session = ServerManagedSimpleSession()
model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"approval_tool", json.dumps({"test": "foo"}), call_id="call-1"
)
],
]
)
first = await Runner.run(agent, input="user_message", session=session)
assert first.interruptions
state = first.to_state()
state.approve(
first.interruptions[0],
override_arguments={"test": "bar"},
save_override_arguments=False,
)
resumed = await Runner.run(agent, state, session=session)
assert resumed.final_output == "result:bar"
saved_items = await session.get_items()
assert cast(dict[str, Any], saved_items[1])["arguments"] == json.dumps({"test": "foo"})
assert state.to_json()["approval_argument_override_modes"] == []
@pytest.mark.asyncio
async def test_resume_rejects_durable_override_with_server_managed_session() -> None:
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
return f"result:{test}"
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = ServerManagedSimpleSession()
model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"approval_tool", json.dumps({"test": "foo"}), call_id="call-1"
)
],
]
)
first = await Runner.run(agent, input="user_message", session=session)
assert first.interruptions
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
with pytest.raises(UserError, match="Server-managed conversations cannot persist"):
await Runner.run(agent, state, session=session)
@pytest.mark.asyncio
async def test_resume_supports_execution_only_override_with_previous_response_id() -> None:
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
return f"result:{test}"
agent = Agent(
name="approval_agent",
model=model,
tools=[approval_tool],
tool_use_behavior="stop_on_first_tool",
)
model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"approval_tool", json.dumps({"test": "foo"}), call_id="call-1"
)
],
]
)
first = await Runner.run(agent, input="user_message", previous_response_id="resp-root")
assert first.interruptions
state = first.to_state()
state.approve(
first.interruptions[0],
override_arguments={"test": "bar"},
save_override_arguments=False,
)
resumed = await Runner.run(agent, state)
assert resumed.final_output == "result:bar"
@pytest.mark.asyncio
async def test_resume_rejects_durable_override_for_non_rewrite_aware_session() -> None:
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
return f"result:{test}"
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = SimpleListSession()
model.add_multiple_turn_outputs(
[
[
get_function_tool_call(
"approval_tool", json.dumps({"test": "foo"}), call_id="call-1"
)
],
[get_text_message("done")],
]
)
first = await Runner.run(agent, input="user_message", session=session)
assert first.interruptions
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
with pytest.raises(UserError, match="supports expected history rewrites"):
await Runner.run(agent, state, session=session)
@pytest.mark.asyncio
async def test_resume_rejects_method_only_history_rewrite_session() -> None:
"""A method-name collision must not opt a session into expected rewrites."""
class MethodOnlyRewriteSession(SimpleListSession):
async def apply_history_mutations(self, _args: Any) -> bool:
return True
model = FakeModel()
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
return f"result:{test}"
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = MethodOnlyRewriteSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=session)
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
with pytest.raises(UserError, match="supports expected history rewrites"):
await Runner.run(agent, state, session=session)
@pytest.mark.asyncio
async def test_resume_rejects_dynamically_forwarded_rewrite_capability() -> None:
"""A transforming proxy must explicitly implement compatible rewrite semantics."""
class ForwardingSession:
def __init__(self) -> None:
self.session_id = "forwarding"
self.underlying_session = RewriteAwareSimpleSession(self.session_id)
def __getattr__(self, name: str) -> Any:
return getattr(self.underlying_session, name)
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
return f"result:{test}"
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = ForwardingSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=cast(Any, session))
state = first.to_state()
state.approve(first.interruptions[0], override_arguments={"test": "bar"})
with pytest.raises(UserError, match="supports expected history rewrites"):
await Runner.run(agent, state, session=cast(Any, session))
@pytest.mark.asyncio
async def test_resume_rejects_dynamically_forwarded_server_managed_capability() -> None:
"""A transforming proxy must explicitly declare server-managed history semantics."""
calls: list[str] = []
class ForwardingSession:
def __init__(self) -> None:
self.session_id = "forwarding-server"
self.underlying_session = ServerManagedSimpleSession(self.session_id)
def __getattr__(self, name: str) -> Any:
return getattr(self.underlying_session, name)
@function_tool(name_override="approval_tool", needs_approval=True)
def approval_tool(test: str) -> str:
calls.append(test)
return f"result:{test}"
model = FakeModel()
agent = Agent(name="approval_agent", model=model, tools=[approval_tool])
session = ForwardingSession()
model.set_next_output(
[get_function_tool_call("approval_tool", '{"test":"foo"}', call_id="call-1")]
)
first = await Runner.run(agent, input="user_message", session=cast(Any, session))
state = first.to_state()
state.approve(
first.interruptions[0],
override_arguments={"test": "bar"},
save_override_arguments=False,
)
with pytest.raises(UserError, match="save_override_arguments=False is only supported"):
await Runner.run(agent, state, session=cast(Any, session))
assert calls == []
class Foo(TypedDict):
bar: str
@pytest.mark.asyncio
async def test_structured_output():
model = FakeModel()
agent_1 = Agent(
name="test",
model=model,
tools=[get_function_tool("bar", "bar_result")],
output_type=Foo,
)
agent_2 = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "foo_result")],
handoffs=[agent_1],
)
model.add_multiple_turn_outputs(
[
# First turn: a tool call
[
get_function_tool_call(
"foo",
json.dumps({"bar": "baz"}),
call_id="call_foo",
)
],
# Second turn: a message and a handoff
[get_text_message("a_message"), get_handoff_tool_call(agent_1)],
# Third turn: tool call with preamble message
[
get_text_message(json.dumps(Foo(bar="preamble"))),
get_function_tool_call(
"bar",
json.dumps({"bar": "baz"}),
call_id="call_bar",
),
],
# Fourth turn: structured output
[get_final_output_message(json.dumps(Foo(bar="baz")))],
]
)
result = await Runner.run(
agent_2,
input=[
get_text_input_item("user_message"),
get_text_input_item("another_message"),
],
run_config=RunConfig(nest_handoff_history=True),
)
assert result.final_output == Foo(bar="baz")
assert len(result.raw_responses) == 4, "should have four model responses"
assert len(result.to_input_list()) == 11, (
"should preserve ordered history segments plus function calls, messages, handoff items, "
"and the final output without replaying the carried-forward message twice"
)
assert len(result.to_input_list(mode="normalized")) == 7, (
"should have normalized replay input: conversation summary, carried-forward message, "
"handoff summary, preamble message, tool call, tool call result, final output"
)
assert result.last_agent == agent_1, "should have handed off to agent_1"
assert result.final_output == Foo(bar="baz"), "should have structured output"
def remove_new_items(handoff_input_data: HandoffInputData) -> HandoffInputData:
return HandoffInputData(
input_history=handoff_input_data.input_history,
pre_handoff_items=(),
new_items=(),
run_context=handoff_input_data.run_context,
)
@pytest.mark.asyncio
async def test_handoff_filters():
model = FakeModel()
agent_1 = Agent(
name="test",
model=model,
)
agent_2 = Agent(
name="test",
model=model,
handoffs=[
handoff(
agent=agent_1,
input_filter=remove_new_items,
)
],
)
model.add_multiple_turn_outputs(
[
[get_text_message("1"), get_text_message("2"), get_handoff_tool_call(agent_1)],
[get_text_message("last")],
]
)
result = await Runner.run(agent_2, input="user_message")
assert result.final_output == "last"
assert len(result.raw_responses) == 2, "should have two model responses"
assert len(result.to_input_list()) == 2, (
"should only have 2 inputs: orig input and last message"
)
@pytest.mark.asyncio
async def test_opt_in_handoff_history_nested_and_filters_respected():
model = FakeModel()
agent_1 = Agent(
name="delegate",
model=model,
)
agent_2 = Agent(
name="triage",
model=model,
handoffs=[agent_1],
)
model.add_multiple_turn_outputs(
[
[get_text_message("triage summary"), get_handoff_tool_call(agent_1)],
[get_text_message("resolution")],
]
)
result = await Runner.run(
agent_2,
input="user_message",
run_config=RunConfig(nest_handoff_history=True),
)
assert isinstance(result.input, list)
assert len(result.input) == 3
summary = _as_message(result.input[0])
assert summary["role"] == "assistant"
summary_content = summary["content"]
assert isinstance(summary_content, str)
assert "<CONVERSATION HISTORY>" in summary_content
assert "triage summary" not in summary_content
assert "user_message" in summary_content
assert _input_message_text(result.input[1]) == "triage summary"
handoff_summary = _input_message_text(result.input[2])
assert "transfer_to_delegate" in handoff_summary
delegate_input = model.last_turn_args["input"]
assert isinstance(delegate_input, list)
assert len(delegate_input) == 3
assert _input_message_text(delegate_input[1]) == "triage summary"
passthrough_model = FakeModel()
delegate = Agent(name="delegate", model=passthrough_model)
def passthrough_filter(data: HandoffInputData) -> HandoffInputData:
return data
triage_with_filter = Agent(
name="triage",
model=passthrough_model,
handoffs=[handoff(delegate, input_filter=passthrough_filter)],
)
passthrough_model.add_multiple_turn_outputs(
[
[get_text_message("triage summary"), get_handoff_tool_call(delegate)],
[get_text_message("resolution")],
]
)
filtered_result = await Runner.run(
triage_with_filter,
input="user_message",
run_config=RunConfig(nest_handoff_history=True),
)
assert isinstance(filtered_result.input, str)
assert filtered_result.input == "user_message"
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
async def test_falsey_per_handoff_input_filter_takes_precedence(streamed: bool) -> None:
triage_model = FakeModel()
delegate_model = FakeModel()
delegate = Agent(name="delegate", model=delegate_model)
class FalseyInputFilter:
def __init__(self) -> None:
self.call_count = 0
def __bool__(self) -> bool:
return False
def __call__(self, data: HandoffInputData) -> HandoffInputData:
self.call_count += 1
return data
per_handoff_filter = FalseyInputFilter()
def global_filter(_data: HandoffInputData) -> HandoffInputData:
raise AssertionError("The run-level filter must not replace the per-handoff filter")
triage = Agent(
name="triage",
model=triage_model,
handoffs=[handoff(delegate, input_filter=per_handoff_filter)],
)
triage_model.add_multiple_turn_outputs([[get_handoff_tool_call(delegate)]])
delegate_model.add_multiple_turn_outputs([[get_text_message("done")]])
result = await _run_agent_with_optional_streaming(
triage,
input="user_message",
streamed=streamed,
run_config=RunConfig(
handoff_input_filter=global_filter,
nest_handoff_history=True,
),
)
assert result.final_output == "done"
assert result.input == "user_message"
assert per_handoff_filter.call_count == 1
@pytest.mark.asyncio
async def test_opt_in_handoff_history_accumulates_across_multiple_handoffs():
triage_model = FakeModel()
delegate_model = FakeModel()
closer_model = FakeModel()
closer = Agent(name="closer", model=closer_model)
delegate = Agent(name="delegate", model=delegate_model, handoffs=[closer])
triage = Agent(name="triage", model=triage_model, handoffs=[delegate])
triage_model.add_multiple_turn_outputs(
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
)
delegate_model.add_multiple_turn_outputs(
[[get_text_message("delegate update"), get_handoff_tool_call(closer)]]
)
closer_model.add_multiple_turn_outputs([[get_text_message("resolution")]])
result = await Runner.run(
triage,
input="user_question",
run_config=RunConfig(nest_handoff_history=True),
)
assert result.final_output == "resolution"
assert closer_model.first_turn_args is not None
closer_input = closer_model.first_turn_args["input"]
assert isinstance(closer_input, list)
summary = _as_message(closer_input[0])
assert summary["role"] == "assistant"
summary_content = summary["content"]
assert isinstance(summary_content, str)
assert summary_content.count("<CONVERSATION HISTORY>") == 1
assert "triage summary" in summary_content
assert "delegate update" not in summary_content
assert "user_question" in summary_content
assert len(closer_input) == 3
assert _input_message_text(closer_input[1]) == "delegate update"
handoff_summary = _input_message_text(closer_input[2])
assert "transfer_to_closer" in handoff_summary
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
@pytest.mark.parametrize("nest_source", ["run_config", "handoff"], ids=["run_config", "handoff"])
async def test_server_managed_handoff_history_auto_disables_with_warning(
streamed: bool,
nest_source: str,
caplog: pytest.LogCaptureFixture,
) -> None:
triage_model = FakeModel()
delegate_model = FakeModel()
delegate = Agent(name="delegate", model=delegate_model)
run_config = RunConfig()
triage_handoffs: list[Agent[Any] | Handoff[Any, Any]]
if nest_source == "handoff":
triage_handoffs = [handoff(delegate, nest_handoff_history=True)]
else:
triage_handoffs = [delegate]
run_config = RunConfig(nest_handoff_history=True)
triage = Agent(name="triage", model=triage_model, handoffs=triage_handoffs)
triage_model.add_multiple_turn_outputs(
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
)
delegate_model.add_multiple_turn_outputs([[get_text_message("done")]])
with caplog.at_level("WARNING", logger="openai.agents"):
result = await _run_agent_with_optional_streaming(
triage,
input="user_message",
streamed=streamed,
run_config=run_config,
auto_previous_response_id=True,
)
assert result.final_output == "done"
assert "do not support nest_handoff_history" in caplog.text
assert delegate_model.first_turn_args is not None
delegate_input = delegate_model.first_turn_args["input"]
assert isinstance(delegate_input, list)
assert len(delegate_input) == 1
handoff_output = delegate_input[0]
assert handoff_output.get("type") == "function_call_output"
assert "delegate" in str(handoff_output.get("output"))
assert not any(
isinstance(item, dict)
and item.get("role") == "assistant"
and "<CONVERSATION HISTORY>" in str(item.get("content"))
for item in delegate_input
)
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"])
@pytest.mark.parametrize("filter_source", ["run_config", "handoff"], ids=["run_config", "handoff"])
async def test_server_managed_handoff_input_filters_still_raise(
streamed: bool,
filter_source: str,
) -> None:
triage_model = FakeModel()
delegate_model = FakeModel()
delegate = Agent(name="delegate", model=delegate_model)
def passthrough_filter(data: HandoffInputData) -> HandoffInputData:
return data
run_config = RunConfig()
triage_handoffs: list[Agent[Any] | Handoff[Any, Any]]
if filter_source == "handoff":
triage_handoffs = [handoff(delegate, input_filter=passthrough_filter)]
else:
triage_handoffs = [delegate]
run_config = RunConfig(handoff_input_filter=passthrough_filter)
triage = Agent(name="triage", model=triage_model, handoffs=triage_handoffs)
triage_model.add_multiple_turn_outputs(
[[get_text_message("triage summary"), get_handoff_tool_call(delegate)]]
)
delegate_model.add_multiple_turn_outputs([[get_text_message("done")]])
with pytest.raises(
UserError,
match="Server-managed conversations do not support handoff input filters",
):
await _run_agent_with_optional_streaming(
triage,
input="user_message",
streamed=streamed,
run_config=run_config,
auto_previous_response_id=True,
)
assert delegate_model.first_turn_args is None
@pytest.mark.asyncio
async def test_async_input_filter_supported():
# DO NOT rename this without updating pyproject.toml
model = FakeModel()
agent_1 = Agent(
name="test",
model=model,
)
async def on_invoke_handoff(_ctx: RunContextWrapper[Any], _input: str) -> Agent[Any]:
return agent_1
async def async_input_filter(data: HandoffInputData) -> HandoffInputData:
return data # pragma: no cover
agent_2 = Agent[None](
name="test",
model=model,
handoffs=[
Handoff(
tool_name=Handoff.default_tool_name(agent_1),
tool_description=Handoff.default_tool_description(agent_1),
input_json_schema={},
on_invoke_handoff=on_invoke_handoff,
agent_name=agent_1.name,
input_filter=async_input_filter,
)
],
)
model.add_multiple_turn_outputs(
[
[get_text_message("1"), get_text_message("2"), get_handoff_tool_call(agent_1)],
[get_text_message("last")],
]
)
result = await Runner.run(agent_2, input="user_message")
assert result.final_output == "last"
@pytest.mark.asyncio
async def test_invalid_input_filter_fails():
model = FakeModel()
agent_1 = Agent(
name="test",
model=model,
)
async def on_invoke_handoff(_ctx: RunContextWrapper[Any], _input: str) -> Agent[Any]:
return agent_1
def invalid_input_filter(data: HandoffInputData) -> HandoffInputData:
# Purposely returning a string to simulate invalid output
return "foo" # type: ignore
agent_2 = Agent[None](
name="test",
model=model,
handoffs=[
Handoff(
tool_name=Handoff.default_tool_name(agent_1),
tool_description=Handoff.default_tool_description(agent_1),
input_json_schema={},
on_invoke_handoff=on_invoke_handoff,
agent_name=agent_1.name,
input_filter=invalid_input_filter,
)
],
)
model.add_multiple_turn_outputs(
[
[get_text_message("1"), get_text_message("2"), get_handoff_tool_call(agent_1)],
[get_text_message("last")],
]
)
with pytest.raises(UserError):
await Runner.run(agent_2, input="user_message")
@pytest.mark.asyncio
async def test_non_callable_input_filter_causes_error():
model = FakeModel()
agent_1 = Agent(
name="test",
model=model,
)
async def on_invoke_handoff(_ctx: RunContextWrapper[Any], _input: str) -> Agent[Any]:
return agent_1
agent_2 = Agent[None](
name="test",
model=model,
handoffs=[
Handoff(
tool_name=Handoff.default_tool_name(agent_1),
tool_description=Handoff.default_tool_description(agent_1),
input_json_schema={},
on_invoke_handoff=on_invoke_handoff,
agent_name=agent_1.name,
# Purposely ignoring the type error here to simulate invalid input
input_filter="foo", # type: ignore
)
],
)
model.add_multiple_turn_outputs(
[
[get_text_message("1"), get_text_message("2"), get_handoff_tool_call(agent_1)],
[get_text_message("last")],
]
)
with pytest.raises(UserError):
await Runner.run(agent_2, input="user_message")
@pytest.mark.asyncio
async def test_handoff_on_input():
call_output: str | None = None
def on_input(_ctx: RunContextWrapper[Any], data: Foo) -> None:
nonlocal call_output
call_output = data["bar"]
model = FakeModel()
agent_1 = Agent(
name="test",
model=model,
)
agent_2 = Agent(
name="test",
model=model,
handoffs=[
handoff(
agent=agent_1,
on_handoff=on_input,
input_type=Foo,
)
],
)
model.add_multiple_turn_outputs(
[
[
get_text_message("1"),
get_text_message("2"),
get_handoff_tool_call(agent_1, args=json.dumps(Foo(bar="test_input"))),
],
[get_text_message("last")],
]
)
result = await Runner.run(agent_2, input="user_message")
assert result.final_output == "last"
assert call_output == "test_input", "should have called the handoff with the correct input"
@pytest.mark.asyncio
async def test_async_handoff_on_input():
call_output: str | None = None
async def on_input(_ctx: RunContextWrapper[Any], data: Foo) -> None:
nonlocal call_output
call_output = data["bar"]
model = FakeModel()
agent_1 = Agent(
name="test",
model=model,
)
agent_2 = Agent(
name="test",
model=model,
handoffs=[
handoff(
agent=agent_1,
on_handoff=on_input,
input_type=Foo,
)
],
)
model.add_multiple_turn_outputs(
[
[
get_text_message("1"),
get_text_message("2"),
get_handoff_tool_call(agent_1, args=json.dumps(Foo(bar="test_input"))),
],
[get_text_message("last")],
]
)
result = await Runner.run(agent_2, input="user_message")
assert result.final_output == "last"
assert call_output == "test_input", "should have called the handoff with the correct input"
@pytest.mark.asyncio
async def test_wrong_params_on_input_causes_error():
agent_1 = Agent(
name="test",
)
def _on_handoff_too_many_params(ctx: RunContextWrapper[Any], foo: Foo, bar: str) -> None:
pass
with pytest.raises(UserError):
handoff(
agent_1,
input_type=Foo,
# Purposely ignoring the type error here to simulate invalid input
on_handoff=_on_handoff_too_many_params, # type: ignore
)
def on_handoff_too_few_params(ctx: RunContextWrapper[Any]) -> None:
pass
with pytest.raises(UserError):
handoff(
agent_1,
input_type=Foo,
# Purposely ignoring the type error here to simulate invalid input
on_handoff=on_handoff_too_few_params, # type: ignore
)
@pytest.mark.asyncio
async def test_invalid_handoff_input_json_causes_error():
agent = Agent(name="test")
h = handoff(agent, input_type=Foo, on_handoff=lambda _ctx, _input: None)
with pytest.raises(ModelBehaviorError):
await h.on_invoke_handoff(
RunContextWrapper(None),
# Purposely ignoring the type error here to simulate invalid input
None, # type: ignore
)
with pytest.raises(ModelBehaviorError):
await h.on_invoke_handoff(RunContextWrapper(None), "invalid")
@pytest.mark.asyncio
async def test_input_guardrail_tripwire_triggered_causes_exception():
def guardrail_function(
context: RunContextWrapper[Any], agent: Agent[Any], input: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(
output_info=None,
tripwire_triggered=True,
)
agent = Agent(
name="test", input_guardrails=[InputGuardrail(guardrail_function=guardrail_function)]
)
model = FakeModel()
model.set_next_output([get_text_message("user_message")])
with pytest.raises(InputGuardrailTripwireTriggered):
await Runner.run(agent, input="user_message")
@pytest.mark.asyncio
async def test_input_guardrail_tripwire_does_not_save_assistant_message_to_session():
async def guardrail_function(
context: RunContextWrapper[Any], agent: Agent[Any], input: Any
) -> GuardrailFunctionOutput:
# Delay to ensure the agent has time to produce output before the guardrail finishes.
await asyncio.sleep(0.01)
return GuardrailFunctionOutput(
output_info=None,
tripwire_triggered=True,
)
session = SimpleListSession()
model = FakeModel()
model.set_next_output([get_text_message("should_not_be_saved")])
agent = Agent(
name="test",
model=model,
input_guardrails=[InputGuardrail(guardrail_function=guardrail_function)],
)
with pytest.raises(InputGuardrailTripwireTriggered):
await Runner.run(agent, input="user_message", session=session)
items = await session.get_items()
assert len(items) == 1
first_item = cast(dict[str, Any], items[0])
assert "role" in first_item
assert first_item["role"] == "user"
@pytest.mark.asyncio
async def test_prepare_input_with_session_keeps_orphan_output_without_limit():
history_item = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_prepare",
"output": "ok",
},
)
session = SimpleListSession(history=[history_item])
prepared_input, session_items = await prepare_input_with_session("hello", session, None)
assert prepared_input == [history_item, {"role": "user", "content": "hello"}]
assert session_items == [{"role": "user", "content": "hello"}]
@pytest.mark.asyncio
async def test_prepare_input_with_session_drops_limited_orphan_history_function_call_outputs():
history_item = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_prepare",
"output": "ok",
},
)
session = SimpleListSession(history=[history_item])
prepared_input, session_items = await prepare_input_with_session(
"hello",
session,
None,
SessionSettings(limit=1),
)
assert prepared_input == [{"role": "user", "content": "hello"}]
assert session_items == [{"role": "user", "content": "hello"}]
@pytest.mark.asyncio
async def test_prepare_input_with_session_preserves_new_function_call_outputs():
new_output = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_prepare",
"output": "ok",
},
)
session = SimpleListSession()
prepared_input, session_items = await prepare_input_with_session(
[new_output],
session,
None,
SessionSettings(limit=1),
)
assert prepared_input == [new_output]
assert session_items == [new_output]
@pytest.mark.asyncio
async def test_prepare_input_with_session_leaves_custom_callback_output_unchanged():
history_output = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_callback",
"output": "ok",
},
)
session = SimpleListSession(history=[history_output])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
return history + new_input
prepared_input, session_items = await prepare_input_with_session(
"hello",
session,
callback,
SessionSettings(limit=1),
)
assert prepared_input == [history_output, {"role": "user", "content": "hello"}]
assert session_items == [{"role": "user", "content": "hello"}]
@pytest.mark.asyncio
async def test_prepare_input_with_session_drops_output_for_program_owned_call_pruned_with_parent():
program = cast(
TResponseInputItem,
{
"type": "program",
"call_id": "program_orphan",
"code": "return await tools.lookup({});",
"fingerprint": "fingerprint:orphan",
},
)
function_call = cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_orphan",
"name": "lookup",
"arguments": "{}",
"caller": {"type": "program", "caller_id": "program_orphan"},
},
)
function_output = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_orphan",
"output": "ok",
},
)
session = SimpleListSession(history=[program, function_call, function_output])
prepared_input, session_items = await prepare_input_with_session(
"hello",
session,
None,
SessionSettings(limit=3),
)
assert prepared_input == [{"role": "user", "content": "hello"}]
assert session_items == [{"role": "user", "content": "hello"}]
@pytest.mark.asyncio
async def test_prepare_input_with_session_keeps_paired_history_function_call_outputs():
function_call = cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_prepare",
"name": "lookup",
"arguments": "{}",
},
)
function_call_output = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_prepare",
"output": "ok",
},
)
session = SimpleListSession(history=[function_call, function_call_output])
prepared_input, session_items = await prepare_input_with_session(
"hello",
session,
None,
SessionSettings(limit=2),
)
assert prepared_input == [
function_call,
function_call_output,
{"role": "user", "content": "hello"},
]
assert session_items == [{"role": "user", "content": "hello"}]
@pytest.mark.asyncio
async def test_prepare_input_with_session_prefers_latest_function_call_output():
history_output = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_latest",
"output": "history-output",
},
)
session = SimpleListSession(history=[history_output])
latest_output = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_latest",
"output": "new-output",
},
)
prepared_input, session_items = await prepare_input_with_session([latest_output], session, None)
assert isinstance(prepared_input, list)
prepared_outputs = [
cast(dict[str, Any], item)
for item in prepared_input
if isinstance(item, dict)
and item.get("type") == "function_call_output"
and item.get("call_id") == "call_latest"
]
assert len(prepared_outputs) == 1
assert prepared_outputs[0]["output"] == "new-output"
assert len(session_items) == 1
assert cast(dict[str, Any], session_items[0])["output"] == "new-output"
@pytest.mark.asyncio
async def test_prepare_input_with_session_drops_orphan_function_calls():
orphan_call = cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "orphan_call",
"name": "tool_orphan",
"arguments": "{}",
},
)
session = SimpleListSession(history=[orphan_call])
prepared_input, session_items = await prepare_input_with_session("hello", session, None)
assert isinstance(prepared_input, list)
assert len(session_items) == 1
assert not any(
isinstance(item, dict)
and item.get("type") == "function_call"
and item.get("call_id") == "orphan_call"
for item in prepared_input
)
assert any(
isinstance(item, dict) and item.get("role") == "user" and item.get("content") == "hello"
for item in prepared_input
)
@pytest.mark.asyncio
async def test_prepare_input_with_session_preserves_pending_new_shell_calls() -> None:
orphan_call = cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "orphan_call",
"name": "tool_orphan",
"arguments": "{}",
},
)
pending_shell_call = cast(
TResponseInputItem,
make_shell_call("manual_shell", id_value="shell_1", commands=["echo hi"]),
)
session = SimpleListSession(history=[orphan_call])
prepared_input, session_items = await prepare_input_with_session(
[pending_shell_call],
session,
None,
)
assert isinstance(prepared_input, list)
assert session_items == [pending_shell_call]
assert not any(
isinstance(item, dict)
and item.get("type") == "function_call"
and item.get("call_id") == "orphan_call"
for item in prepared_input
)
assert any(
isinstance(item, dict)
and item.get("type") == "shell_call"
and item.get("call_id") == "manual_shell"
for item in prepared_input
)
def test_ensure_api_input_item_handles_model_dump_objects():
class _ModelDumpItem:
def model_dump(self, exclude_unset: bool = True) -> dict[str, Any]:
return {
"type": "function_call_output",
"call_id": "call_model_dump",
"output": "dumped",
}
dummy_item: Any = _ModelDumpItem()
converted = ensure_input_item_format(dummy_item)
assert converted["type"] == "function_call_output"
assert converted["output"] == "dumped"
def test_ensure_api_input_item_avoids_pydantic_serialization_warnings():
annotation = AnnotationFileCitation.model_construct(
type="container_file_citation",
file_id="file_123",
filename="result.txt",
index=0,
)
output_text = ResponseOutputText.model_construct(
type="output_text",
text="done",
annotations=[annotation],
)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always")
converted = ensure_input_item_format(cast(Any, output_text))
converted_payload = cast(dict[str, Any], converted)
assert captured == []
assert converted_payload["type"] == "output_text"
assert converted_payload["annotations"][0]["type"] == "container_file_citation"
def test_ensure_api_input_item_preserves_object_output():
payload = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_object",
"output": {"complex": "value"},
},
)
converted = ensure_input_item_format(payload)
assert converted["type"] == "function_call_output"
assert isinstance(converted["output"], dict)
assert converted["output"] == {"complex": "value"}
@pytest.mark.asyncio
async def test_prepare_input_with_session_uses_sync_callback():
history_item = cast(TResponseInputItem, {"role": "user", "content": "hi"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
first = cast(dict[str, Any], history[0])
assert first["role"] == "user"
return history + new_input
prepared, session_items = await prepare_input_with_session("second", session, callback)
assert len(prepared) == 2
last_item = cast(dict[str, Any], prepared[-1])
assert last_item["role"] == "user"
assert last_item.get("content") == "second"
# session_items should contain only the new turn input
assert len(session_items) == 1
assert cast(dict[str, Any], session_items[0]).get("role") == "user"
@pytest.mark.asyncio
async def test_prepare_input_with_session_awaits_async_callback():
history_item = cast(TResponseInputItem, {"role": "user", "content": "initial"})
session = SimpleListSession(history=[history_item])
async def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
await asyncio.sleep(0)
return history + new_input
prepared, session_items = await prepare_input_with_session("later", session, callback)
assert len(prepared) == 2
first_item = cast(dict[str, Any], prepared[0])
assert first_item["role"] == "user"
assert first_item.get("content") == "initial"
assert len(session_items) == 1
assert cast(dict[str, Any], session_items[0]).get("role") == "user"
@pytest.mark.asyncio
async def test_prepare_input_with_session_callback_drops_new_items():
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
_ = new_input
return history
prepared, session_items = await prepare_input_with_session("new", session, callback)
assert prepared == [history_item]
assert session_items == []
@pytest.mark.asyncio
async def test_prepare_input_with_session_callback_reorders_new_items():
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
return [new_input[1], history[0], new_input[0]]
new_input = [get_text_input_item("first"), get_text_input_item("second")]
prepared, session_items = await prepare_input_with_session(new_input, session, callback)
assert cast(dict[str, Any], prepared[0]).get("content") == "second"
assert cast(dict[str, Any], prepared[1]).get("content") == "history"
assert cast(dict[str, Any], prepared[2]).get("content") == "first"
assert [cast(dict[str, Any], item).get("content") for item in session_items] == [
"second",
"first",
]
@pytest.mark.asyncio
async def test_prepare_input_with_session_callback_accepts_extra_items():
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
session = SimpleListSession(history=[history_item])
extra_item = cast(TResponseInputItem, {"role": "assistant", "content": "extra"})
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
return [extra_item, history[0], new_input[0]]
prepared, session_items = await prepare_input_with_session("new", session, callback)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == [
"extra",
"history",
"new",
]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == [
"extra",
"new",
]
@pytest.mark.asyncio
async def test_prepare_input_with_session_ignores_callback_without_history():
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
_ = history
_ = new_input
return []
prepared, session_items = await prepare_input_with_session(
"new",
session,
callback,
include_history_in_prepared_input=False,
preserve_dropped_new_items=True,
)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == ["new"]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == ["new"]
@pytest.mark.asyncio
async def test_prepare_input_with_session_rejects_non_callable_callback():
session = SimpleListSession()
with pytest.raises(UserError, match="session_input_callback"):
await prepare_input_with_session("hello", session, cast(Any, "bad_callback"))
@pytest.mark.asyncio
async def test_prepare_input_with_session_rejects_non_list_callback_result():
session = SimpleListSession()
def callback(history: list[TResponseInputItem], new_input: list[TResponseInputItem]) -> str:
_ = history
_ = new_input
return "not-a-list"
with pytest.raises(UserError, match="Session input callback must return a list"):
await prepare_input_with_session("hello", session, cast(Any, callback))
@pytest.mark.asyncio
async def test_prepare_input_with_session_matches_copied_items_by_content() -> None:
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
return [
cast(TResponseInputItem, dict(cast(dict[str, Any], history[0]))),
cast(TResponseInputItem, dict(cast(dict[str, Any], new_input[0]))),
]
prepared, session_items = await prepare_input_with_session("new", session, callback)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == [
"history",
"new",
]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == ["new"]
@pytest.mark.asyncio
async def test_prepare_input_with_session_repeated_history_keeps_equal_new_item() -> None:
history_item = cast(TResponseInputItem, {"role": "user", "content": "same"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
return [history[0], history[0], new_input[0]]
prepared, session_items = await prepare_input_with_session("same", session, callback)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == [
"same",
"same",
"same",
]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == ["same"]
@pytest.mark.asyncio
async def test_prepare_input_with_session_async_callback_moves_repeated_history_item() -> None:
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
session = SimpleListSession(history=[history_item])
async def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
await asyncio.sleep(0)
moved = history.pop(0)
return [moved, new_input[0], moved]
prepared, session_items = await prepare_input_with_session("new", session, callback)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == [
"history",
"new",
"history",
]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == ["new"]
@pytest.mark.asyncio
async def test_prepare_input_with_session_history_moved_to_new_input_stays_history() -> None:
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
moved = history.pop(0)
new_input.insert(0, moved)
return new_input + [moved]
prepared, session_items = await prepare_input_with_session("new", session, callback)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == [
"history",
"new",
"history",
]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == ["new"]
@pytest.mark.asyncio
async def test_prepare_input_with_session_callback_replaces_history_item() -> None:
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
replacement = cast(TResponseInputItem, {"role": "user", "content": "summary"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
history[0] = replacement
return history + new_input
prepared, session_items = await prepare_input_with_session("new", session, callback)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == [
"summary",
"new",
]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == ["new"]
@pytest.mark.asyncio
async def test_prepare_input_with_session_extra_reconstructed_history_item_stays_new() -> None:
history_item = cast(TResponseInputItem, {"role": "user", "content": "history"})
session = SimpleListSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
rebuilt = cast(TResponseInputItem, dict(cast(dict[str, Any], history[0])))
return [history[0], rebuilt, new_input[0]]
_, session_items = await prepare_input_with_session("new", session, callback)
assert [cast(dict[str, Any], item).get("content") for item in session_items] == [
"history",
"new",
]
@pytest.mark.asyncio
async def test_prepare_input_with_openai_conversation_strips_assistant_history_ids() -> None:
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self, history: list[TResponseInputItem]) -> None:
self.history = history
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
if limit is None:
return list(self.history)
return self.history[-limit:]
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.history.extend(items)
async def pop_item(self) -> TResponseInputItem | None:
return self.history.pop() if self.history else None
async def clear_session(self) -> None:
self.history.clear()
history_item = cast(
TResponseInputItem,
{
"id": "conv_item_assistant",
"type": "message",
"role": "assistant",
"content": "history",
"provider_data": {"server": "metadata"},
},
)
user_history_item = cast(
TResponseInputItem,
{
"id": "conv_item_user",
"type": "message",
"role": "user",
"content": "user history",
"provider_data": {"server": "metadata"},
},
)
function_call_item = cast(
TResponseInputItem,
{
"id": "conv_item_call",
"type": "function_call",
"call_id": "call_history",
"name": "lookup",
"arguments": "{}",
},
)
function_call_output_item = cast(
TResponseInputItem,
{
"id": "conv_item_output",
"type": "function_call_output",
"call_id": "call_history",
"output": "ok",
},
)
session = DummyOpenAIConversationsSession(
history=[user_history_item, history_item, function_call_item, function_call_output_item]
)
prepared, session_items = await prepare_input_with_session("new", session, None)
assert isinstance(prepared, list)
user_payload = cast(dict[str, Any], prepared[0])
history_payload = cast(dict[str, Any], prepared[1])
call_payload = cast(dict[str, Any], prepared[2])
output_payload = cast(dict[str, Any], prepared[3])
new_payload = cast(dict[str, Any], prepared[4])
assert user_payload["role"] == "user"
assert user_payload["id"] == "conv_item_user"
assert "provider_data" in user_payload
assert history_payload["role"] == "assistant"
assert "id" not in history_payload
assert "provider_data" not in history_payload
assert call_payload["id"] == "conv_item_call"
assert output_payload["id"] == "conv_item_output"
assert new_payload["role"] == "user"
assert new_payload["content"] == "new"
assert [cast(dict[str, Any], item).get("content") for item in session_items] == ["new"]
@pytest.mark.asyncio
async def test_prepare_input_with_regular_session_preserves_history_ids() -> None:
history_item = cast(
TResponseInputItem,
{
"id": "message_id",
"type": "message",
"role": "assistant",
"content": "history",
},
)
session = SimpleListSession(history=[history_item])
prepared, _ = await prepare_input_with_session("new", session, None)
assert isinstance(prepared, list)
history_payload = cast(dict[str, Any], prepared[0])
assert history_payload["id"] == "message_id"
@pytest.mark.asyncio
async def test_prepare_input_with_openai_conversation_callback_matches_assistant_no_ids() -> None:
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self, history: list[TResponseInputItem]) -> None:
self.history = history
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
if limit is None:
return list(self.history)
return self.history[-limit:]
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.history.extend(items)
async def pop_item(self) -> TResponseInputItem | None:
return self.history.pop() if self.history else None
async def clear_session(self) -> None:
self.history.clear()
history_item = cast(
TResponseInputItem,
{
"id": "conv_item_assistant",
"type": "message",
"role": "assistant",
"content": "history",
"provider_data": {"server": "metadata"},
},
)
session = DummyOpenAIConversationsSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
history_copy = dict(cast(dict[str, Any], history[0]))
history_copy.pop("id", None)
history_copy.pop("provider_data", None)
return [
cast(TResponseInputItem, history_copy),
cast(TResponseInputItem, dict(cast(dict[str, Any], new_input[0]))),
]
prepared, session_items = await prepare_input_with_session("new", session, callback)
assert isinstance(prepared, list)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == [
"history",
"new",
]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == ["new"]
@pytest.mark.asyncio
async def test_prepare_input_with_openai_conversation_callback_keeps_user_ids_distinct() -> None:
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self, history: list[TResponseInputItem]) -> None:
self.history = history
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
if limit is None:
return list(self.history)
return self.history[-limit:]
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.history.extend(items)
async def pop_item(self) -> TResponseInputItem | None:
return self.history.pop() if self.history else None
async def clear_session(self) -> None:
self.history.clear()
history_item = cast(
TResponseInputItem,
{
"id": "conv_item_user",
"type": "message",
"role": "user",
"content": "history",
"provider_data": {"server": "metadata"},
},
)
session = DummyOpenAIConversationsSession(history=[history_item])
def callback(
history: list[TResponseInputItem], new_input: list[TResponseInputItem]
) -> list[TResponseInputItem]:
history_copy = dict(cast(dict[str, Any], history[0]))
history_copy.pop("id", None)
history_copy.pop("provider_data", None)
return [
cast(TResponseInputItem, history_copy),
cast(TResponseInputItem, dict(cast(dict[str, Any], new_input[0]))),
]
prepared, session_items = await prepare_input_with_session("new", session, callback)
assert isinstance(prepared, list)
assert [cast(dict[str, Any], item).get("content") for item in prepared] == [
"history",
"new",
]
assert [cast(dict[str, Any], item).get("content") for item in session_items] == [
"history",
"new",
]
@pytest.mark.asyncio
async def test_persist_session_items_for_guardrail_trip_uses_original_input_when_missing() -> None:
session = SimpleListSession()
agent = Agent(name="agent", model=FakeModel())
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
persisted = await persist_session_items_for_guardrail_trip(
session,
None,
None,
"guardrail input",
run_state,
)
assert persisted == [{"role": "user", "content": "guardrail input"}]
assert await session.get_items() == persisted
@pytest.mark.asyncio
async def test_wait_for_session_cleanup_retries_after_get_items_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
target = cast(TResponseInputItem, {"id": "msg-1", "type": "message", "content": "hello"})
serialized_target = fingerprint_input_item(target)
class FlakyCleanupSession(SimpleListSession):
def __init__(self) -> None:
super().__init__()
self.get_items_calls = 0
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
self.get_items_calls += 1
if self.get_items_calls == 1:
raise RuntimeError("temporary failure")
return []
session = FlakyCleanupSession()
sleeps: list[float] = []
async def fake_sleep(delay: float) -> None:
sleeps.append(delay)
monkeypatch.setattr(asyncio, "sleep", fake_sleep)
assert serialized_target is not None
await wait_for_session_cleanup(session, [serialized_target])
assert session.get_items_calls == 2
assert sleeps == [0.1]
@pytest.mark.asyncio
async def test_wait_for_session_cleanup_logs_when_targets_linger(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
target = cast(TResponseInputItem, {"id": "msg-1", "type": "message", "content": "hello"})
session = SimpleListSession(history=[target])
serialized_target = fingerprint_input_item(target)
sleeps: list[float] = []
async def fake_sleep(delay: float) -> None:
sleeps.append(delay)
monkeypatch.setattr(asyncio, "sleep", fake_sleep)
assert serialized_target is not None
with caplog.at_level("DEBUG", logger="openai.agents"):
await wait_for_session_cleanup(session, [serialized_target], max_attempts=2)
assert sleeps == [0.1, 0.2]
assert "Session cleanup verification exhausted attempts" in caplog.text
@pytest.mark.asyncio
async def test_conversation_lock_rewind_skips_when_no_snapshot() -> None:
history_item = cast(TResponseInputItem, {"id": "old", "type": "message"})
new_item = cast(TResponseInputItem, {"id": "new", "type": "message"})
session = CountingSession(history=[history_item])
request = httpx.Request("POST", "https://example.com")
response = httpx.Response(
400,
request=request,
json={"error": {"code": "conversation_locked", "message": "locked"}},
)
locked_error = BadRequestError(
"locked",
response=response,
body={"error": {"code": "conversation_locked"}},
)
locked_error.code = "conversation_locked"
model = FakeModel()
model.add_multiple_turn_outputs([locked_error, [get_text_message("ok")]])
agent = Agent(name="test", model=model)
result = await get_new_response(
bindings=bind_public_agent(agent),
system_prompt=None,
input=[history_item, new_item],
output_schema=None,
all_tools=[],
handoffs=[],
hooks=RunHooks(),
context_wrapper=RunContextWrapper(context={}),
run_config=RunConfig(),
tool_use_tracker=AgentToolUseTracker(),
server_conversation_tracker=None,
prompt_config=None,
session=session,
session_items_to_rewind=[],
)
assert isinstance(result, ModelResponse)
assert session.pop_calls == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("session_backend", ["memory", "sqlite"])
async def test_non_streamed_model_retry_does_not_rewind_committed_session_input(
tmp_path: Path, session_backend: str
) -> None:
model = FakeModel()
model.add_multiple_turn_outputs(
[
APIConnectionError(
message="connection error",
request=httpx.Request("POST", "https://example.com"),
),
[get_text_message("done")],
]
)
agent = Agent(
name="test",
model=model,
model_settings=ModelSettings(
retry=ModelRetrySettings(
max_retries=1,
policy=retry_policies.network_error(),
)
),
)
session: CountingSession | SQLiteSession
if session_backend == "sqlite":
session = SQLiteSession("retry-session", tmp_path / "retry.sqlite3")
await session.add_items([get_text_input_item("previous")])
else:
session = CountingSession(history=[get_text_input_item("previous")])
try:
result = await Runner.run(agent, input="test", session=session)
saved_items = await session.get_items()
finally:
if isinstance(session, SQLiteSession):
session.close()
assert result.final_output == "done"
assert [item.get("role") for item in saved_items] == ["user", "user", "assistant"]
assert [item.get("content") for item in saved_items[:2]] == ["previous", "test"]
if isinstance(session, CountingSession):
assert session.pop_calls == 0
@pytest.mark.asyncio
async def test_get_new_response_uses_agent_retry_settings() -> None:
model = FakeModel()
model.set_hardcoded_usage(Usage(requests=1))
model.add_multiple_turn_outputs(
[
APIConnectionError(
message="connection error",
request=httpx.Request("POST", "https://example.com"),
),
[get_text_message("ok")],
]
)
agent = Agent(
name="test",
model=model,
model_settings=ModelSettings(
retry=ModelRetrySettings(
max_retries=1,
policy=retry_policies.network_error(),
)
),
)
result = await get_new_response(
bindings=bind_public_agent(agent),
system_prompt=None,
input=[get_text_input_item("hello")],
output_schema=None,
all_tools=[],
handoffs=[],
hooks=RunHooks(),
context_wrapper=RunContextWrapper(context={}),
run_config=RunConfig(),
tool_use_tracker=AgentToolUseTracker(),
server_conversation_tracker=None,
prompt_config=None,
session=None,
session_items_to_rewind=[],
)
assert isinstance(result, ModelResponse)
assert result.usage.requests == 2
@pytest.mark.asyncio
async def test_save_result_to_session_preserves_function_outputs():
session = SimpleListSession()
original_item = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_original",
"output": "1",
},
)
run_item_payload = {
"type": "function_call_output",
"call_id": "call_result",
"output": "2",
}
dummy_run_item = _DummyRunItem(run_item_payload)
await save_result_to_session(
session,
[original_item],
[cast(RunItem, dummy_run_item)],
None,
)
assert len(session.saved_items) == 2
for saved in session.saved_items:
saved_dict = cast(dict[str, Any], saved)
assert saved_dict["type"] == "function_call_output"
assert "output" in saved_dict
@pytest.mark.asyncio
async def test_save_result_to_session_prefers_latest_duplicate_function_outputs():
session = SimpleListSession()
original_item = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_duplicate",
"output": "old-output",
},
)
new_item_payload = {
"type": "function_call_output",
"call_id": "call_duplicate",
"output": "new-output",
}
new_item = _DummyRunItem(new_item_payload)
await save_result_to_session(
session,
[original_item],
[cast(RunItem, new_item)],
None,
)
duplicates = [
cast(dict[str, Any], item)
for item in session.saved_items
if isinstance(item, dict)
and item.get("type") == "function_call_output"
and item.get("call_id") == "call_duplicate"
]
assert len(duplicates) == 1
assert duplicates[0]["output"] == "new-output"
@pytest.mark.asyncio
async def test_save_result_to_session_keeps_tool_call_before_its_output():
session = SimpleListSession()
call_item = cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_ordered",
"name": "tool_ordered",
"arguments": "{}",
},
)
output_item = cast(
TResponseInputItem,
{"type": "function_call_output", "call_id": "call_ordered", "output": "result"},
)
# A resumed turn can replay a tool call the input list already carries. Collapsing the
# duplicate must not move the call behind its output in the persisted history.
repeated_call = _DummyRunItem(
{
"type": "function_call",
"call_id": "call_ordered",
"name": "tool_ordered",
"arguments": "{}",
},
item_type="tool_call_item",
)
await save_result_to_session(
session,
[call_item, output_item],
[cast(RunItem, repeated_call)],
None,
)
saved_types = [
cast(dict[str, Any], item).get("type")
for item in session.saved_items
if isinstance(item, dict)
]
assert saved_types == ["function_call", "function_call_output"]
@pytest.mark.asyncio
async def test_save_result_to_session_keeps_latest_output_after_its_call():
session = SimpleListSession()
old_output = cast(
TResponseInputItem,
{"type": "function_call_output", "call_id": "call_ordered", "output": "old"},
)
call_item = cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_ordered",
"name": "tool_ordered",
"arguments": "{}",
},
)
new_output = _DummyRunItem(
{"type": "function_call_output", "call_id": "call_ordered", "output": "new"}
)
await save_result_to_session(
session,
[old_output, call_item],
[cast(RunItem, new_output)],
None,
)
saved_items = [cast(dict[str, Any], item) for item in session.saved_items]
assert [item.get("type") for item in saved_items] == [
"function_call",
"function_call_output",
]
assert saved_items[1]["output"] == "new"
@pytest.mark.asyncio
async def test_rewind_handles_id_stripped_sessions() -> None:
session = IdStrippingSession()
item = cast(TResponseInputItem, {"id": "message-1", "type": "message", "content": "hello"})
await session.add_items([item])
await rewind_session_items(session, [item])
assert session.pop_calls == 1
assert session.saved_items == []
@pytest.mark.asyncio
@pytest.mark.parametrize("redacted", [True, False])
async def test_rewind_debug_logging_respects_model_and_tool_policies(
monkeypatch, redacted: bool
) -> None:
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted)
secret = "SECRET_REWIND_SESSION_CONTENT"
session = IdStrippingSession()
item = cast(
TResponseInputItem,
{"id": "message-1", "type": "message", "role": "user", "content": secret},
)
await session.add_items([item])
with patch("agents.run_internal.session_persistence.logger") as mock_logger:
await rewind_session_items(session, [item])
logged = str(mock_logger.debug.call_args_list)
assert (secret not in logged) is redacted
@pytest.mark.asyncio
async def test_rewind_failure_uses_placeholder_free_shared_logger_message() -> None:
class FailingTailSession(SimpleListSession):
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
raise RuntimeError("tail failure")
item = cast(TResponseInputItem, {"type": "message", "role": "user", "content": "hi"})
session = FailingTailSession(history=[item])
with patch(
"agents.run_internal.session_persistence.log_model_and_tool_action_warning"
) as mock_warning:
await rewind_session_items(session, [item])
assert mock_warning.call_args.args[1] == "Failed to rewind session item"
@pytest.mark.asyncio
async def test_rewind_skips_mismatched_tail_suffix() -> None:
target = cast(TResponseInputItem, {"type": "message", "role": "user", "content": "target"})
unrelated = cast(
TResponseInputItem,
{"type": "message", "role": "user", "content": "unrelated tail item"},
)
session = CountingSession(history=[target, unrelated])
await rewind_session_items(session, [target])
assert session.pop_calls == 0
assert session.saved_items == [target, unrelated]
@pytest.mark.asyncio
async def test_rewind_preserves_unrelated_tail_items_when_server_tracker_cleanup_runs() -> None:
known_server_item = cast(
TResponseInputItem,
{"id": "msg_server_1", "type": "message", "role": "assistant", "content": "server item"},
)
unrelated = cast(
TResponseInputItem,
{"type": "message", "role": "user", "content": "unrelated tail item"},
)
target = cast(TResponseInputItem, {"type": "message", "role": "user", "content": "target"})
session = CountingSession(history=[known_server_item, unrelated, target])
tracker = OpenAIServerConversationTracker()
tracker.server_item_ids.add("msg_server_1")
await rewind_session_items(session, [target], tracker)
assert session.pop_calls == 1
assert session.saved_items == [known_server_item, unrelated]
@pytest.mark.asyncio
async def test_rewind_strips_only_retry_owned_tail_items_before_known_server_item() -> None:
known_server_item = cast(
TResponseInputItem,
{"id": "msg_server_1", "type": "message", "role": "assistant", "content": "server item"},
)
retry_owned_tail = cast(
TResponseInputItem,
{"type": "message", "role": "user", "content": "retry-owned local item"},
)
target = cast(TResponseInputItem, {"type": "message", "role": "user", "content": "target"})
session = CountingSession(history=[known_server_item, retry_owned_tail, target])
tracker = OpenAIServerConversationTracker()
tracker.server_item_ids.add("msg_server_1")
retry_owned_fingerprint = fingerprint_input_item(retry_owned_tail)
assert retry_owned_fingerprint is not None
tracker.sent_item_fingerprints.add(retry_owned_fingerprint)
await rewind_session_items(session, [target], tracker)
assert session.pop_calls == 2
assert session.saved_items == [known_server_item]
def test_collect_retry_owned_tail_serializations_returns_empty_for_empty_session() -> None:
tracker = OpenAIServerConversationTracker()
assert (
_collect_retry_owned_tail_serializations(
[],
server_tracker=tracker,
ignore_ids_for_matching=False,
)
== []
)
@pytest.mark.asyncio
async def test_save_result_to_session_does_not_increment_counter_when_nothing_saved() -> None:
session = SimpleListSession()
agent = Agent(name="agent", model=FakeModel())
approval_item = ToolApprovalItem(
agent=agent,
raw_item={"type": "function_call", "call_id": "call-1", "name": "tool"},
)
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
await save_result_to_session(
session,
[],
cast(list[RunItem], [approval_item]),
run_state,
)
assert run_state._current_turn_persisted_item_count == 0
assert session.saved_items == []
@pytest.mark.asyncio
async def test_save_result_to_session_does_not_compact_empty_no_override_save() -> None:
client = MagicMock()
client.responses.compact = AsyncMock()
session = OpenAIResponsesCompactionSession(
session_id="test",
underlying_session=SimpleListSession(),
client=client,
should_trigger_compaction=lambda _: True,
)
agent = Agent(name="agent", model=FakeModel())
approval_item = ToolApprovalItem(
agent=agent,
raw_item={"type": "function_call", "call_id": "call-1", "name": "tool"},
)
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [approval_item]),
run_state,
response_id="resp-test",
)
assert saved_count == 0
assert await session.get_items() == []
client.responses.compact.assert_not_awaited()
@pytest.mark.asyncio
async def test_save_result_to_session_compacts_empty_save_and_retains_history_mutation() -> None:
class CompactingRewriteSession(RewriteAwareSimpleSession):
def __init__(self, history: list[TResponseInputItem]) -> None:
super().__init__(history=history)
self.compaction_calls = 0
async def run_compaction(self, _args: Any = None) -> None:
self.compaction_calls += 1
original_call = cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call-1",
"name": "tool",
"arguments": '{"value":"old"}',
},
)
session = CompactingRewriteSession([original_call])
agent = Agent(name="agent", model=FakeModel())
approval_item = ToolApprovalItem(agent=agent, raw_item=original_call)
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
run_state._session_history_mutations.append(
{
"type": "replace_function_call",
"call_id": "call-1",
"expected": original_call,
"replacement": {
"type": "function_call",
"call_id": "call-1",
"name": "tool",
"arguments": '{"value":"new"}',
},
}
)
run_state._approval_argument_override_modes["call-1"] = "durable"
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [approval_item]),
run_state,
response_id="resp-test",
)
assert saved_count == 0
assert session.compaction_calls == 1
assert [mutation["call_id"] for mutation in run_state._get_session_history_mutations()] == [
"call-1"
]
assert cast(dict[str, Any], (await session.get_items())[0])["arguments"] == '{"value":"new"}'
@pytest.mark.asyncio
async def test_save_result_to_session_returns_count_and_updates_state() -> None:
session = SimpleListSession()
agent = Agent(name="agent", model=FakeModel())
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
approval_item = ToolApprovalItem(
agent=agent,
raw_item={"type": "function_call", "call_id": "call-2", "name": "tool"},
)
output_item = _DummyRunItem(
{"type": "message", "role": "assistant", "content": "ok"},
"message_output_item",
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [output_item, approval_item]),
run_state,
)
assert saved_count == 1
assert run_state._current_turn_persisted_item_count == 1
assert len(session.saved_items) == 1
assert cast(dict[str, Any], session.saved_items[0]).get("content") == "ok"
@pytest.mark.asyncio
async def test_save_result_to_session_counts_sanitized_openai_items() -> None:
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self) -> None:
self.saved_items: list[TResponseInputItem] = []
async def _get_session_id(self) -> str:
return "conv_test"
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.saved_items.extend(items)
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
return []
async def pop_item(self) -> TResponseInputItem | None:
return None
async def clear_session(self) -> None:
return None
session = DummyOpenAIConversationsSession()
agent = Agent(name="agent", model=FakeModel())
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
output_item = _DummyRunItem(
{
"type": "message",
"role": "assistant",
"content": "ok",
"provider_data": {"model": "litellm/test"},
},
"message_output_item",
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [output_item]),
run_state,
)
assert saved_count == 1
assert run_state._current_turn_persisted_item_count == 1
assert len(session.saved_items) == 1
saved = cast(dict[str, Any], session.saved_items[0])
assert "provider_data" not in saved
@pytest.mark.asyncio
async def test_save_result_to_session_omits_reasoning_ids_when_policy_is_omit() -> None:
session = SimpleListSession()
agent = Agent(name="agent", model=FakeModel())
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
run_state.set_reasoning_item_id_policy("omit")
reasoning_item = ReasoningItem(
agent=agent,
raw_item=ResponseReasoningItem(type="reasoning", id="rs_stream", summary=[]),
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [reasoning_item]),
run_state,
)
assert saved_count == 1
assert len(session.saved_items) == 1
saved_reasoning = cast(dict[str, Any], session.saved_items[0])
assert saved_reasoning.get("type") == "reasoning"
assert "id" not in saved_reasoning
@pytest.mark.asyncio
async def test_save_result_to_openai_conversation_preserves_reasoning_id_when_policy_is_omit() -> (
None
):
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self) -> None:
self.saved_items: list[TResponseInputItem] = []
async def _get_session_id(self) -> str:
return "conv_test"
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.saved_items.extend(items)
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
return []
async def pop_item(self) -> TResponseInputItem | None:
return None
async def clear_session(self) -> None:
return None
session = DummyOpenAIConversationsSession()
agent = Agent(name="agent", model=FakeModel())
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
run_state.set_reasoning_item_id_policy("omit")
reasoning_item = ReasoningItem(
agent=agent,
raw_item=ResponseReasoningItem(
type="reasoning",
id="rs_openai_conversation",
summary=[Summary(text="thinking", type="summary_text")],
),
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [reasoning_item]),
run_state,
)
assert saved_count == 1
assert run_state._current_turn_persisted_item_count == 1
assert len(session.saved_items) == 1
saved_reasoning = cast(dict[str, Any], session.saved_items[0])
assert saved_reasoning.get("type") == "reasoning"
assert saved_reasoning.get("id") == "rs_openai_conversation"
@pytest.mark.asyncio
async def test_save_result_to_openai_conversation_drops_unpersistable_reasoning_item() -> None:
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self) -> None:
self.saved_items: list[TResponseInputItem] = []
async def _get_session_id(self) -> str:
return "conv_test"
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.saved_items.extend(items)
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
return []
async def pop_item(self) -> TResponseInputItem | None:
return None
async def clear_session(self) -> None:
return None
session = DummyOpenAIConversationsSession()
agent = Agent(name="agent", model=FakeModel())
run_state: RunState[Any] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=1,
)
malformed_reasoning = _DummyRunItem(
{"type": "reasoning", "summary": [], "content": []},
"reasoning_item",
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [malformed_reasoning]),
run_state,
)
assert saved_count == 1
assert run_state._current_turn_persisted_item_count == 1
assert session.saved_items == []
@pytest.mark.asyncio
async def test_save_result_to_openai_conversation_keeps_reasoning_encrypted_content() -> None:
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self) -> None:
self.saved_items: list[TResponseInputItem] = []
async def _get_session_id(self) -> str:
return "conv_test"
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.saved_items.extend(items)
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
return []
async def pop_item(self) -> TResponseInputItem | None:
return None
async def clear_session(self) -> None:
return None
session = DummyOpenAIConversationsSession()
encrypted_reasoning = _DummyRunItem(
{
"type": "reasoning",
"summary": [],
"content": [],
"encrypted_content": "encrypted",
},
"reasoning_item",
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [encrypted_reasoning]),
None,
)
assert saved_count == 1
assert len(session.saved_items) == 1
saved_reasoning = cast(dict[str, Any], session.saved_items[0])
assert saved_reasoning["encrypted_content"] == "encrypted"
@pytest.mark.asyncio
async def test_save_result_to_openai_conversation_drops_placeholder_id_reasoning_item() -> None:
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self) -> None:
self.saved_items: list[TResponseInputItem] = []
async def _get_session_id(self) -> str:
return "conv_test"
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.saved_items.extend(items)
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
return []
async def pop_item(self) -> TResponseInputItem | None:
return None
async def clear_session(self) -> None:
return None
session = DummyOpenAIConversationsSession()
agent = Agent(name="agent", model=FakeModel())
# Chat Completions providers have no server-assigned reasoning ID, so the SDK stamps its
# own placeholder. That placeholder is not a server identity, so the item is no more
# persistable than one with no ID at all.
placeholder_reasoning = ReasoningItem(
agent=agent,
raw_item=ResponseReasoningItem(
type="reasoning",
id=FAKE_RESPONSES_ID,
summary=[Summary(text="thinking", type="summary_text")],
),
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [placeholder_reasoning]),
None,
)
assert saved_count == 1
assert session.saved_items == []
@pytest.mark.asyncio
async def test_save_result_to_openai_conversation_strips_placeholder_reasoning_id() -> None:
class DummyOpenAIConversationsSession(OpenAIConversationsSession):
def __init__(self) -> None:
self.saved_items: list[TResponseInputItem] = []
async def _get_session_id(self) -> str:
return "conv_test"
async def add_items(self, items: list[TResponseInputItem]) -> None:
self.saved_items.extend(items)
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
return []
async def pop_item(self) -> TResponseInputItem | None:
return None
async def clear_session(self) -> None:
return None
session = DummyOpenAIConversationsSession()
agent = Agent(name="agent", model=FakeModel())
placeholder_reasoning = ReasoningItem(
agent=agent,
raw_item=ResponseReasoningItem(
type="reasoning",
id=FAKE_RESPONSES_ID,
summary=[],
encrypted_content="encrypted",
),
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [placeholder_reasoning]),
None,
)
assert saved_count == 1
assert len(session.saved_items) == 1
saved_reasoning = cast(dict[str, Any], session.saved_items[0])
assert saved_reasoning["encrypted_content"] == "encrypted"
assert "id" not in saved_reasoning
@pytest.mark.asyncio
async def test_save_result_to_session_keeps_tool_call_payload_api_safe() -> None:
session = SimpleListSession()
agent = Agent(name="agent", model=FakeModel())
tool_call = ToolCallItem(
agent=agent,
raw_item=ResponseFunctionToolCall(
id="fc_session",
call_id="call_session",
name="lookup_account",
arguments="{}",
type="function_call",
status="completed",
),
description="Lookup customer records.",
title="Lookup Account",
)
saved_count = await save_result_to_session(
session,
[],
cast(list[RunItem], [tool_call]),
None,
)
assert saved_count == 1
assert len(session.saved_items) == 1
saved_tool_call = cast(dict[str, Any], session.saved_items[0])
assert saved_tool_call["type"] == "function_call"
assert TOOL_CALL_SESSION_DESCRIPTION_KEY not in saved_tool_call
assert TOOL_CALL_SESSION_TITLE_KEY not in saved_tool_call
assert "description" not in saved_tool_call
assert "title" not in saved_tool_call
@pytest.mark.asyncio
async def test_save_result_to_session_sanitizes_original_input_items() -> None:
session = SimpleListSession()
saved_count = await save_result_to_session(
session,
[
cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_input",
"name": "lookup_account",
"arguments": "{}",
TOOL_CALL_SESSION_DESCRIPTION_KEY: "Lookup customer records.",
TOOL_CALL_SESSION_TITLE_KEY: "Lookup Account",
},
)
],
[],
None,
)
assert saved_count == 0
assert len(session.saved_items) == 1
saved_tool_call = cast(dict[str, Any], session.saved_items[0])
assert saved_tool_call["type"] == "function_call"
assert TOOL_CALL_SESSION_DESCRIPTION_KEY not in saved_tool_call
assert TOOL_CALL_SESSION_TITLE_KEY not in saved_tool_call
assert "description" not in saved_tool_call
assert "title" not in saved_tool_call
@pytest.mark.asyncio
async def test_prepare_input_with_session_strips_internal_tool_call_metadata() -> None:
tool_call = cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_history",
"name": "lookup_account",
"arguments": "{}",
TOOL_CALL_SESSION_DESCRIPTION_KEY: "Lookup customer records.",
TOOL_CALL_SESSION_TITLE_KEY: "Lookup Account",
},
)
tool_output = cast(
TResponseInputItem,
{
"type": "function_call_output",
"call_id": "call_history",
"output": "ok",
},
)
session = SimpleListSession(history=[tool_call, tool_output])
prepared_input, session_items = await prepare_input_with_session("hello", session, None)
assert isinstance(prepared_input, list)
prepared_tool_calls = [
cast(dict[str, Any], item)
for item in prepared_input
if isinstance(item, dict)
and item.get("type") == "function_call"
and item.get("call_id") == "call_history"
]
assert len(prepared_tool_calls) == 1
assert TOOL_CALL_SESSION_DESCRIPTION_KEY not in prepared_tool_calls[0]
assert TOOL_CALL_SESSION_TITLE_KEY not in prepared_tool_calls[0]
assert len(session_items) == 1
assert cast(dict[str, Any], session_items[0])["role"] == "user"
@pytest.mark.asyncio
async def test_prepare_input_with_session_sanitizes_new_tool_call_session_items() -> None:
prepared_input, session_items = await prepare_input_with_session(
[
cast(
TResponseInputItem,
{
"type": "function_call",
"call_id": "call_new",
"name": "lookup_account",
"arguments": "{}",
TOOL_CALL_SESSION_DESCRIPTION_KEY: "Lookup customer records.",
TOOL_CALL_SESSION_TITLE_KEY: "Lookup Account",
},
)
],
SimpleListSession(),
None,
)
assert isinstance(prepared_input, list)
assert len(prepared_input) == 1
prepared_tool_call = cast(dict[str, Any], prepared_input[0])
assert prepared_tool_call["type"] == "function_call"
assert TOOL_CALL_SESSION_DESCRIPTION_KEY not in prepared_tool_call
assert TOOL_CALL_SESSION_TITLE_KEY not in prepared_tool_call
assert len(session_items) == 1
session_tool_call = cast(dict[str, Any], session_items[0])
assert session_tool_call["type"] == "function_call"
assert TOOL_CALL_SESSION_DESCRIPTION_KEY not in session_tool_call
assert TOOL_CALL_SESSION_TITLE_KEY not in session_tool_call
@pytest.mark.asyncio
async def test_session_persists_only_new_step_items(monkeypatch: pytest.MonkeyPatch) -> None:
"""Ensure only per-turn new_step_items are persisted to the session."""
session = SimpleListSession()
agent = Agent(name="agent", model=FakeModel())
pre_item = _DummyRunItem(
{"type": "message", "role": "assistant", "content": "old"}, "message_output_item"
)
new_item = _DummyRunItem(
{"type": "message", "role": "assistant", "content": "new"}, "message_output_item"
)
new_response = ModelResponse(output=[], usage=Usage(), response_id="resp-1")
turn_result = SingleStepResult(
original_input="hello",
model_response=new_response,
pre_step_items=[cast(RunItem, pre_item)],
new_step_items=[cast(RunItem, new_item)],
next_step=NextStepFinalOutput(output="done"),
tool_input_guardrail_results=[],
tool_output_guardrail_results=[],
)
calls: list[list[RunItem]] = []
from agents.run_internal import session_persistence as sp
real_save_result = sp.save_result_to_session
async def save_wrapper(
sess: Any,
original_input: Any,
new_items: list[RunItem],
run_state: RunState | None = None,
**kwargs: Any,
) -> None:
calls.append(list(new_items))
await real_save_result(sess, original_input, new_items, run_state, **kwargs)
async def fake_run_single_turn(**_: Any) -> SingleStepResult:
return turn_result
async def fake_run_output_guardrails(*_: Any, **__: Any) -> list[Any]:
return []
async def noop_initialize_computer_tools(
*args: Any, tools: list[Any], **kwargs: Any
) -> list[Any]:
return tools
monkeypatch.setattr("agents.run.save_result_to_session", save_wrapper)
monkeypatch.setattr(
"agents.run_internal.session_persistence.save_result_to_session", save_wrapper
)
monkeypatch.setattr(
"agents.run_internal.agent_runner_helpers.save_result_to_session", save_wrapper
)
monkeypatch.setattr("agents.run.run_single_turn", fake_run_single_turn)
monkeypatch.setattr("agents.run_internal.run_loop.run_single_turn", fake_run_single_turn)
monkeypatch.setattr("agents.run.run_output_guardrails", fake_run_output_guardrails)
monkeypatch.setattr(
"agents.run_internal.run_loop.run_output_guardrails", fake_run_output_guardrails
)
async def fake_get_all_tools(*_: Any, **__: Any) -> list[Any]:
return []
monkeypatch.setattr("agents.run.get_all_tools", fake_get_all_tools)
monkeypatch.setattr("agents.run_internal.run_loop.get_all_tools", fake_get_all_tools)
monkeypatch.setattr("agents.run.initialize_computer_tools", noop_initialize_computer_tools)
monkeypatch.setattr(
"agents.run_internal.run_loop.initialize_computer_tools", noop_initialize_computer_tools
)
result = await Runner.run(agent, input="hello", session=session)
assert result.final_output == "done"
# First save writes the user input; second save should contain only the new_step_items.
assert len(calls) >= 2
assert calls[-1] == [cast(RunItem, new_item)]
items = await session.get_items()
assert len(items) == 2
assert any("new" in cast(dict[str, Any], item).get("content", "") for item in items)
assert not any("old" in cast(dict[str, Any], item).get("content", "") for item in items)
@pytest.mark.asyncio
async def test_output_guardrail_tripwire_triggered_causes_exception():
def guardrail_function(
context: RunContextWrapper[Any], agent: Agent[Any], agent_output: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(
output_info=None,
tripwire_triggered=True,
)
model = FakeModel()
agent = Agent(
name="test",
output_guardrails=[OutputGuardrail(guardrail_function=guardrail_function)],
model=model,
)
model.set_next_output([get_text_message("user_message")])
with pytest.raises(OutputGuardrailTripwireTriggered):
await Runner.run(agent, input="user_message")
def test_output_guardrail_tripwire_does_not_save_assistant_message_to_session_sync() -> None:
def guardrail_function(
_context: RunContextWrapper[Any], _agent: Agent[Any], _agent_output: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=True)
session = SimpleListSession()
model = FakeModel()
model.set_next_output([get_text_message("should_not_be_saved")])
agent = Agent(
name="test",
model=model,
output_guardrails=[OutputGuardrail(guardrail_function=guardrail_function)],
)
with pytest.raises(OutputGuardrailTripwireTriggered):
Runner.run_sync(agent, input="user_message", session=session)
items = asyncio.run(session.get_items())
assert [
cast(dict[str, Any], item).get("type") or cast(dict[str, Any], item).get("role")
for item in items
] == ["user"]
@pytest.mark.asyncio
async def test_output_guardrail_error_preserves_final_output_in_session() -> None:
def guardrail_function(
_context: RunContextWrapper[Any], _agent: Agent[Any], _agent_output: Any
) -> GuardrailFunctionOutput:
raise RuntimeError("guardrail failed")
session = SimpleListSession()
model = FakeModel()
model.set_next_output([get_text_message("preserved_on_guardrail_error")])
agent = Agent(
name="test",
model=model,
output_guardrails=[OutputGuardrail(guardrail_function=guardrail_function)],
)
with pytest.raises(RuntimeError, match="guardrail failed"):
await Runner.run(agent, input="user_message", session=session)
items = await session.get_items()
assert [
cast(dict[str, Any], item).get("type") or cast(dict[str, Any], item).get("role")
for item in items
] == ["user", "message"]
@pytest.mark.asyncio
async def test_output_guardrail_cancellation_preserves_final_output_in_session() -> None:
guardrail_started = asyncio.Event()
async def guardrail_function(
_context: RunContextWrapper[Any], _agent: Agent[Any], _agent_output: Any
) -> GuardrailFunctionOutput:
guardrail_started.set()
await asyncio.Event().wait()
raise AssertionError("unreachable")
session = SimpleListSession()
model = FakeModel()
model.set_next_output([get_text_message("preserved_on_guardrail_cancellation")])
agent = Agent(
name="test",
model=model,
output_guardrails=[OutputGuardrail(guardrail_function=guardrail_function)],
)
run_task = asyncio.create_task(Runner.run(agent, input="user_message", session=session))
await guardrail_started.wait()
run_task.cancel()
with pytest.raises(asyncio.CancelledError):
await run_task
items = await session.get_items()
assert [
cast(dict[str, Any], item).get("type") or cast(dict[str, Any], item).get("role")
for item in items
] == ["user", "message"]
@pytest.mark.asyncio
async def test_resumed_final_output_persists_once_after_passing_output_guardrail() -> None:
def guardrail_function(
_context: RunContextWrapper[Any], _agent: Agent[Any], _agent_output: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False)
@function_tool
def foo(a: str) -> str:
return f"result:{a}"
session = SimpleListSession()
model = FakeModel()
model.set_next_output([get_function_tool_call("foo", json.dumps({"a": "b"}))])
agent = Agent(
name="test",
model=model,
tools=[foo],
output_guardrails=[OutputGuardrail(guardrail_function=guardrail_function)],
)
streamed = Runner.run_streamed(agent, input="user_message", session=session)
async for event in streamed.stream_events():
if event.type == "run_item_stream_event" and event.name == "tool_output":
streamed.cancel(mode="after_turn")
items_before_resume = await session.get_items()
state = streamed.to_state()
state._current_turn_persisted_item_count = 2
model.set_next_output([get_text_message("accepted_final")])
resumed = await Runner.run(agent, state, session=session)
assert resumed.final_output == "accepted_final"
items_after_resume = await session.get_items()
assert items_after_resume[: len(items_before_resume)] == items_before_resume
assistant_messages = [
item
for item in items_after_resume
if cast(dict[str, Any], item).get("role") == "assistant"
and cast(dict[str, Any], item).get("type") == "message"
]
assert len(assistant_messages) == 1
content = cast(dict[str, Any], assistant_messages[0]).get("content")
assert isinstance(content, list)
assert any(isinstance(part, dict) and part.get("text") == "accepted_final" for part in content)
@pytest.mark.parametrize("tripwire_triggered", [False, True])
@pytest.mark.asyncio
async def test_resumed_final_tool_persists_call_and_output_after_output_guardrail(
tripwire_triggered: bool,
) -> None:
def guardrail_function(
_context: RunContextWrapper[Any], _agent: Agent[Any], _agent_output: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(
output_info=None,
tripwire_triggered=tripwire_triggered,
)
@function_tool(name_override="commit_tool")
def commit_tool() -> str:
return "committed-result"
session = SimpleListSession()
model = FakeModel()
model.set_next_output([get_function_tool_call("commit_tool", "{}", call_id="call-first")])
agent = Agent(
name="test",
model=model,
tools=[commit_tool],
output_guardrails=[OutputGuardrail(guardrail_function=guardrail_function)],
)
streamed = Runner.run_streamed(agent, input="user_message", session=session)
async for event in streamed.stream_events():
if event.type == "run_item_stream_event" and event.name == "tool_output":
streamed.cancel(mode="after_turn")
state = streamed.to_state()
assert state._current_turn_persisted_item_count == 2
agent.tool_use_behavior = "stop_on_first_tool"
model.set_next_output([get_function_tool_call("commit_tool", "{}", call_id="call-second")])
if tripwire_triggered:
with pytest.raises(OutputGuardrailTripwireTriggered):
await Runner.run(agent, state, session=session)
else:
result = await Runner.run(agent, state, session=session)
assert result.final_output == "committed-result"
assert state._current_turn_persisted_item_count == 4
items = await session.get_items()
assert [
(
cast(dict[str, Any], item).get("type") or cast(dict[str, Any], item).get("role"),
cast(dict[str, Any], item).get("call_id"),
)
for item in items
] == [
("user", None),
("function_call", "call-first"),
("function_call_output", "call-first"),
("function_call", "call-second"),
("function_call_output", "call-second"),
]
@pytest.mark.asyncio
async def test_input_guardrail_no_tripwire_continues_execution():
"""Test input guardrail that doesn't trigger tripwire continues execution."""
def guardrail_function(
context: RunContextWrapper[Any], agent: Agent[Any], input: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(
output_info=None,
tripwire_triggered=False, # Doesn't trigger tripwire
)
model = FakeModel()
model.set_next_output([get_text_message("response")])
agent = Agent(
name="test",
model=model,
input_guardrails=[InputGuardrail(guardrail_function=guardrail_function)],
)
# Should complete successfully without raising exception
result = await Runner.run(agent, input="user_message")
assert result.final_output == "response"
@pytest.mark.asyncio
async def test_output_guardrail_no_tripwire_continues_execution():
"""Test output guardrail that doesn't trigger tripwire continues execution."""
def guardrail_function(
context: RunContextWrapper[Any], agent: Agent[Any], agent_output: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(
output_info=None,
tripwire_triggered=False, # Doesn't trigger tripwire
)
model = FakeModel()
model.set_next_output([get_text_message("response")])
agent = Agent(
name="test",
model=model,
output_guardrails=[OutputGuardrail(guardrail_function=guardrail_function)],
)
# Should complete successfully without raising exception
result = await Runner.run(agent, input="user_message")
assert result.final_output == "response"
@function_tool
def test_tool_one():
return Foo(bar="tool_one_result")
@function_tool
def test_tool_two():
return "tool_two_result"
@pytest.mark.asyncio
async def test_tool_use_behavior_first_output():
class FalsyAgentOutputSchema(AgentOutputSchema):
def __bool__(self) -> bool:
return False
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "tool_result"), test_tool_one, test_tool_two],
tool_use_behavior="stop_on_first_tool",
output_type=FalsyAgentOutputSchema(Foo),
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[
get_text_message("a_message"),
get_function_tool_call("test_tool_one", None, call_id="tool-one"),
get_function_tool_call("test_tool_two", None, call_id="tool-two"),
],
]
)
result = await Runner.run(agent, input="user_message")
assert result.final_output == Foo(bar="tool_one_result"), (
"should have used the first tool result"
)
def custom_tool_use_behavior(
context: RunContextWrapper[Any], results: list[FunctionToolResult]
) -> ToolsToFinalOutputResult:
if "test_tool_one" in [result.tool.name for result in results]:
return ToolsToFinalOutputResult(is_final_output=True, final_output="the_final_output")
else:
return ToolsToFinalOutputResult(is_final_output=False, final_output=None)
@pytest.mark.asyncio
async def test_tool_use_behavior_custom_function():
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "tool_result"), test_tool_one, test_tool_two],
tool_use_behavior=custom_tool_use_behavior,
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[
get_text_message("a_message"),
get_function_tool_call("test_tool_two", None, call_id="call-tool-two-first"),
],
# Second turn: a message and tool call
[
get_text_message("a_message"),
get_function_tool_call("test_tool_one", None, call_id="call-tool-one"),
get_function_tool_call("test_tool_two", None, call_id="call-tool-two-second"),
],
]
)
result = await Runner.run(agent, input="user_message")
assert len(result.raw_responses) == 2, "should have two model responses"
assert result.final_output == "the_final_output", "should have used the custom function"
@pytest.mark.asyncio
async def test_model_settings_override():
model = FakeModel()
agent = Agent(
name="test", model=model, model_settings=ModelSettings(temperature=1.0, max_tokens=1000)
)
model.add_multiple_turn_outputs(
[
[
get_text_message("a_message"),
],
]
)
await Runner.run(
agent,
input="user_message",
run_config=RunConfig(model_settings=ModelSettings(0.5)),
)
# temperature is overridden by Runner.run, but max_tokens is not
assert model.last_turn_args["model_settings"].temperature == 0.5
assert model.last_turn_args["model_settings"].max_tokens == 1000
@pytest.mark.asyncio
async def test_previous_response_id_passed_between_runs():
"""Test that previous_response_id is passed to the model on subsequent runs."""
model = FakeModel()
model.set_next_output([get_text_message("done")])
agent = Agent(name="test", model=model)
assert model.last_turn_args.get("previous_response_id") is None
await Runner.run(agent, input="test", previous_response_id="resp-non-streamed-test")
assert model.last_turn_args.get("previous_response_id") == "resp-non-streamed-test"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"run_kwargs",
[
{"conversation_id": "conv-test"},
{"previous_response_id": "resp-test"},
{"auto_previous_response_id": True},
],
)
async def test_run_rejects_session_with_server_managed_conversation(run_kwargs: dict[str, Any]):
model = FakeModel()
model.set_next_output([get_text_message("done")])
agent = Agent(name="test", model=model)
session = SimpleListSession()
with pytest.raises(UserError, match="Session persistence"):
await Runner.run(agent, input="test", session=session, **run_kwargs)
@pytest.mark.asyncio
async def test_run_rejects_session_with_resumed_conversation_state():
model = FakeModel()
agent = Agent(name="test", model=model)
session = SimpleListSession()
context_wrapper = RunContextWrapper(context=None)
state = RunState(
context=context_wrapper,
original_input="hello",
starting_agent=agent,
conversation_id="conv-test",
)
with pytest.raises(UserError, match="Session persistence"):
await Runner.run(agent, state, session=session)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"run_kwargs",
[
{"conversation_id": "conv-test"},
{"previous_response_id": "resp-test"},
{"auto_previous_response_id": True},
],
)
async def test_run_streamed_rejects_session_with_server_managed_conversation(
run_kwargs: dict[str, Any],
):
model = FakeModel()
model.set_next_output([get_text_message("done")])
agent = Agent(name="test", model=model)
session = SimpleListSession()
with pytest.raises(UserError, match="Session persistence"):
Runner.run_streamed(agent, input="test", session=session, **run_kwargs)
@pytest.mark.asyncio
async def test_run_streamed_rejects_session_with_resumed_conversation_state():
model = FakeModel()
agent = Agent(name="test", model=model)
session = SimpleListSession()
context_wrapper = RunContextWrapper(context=None)
state = RunState(
context=context_wrapper,
original_input="hello",
starting_agent=agent,
conversation_id="conv-test",
)
with pytest.raises(UserError, match="Session persistence"):
Runner.run_streamed(agent, state, session=session)
@pytest.mark.asyncio
async def test_multi_turn_previous_response_id_passed_between_runs():
"""Test that previous_response_id is passed to the model on subsequent runs."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("foo", json.dumps({"a": "b"}))],
# Second turn: text message
[get_text_message("done")],
]
)
assert model.last_turn_args.get("previous_response_id") is None
await Runner.run(agent, input="test", previous_response_id="resp-test-123")
assert model.last_turn_args.get("previous_response_id") == "resp-789"
@pytest.mark.asyncio
async def test_previous_response_id_passed_between_runs_streamed():
"""Test that previous_response_id is passed to the model on subsequent streamed runs."""
model = FakeModel()
model.set_next_output([get_text_message("done")])
agent = Agent(
name="test",
model=model,
)
assert model.last_turn_args.get("previous_response_id") is None
result = Runner.run_streamed(agent, input="test", previous_response_id="resp-stream-test")
async for _ in result.stream_events():
pass
assert model.last_turn_args.get("previous_response_id") == "resp-stream-test"
@pytest.mark.asyncio
async def test_previous_response_id_passed_between_runs_streamed_multi_turn():
"""Test that previous_response_id is passed to the model on subsequent streamed runs."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("foo", json.dumps({"a": "b"}))],
# Second turn: text message
[get_text_message("done")],
]
)
assert model.last_turn_args.get("previous_response_id") is None
result = Runner.run_streamed(agent, input="test", previous_response_id="resp-stream-test")
async for _ in result.stream_events():
pass
assert model.last_turn_args.get("previous_response_id") == "resp-789"
@pytest.mark.asyncio
async def test_conversation_id_only_sends_new_items_multi_turn():
"""Test that conversation_id mode only sends new items on subsequent turns."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[
get_text_message("a_message"),
get_function_tool_call(
"test_func", '{"arg": "foo"}', call_id="call-test-func-first"
),
],
# Second turn: another message and tool call
[
get_text_message("b_message"),
get_function_tool_call(
"test_func", '{"arg": "bar"}', call_id="call-test-func-second"
),
],
# Third turn: final text message
[get_text_message("done")],
]
)
result = await Runner.run(agent, input="user_message", conversation_id="conv-test-123")
assert result.final_output == "done"
# Check the first call - it should include the original input since generated_items is empty
assert model.first_turn_args is not None
first_input = model.first_turn_args["input"]
# First call should include the original user input
assert isinstance(first_input, list)
assert len(first_input) == 1 # Should contain the user message
# The input should be the user message
user_message = first_input[0]
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# Check the input from the last turn (third turn after function execution)
last_input = model.last_turn_args["input"]
# In conversation_id mode, the third turn should only contain the tool output
assert isinstance(last_input, list)
assert len(last_input) == 1
# The single item should be a tool result
tool_result_item = last_input[0]
assert tool_result_item.get("type") == "function_call_output"
assert tool_result_item.get("call_id") is not None
@pytest.mark.asyncio
async def test_conversation_id_only_sends_new_items_multi_turn_streamed():
"""Test that conversation_id mode only sends new items on subsequent turns (streamed mode)."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[
get_text_message("a_message"),
get_function_tool_call(
"test_func", '{"arg": "foo"}', call_id="call-test-func-first"
),
],
# Second turn: another message and tool call
[
get_text_message("b_message"),
get_function_tool_call(
"test_func", '{"arg": "bar"}', call_id="call-test-func-second"
),
],
# Third turn: final text message
[get_text_message("done")],
]
)
result = Runner.run_streamed(agent, input="user_message", conversation_id="conv-test-123")
async for _ in result.stream_events():
pass
assert result.final_output == "done"
# Check the first call - it should include the original input since generated_items is empty
assert model.first_turn_args is not None
first_input = model.first_turn_args["input"]
# First call should include the original user input
assert isinstance(first_input, list)
assert len(first_input) == 1 # Should contain the user message
# The input should be the user message
user_message = first_input[0]
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# Check the input from the last turn (third turn after function execution)
last_input = model.last_turn_args["input"]
# In conversation_id mode, the third turn should only contain the tool output
assert isinstance(last_input, list)
assert len(last_input) == 1
# The single item should be a tool result
tool_result_item = last_input[0]
assert tool_result_item.get("type") == "function_call_output"
assert tool_result_item.get("call_id") is not None
@pytest.mark.asyncio
async def test_previous_response_id_only_sends_new_items_multi_turn():
"""Test that previous_response_id mode only sends new items and updates
previous_response_id between turns."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
# Second turn: final text message
[get_text_message("done")],
]
)
result = await Runner.run(
agent, input="user_message", previous_response_id="initial-response-123"
)
assert result.final_output == "done"
# Check the first call - it should include the original input since generated_items is empty
assert model.first_turn_args is not None
first_input = model.first_turn_args["input"]
# First call should include the original user input
assert isinstance(first_input, list)
assert len(first_input) == 1 # Should contain the user message
# The input should be the user message
user_message = first_input[0]
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# Check the input from the last turn (second turn after function execution)
last_input = model.last_turn_args["input"]
# In previous_response_id mode, the third turn should only contain the tool output
assert isinstance(last_input, list)
assert len(last_input) == 1 # Only the function result
# The single item should be a tool result
tool_result_item = last_input[0]
assert tool_result_item.get("type") == "function_call_output"
assert tool_result_item.get("call_id") is not None
# Verify that previous_response_id is modified according to fake_model behavior
assert model.last_turn_args.get("previous_response_id") == "resp-789"
@pytest.mark.asyncio
async def test_previous_response_id_retry_does_not_resend_initial_input_multi_turn():
class StatefulRetrySafeFakeModel(FakeModel):
def get_retry_advice(self, request):
if request.previous_response_id or request.conversation_id:
return ModelRetryAdvice(suggested=True, replay_safety="safe")
return None
model = StatefulRetrySafeFakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
model_settings=ModelSettings(
retry=ModelRetrySettings(
max_retries=1,
policy=retry_policies.network_error(),
)
),
)
model.add_multiple_turn_outputs(
[
APIConnectionError(
message="connection error",
request=httpx.Request("POST", "https://example.com"),
),
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
[get_text_message("done")],
]
)
result = await Runner.run(
agent, input="user_message", previous_response_id="initial-response-123"
)
assert result.final_output == "done"
last_input = model.last_turn_args["input"]
assert isinstance(last_input, list)
assert len(last_input) == 1
assert last_input[0].get("type") == "function_call_output"
@pytest.mark.asyncio
async def test_auto_previous_response_id_retries_when_policy_approves_unsafe_replay():
seen: list[RetryPolicyContext] = []
class StatefulRetryUnsafeFakeModel(FakeModel):
def get_retry_advice(self, request):
if request.previous_response_id or request.conversation_id:
return ModelRetryAdvice(
suggested=False,
replay_safety="unsafe",
response_started=True,
)
return None
def policy(context: RetryPolicyContext) -> RetryDecision:
seen.append(context)
return RetryDecision(retry=True, approve_unsafe_replay=True)
model = StatefulRetryUnsafeFakeModel()
model.add_multiple_turn_outputs(
[
[get_function_tool_call("test_func", '{"arg": "foo"}')],
APIConnectionError(
message="connection closed after response processing started",
request=httpx.Request("POST", "https://example.com"),
),
[get_text_message("done")],
]
)
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
model_settings=ModelSettings(
retry=ModelRetrySettings(max_retries=1, policy=policy),
),
)
result = await Runner.run(agent, input="user_message", auto_previous_response_id=True)
assert result.final_output == "done"
assert len(seen) == 1
assert seen[0].previous_response_id == "resp-789"
assert seen[0].conversation_id is None
assert seen[0].stateful_request is True
assert seen[0].response_started is True
assert seen[0].replay_safety == "unsafe"
@pytest.mark.asyncio
async def test_previous_response_id_only_sends_new_items_multi_turn_streamed():
"""Test that previous_response_id mode only sends new items and updates
previous_response_id between turns (streamed mode)."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
# Second turn: final text message
[get_text_message("done")],
]
)
result = Runner.run_streamed(
agent, input="user_message", previous_response_id="initial-response-123"
)
async for _ in result.stream_events():
pass
assert result.final_output == "done"
# Check the first call - it should include the original input since generated_items is empty
assert model.first_turn_args is not None
first_input = model.first_turn_args["input"]
# First call should include the original user input
assert isinstance(first_input, list)
assert len(first_input) == 1 # Should contain the user message
# The input should be the user message
user_message = first_input[0]
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# Check the input from the last turn (second turn after function execution)
last_input = model.last_turn_args["input"]
# In previous_response_id mode, the third turn should only contain the tool output
assert isinstance(last_input, list)
assert len(last_input) == 1 # Only the function result
# The single item should be a tool result
tool_result_item = last_input[0]
assert tool_result_item.get("type") == "function_call_output"
assert tool_result_item.get("call_id") is not None
# Verify that previous_response_id is modified according to fake_model behavior
assert model.last_turn_args.get("previous_response_id") == "resp-789"
@pytest.mark.asyncio
async def test_previous_response_id_retry_does_not_resend_initial_input_multi_turn_streamed():
class StatefulRetrySafeFakeModel(FakeModel):
def get_retry_advice(self, request):
if request.previous_response_id or request.conversation_id:
return ModelRetryAdvice(suggested=True, replay_safety="safe")
return None
model = StatefulRetrySafeFakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
model_settings=ModelSettings(
retry=ModelRetrySettings(
max_retries=1,
policy=retry_policies.network_error(),
)
),
)
model.add_multiple_turn_outputs(
[
APIConnectionError(
message="connection error",
request=httpx.Request("POST", "https://example.com"),
),
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
[get_text_message("done")],
]
)
result = Runner.run_streamed(
agent, input="user_message", previous_response_id="initial-response-123"
)
async for _ in result.stream_events():
pass
assert result.final_output == "done"
last_input = model.last_turn_args["input"]
assert isinstance(last_input, list)
assert len(last_input) == 1
assert last_input[0].get("type") == "function_call_output"
@pytest.mark.asyncio
async def test_default_send_all_items():
"""Test that without conversation_id or previous_response_id, all items are sent."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
# Second turn: final text message
[get_text_message("done")],
]
)
result = await Runner.run(
agent, input="user_message"
) # No conversation_id or previous_response_id
assert result.final_output == "done"
# Check the input from the last turn (second turn after function execution)
last_input = model.last_turn_args["input"]
# In default, the second turn should contain ALL items:
# 1. Original user message
# 2. Assistant response message
# 3. Function call
# 4. Function result
assert isinstance(last_input, list)
assert (
len(last_input) == 4
) # User message + assistant message + function call + function result
# Verify the items are in the expected order
user_message = last_input[0]
assistant_message = last_input[1]
function_call = last_input[2]
function_result = last_input[3]
# Check user message
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# Check assistant message
assert assistant_message.get("role") == "assistant"
# Check function call
assert function_call.get("name") == "test_func"
assert function_call.get("arguments") == '{"arg": "foo"}'
# Check function result
assert function_result.get("type") == "function_call_output"
assert function_result.get("call_id") is not None
@pytest.mark.asyncio
async def test_default_send_all_items_streamed():
"""Test that without conversation_id or previous_response_id, all items are sent
(streamed mode)."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
# Second turn: final text message
[get_text_message("done")],
]
)
result = Runner.run_streamed(
agent, input="user_message"
) # No conversation_id or previous_response_id
async for _ in result.stream_events():
pass
assert result.final_output == "done"
# Check the input from the last turn (second turn after function execution)
last_input = model.last_turn_args["input"]
# In default mode, the second turn should contain ALL items:
# 1. Original user message
# 2. Assistant response message
# 3. Function call
# 4. Function result
assert isinstance(last_input, list)
assert (
len(last_input) == 4
) # User message + assistant message + function call + function result
# Verify the items are in the expected order
user_message = last_input[0]
assistant_message = last_input[1]
function_call = last_input[2]
function_result = last_input[3]
# Check user message
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# Check assistant message
assert assistant_message.get("role") == "assistant"
# Check function call
assert function_call.get("name") == "test_func"
assert function_call.get("arguments") == '{"arg": "foo"}'
# Check function result
assert function_result.get("type") == "function_call_output"
assert function_result.get("call_id") is not None
@pytest.mark.asyncio
async def test_default_multi_turn_drops_orphan_hosted_shell_calls() -> None:
model = FakeModel()
agent = Agent(
name="hosted-shell",
model=model,
tools=[ShellTool(environment={"type": "container_auto"})],
)
model.add_multiple_turn_outputs(
[
[make_shell_call("call_shell_1", id_value="shell_1", commands=["echo hi"])],
[get_text_message("done")],
]
)
result = await Runner.run(agent, input="user_message")
assert result.final_output == "done"
last_input = model.last_turn_args["input"]
assert isinstance(last_input, list)
assert len(last_input) == 1
assert not any(
isinstance(item, dict) and item.get("type") == "shell_call" for item in last_input
)
assert last_input[0].get("role") == "user"
assert last_input[0].get("content") == "user_message"
@pytest.mark.asyncio
async def test_manual_pending_shell_call_input_is_preserved_non_streamed() -> None:
model = FakeModel()
agent = Agent(
name="manual-shell",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
pending_shell_call = cast(
TResponseInputItem,
make_shell_call("manual_shell", id_value="shell_1", commands=["echo hi"]),
)
model.add_multiple_turn_outputs(
[
[get_function_tool_call("test_func", '{"arg": "foo"}')],
[get_text_message("done")],
]
)
result = await Runner.run(agent, input=[pending_shell_call])
assert result.final_output == "done"
assert isinstance(model.first_turn_args, dict)
assert any(
isinstance(item, dict)
and item.get("type") == "shell_call"
and item.get("call_id") == "manual_shell"
for item in model.first_turn_args["input"]
)
last_input = model.last_turn_args["input"]
assert isinstance(last_input, list)
assert any(
isinstance(item, dict)
and item.get("type") == "shell_call"
and item.get("call_id") == "manual_shell"
for item in last_input
)
@pytest.mark.asyncio
async def test_manual_pending_shell_call_input_is_preserved_non_streamed_with_session() -> None:
model = FakeModel()
agent = Agent(
name="manual-shell",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
session = SimpleListSession()
pending_shell_call = cast(
TResponseInputItem,
make_shell_call("manual_shell", id_value="shell_1", commands=["echo hi"]),
)
model.add_multiple_turn_outputs(
[
[get_function_tool_call("test_func", '{"arg": "foo"}')],
[get_text_message("done")],
]
)
result = await Runner.run(agent, input=[pending_shell_call], session=session)
assert result.final_output == "done"
assert isinstance(model.first_turn_args, dict)
assert any(
isinstance(item, dict)
and item.get("type") == "shell_call"
and item.get("call_id") == "manual_shell"
for item in model.first_turn_args["input"]
)
last_input = model.last_turn_args["input"]
assert isinstance(last_input, list)
assert any(
isinstance(item, dict)
and item.get("type") == "shell_call"
and item.get("call_id") == "manual_shell"
for item in last_input
)
@pytest.mark.asyncio
async def test_default_multi_turn_streamed_drops_orphan_hosted_shell_calls() -> None:
model = FakeModel()
agent = Agent(
name="hosted-shell",
model=model,
tools=[ShellTool(environment={"type": "container_auto"})],
)
model.add_multiple_turn_outputs(
[
[make_shell_call("call_shell_1", id_value="shell_1", commands=["echo hi"])],
[get_text_message("done")],
]
)
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
assert result.final_output == "done"
last_input = model.last_turn_args["input"]
assert isinstance(last_input, list)
assert len(last_input) == 1
assert not any(
isinstance(item, dict) and item.get("type") == "shell_call" for item in last_input
)
assert last_input[0].get("role") == "user"
assert last_input[0].get("content") == "user_message"
@pytest.mark.asyncio
async def test_manual_pending_shell_call_input_is_preserved_streamed() -> None:
model = FakeModel()
agent = Agent(name="manual-shell", model=model)
pending_shell_call = cast(
TResponseInputItem,
make_shell_call("manual_shell", id_value="shell_1", commands=["echo hi"]),
)
model.set_next_output([get_text_message("done")])
result = Runner.run_streamed(agent, input=[pending_shell_call])
async for _ in result.stream_events():
pass
assert result.final_output == "done"
last_input = model.last_turn_args["input"]
assert isinstance(last_input, list)
assert any(
isinstance(item, dict)
and item.get("type") == "shell_call"
and item.get("call_id") == "manual_shell"
for item in last_input
)
@pytest.mark.asyncio
async def test_manual_pending_shell_call_input_is_preserved_streamed_with_session() -> None:
model = FakeModel()
agent = Agent(name="manual-shell", model=model)
session = SimpleListSession()
pending_shell_call = cast(
TResponseInputItem,
make_shell_call("manual_shell", id_value="shell_1", commands=["echo hi"]),
)
model.set_next_output([get_text_message("done")])
result = Runner.run_streamed(agent, input=[pending_shell_call], session=session)
async for _ in result.stream_events():
pass
assert result.final_output == "done"
last_input = model.last_turn_args["input"]
assert isinstance(last_input, list)
assert any(
isinstance(item, dict)
and item.get("type") == "shell_call"
and item.get("call_id") == "manual_shell"
for item in last_input
)
@pytest.mark.asyncio
async def test_auto_previous_response_id_multi_turn():
"""Test that auto_previous_response_id=True enables
chaining from the first internal turn."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
# Second turn: final text message
[get_text_message("done")],
]
)
result = await Runner.run(agent, input="user_message", auto_previous_response_id=True)
assert result.final_output == "done"
# Check the first call
assert model.first_turn_args is not None
first_input = model.first_turn_args["input"]
# First call should include the original user input
assert isinstance(first_input, list)
assert len(first_input) == 1 # Should contain the user message
# The input should be the user message
user_message = first_input[0]
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# With auto_previous_response_id=True, first call should NOT have previous_response_id
assert model.first_turn_args.get("previous_response_id") is None
# Check the input from the second turn (after function execution)
last_input = model.last_turn_args["input"]
# With auto_previous_response_id=True, the second turn should only contain the tool output
assert isinstance(last_input, list)
assert len(last_input) == 1 # Only the function result
# The single item should be a tool result
tool_result_item = last_input[0]
assert tool_result_item.get("type") == "function_call_output"
assert tool_result_item.get("call_id") is not None
# With auto_previous_response_id=True, second call should have
# previous_response_id set to the first response
assert model.last_turn_args.get("previous_response_id") == "resp-789"
@pytest.mark.asyncio
async def test_auto_previous_response_id_multi_turn_streamed():
"""Test that auto_previous_response_id=True enables
chaining from the first internal turn (streamed mode)."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
# Second turn: final text message
[get_text_message("done")],
]
)
result = Runner.run_streamed(agent, input="user_message", auto_previous_response_id=True)
async for _ in result.stream_events():
pass
assert result.final_output == "done"
# Check the first call
assert model.first_turn_args is not None
first_input = model.first_turn_args["input"]
# First call should include the original user input
assert isinstance(first_input, list)
assert len(first_input) == 1 # Should contain the user message
# The input should be the user message
user_message = first_input[0]
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# With auto_previous_response_id=True, first call should NOT have previous_response_id
assert model.first_turn_args.get("previous_response_id") is None
# Check the input from the second turn (after function execution)
last_input = model.last_turn_args["input"]
# With auto_previous_response_id=True, the second turn should only contain the tool output
assert isinstance(last_input, list)
assert len(last_input) == 1 # Only the function result
# The single item should be a tool result
tool_result_item = last_input[0]
assert tool_result_item.get("type") == "function_call_output"
assert tool_result_item.get("call_id") is not None
# With auto_previous_response_id=True, second call should have
# previous_response_id set to the first response
assert model.last_turn_args.get("previous_response_id") == "resp-789"
@pytest.mark.asyncio
async def test_without_previous_response_id_and_auto_previous_response_id_no_chaining():
"""Test that without previous_response_id and auto_previous_response_id,
internal turns don't chain."""
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("test_func", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')],
# Second turn: final text message
[get_text_message("done")],
]
)
# Call without passing previous_response_id and without passing auto_previous_response_id
result = await Runner.run(agent, input="user_message")
assert result.final_output == "done"
# Check the first call
assert model.first_turn_args is not None
first_input = model.first_turn_args["input"]
# First call should include the original user input
assert isinstance(first_input, list)
assert len(first_input) == 1 # Should contain the user message
# The input should be the user message
user_message = first_input[0]
assert user_message.get("role") == "user"
assert user_message.get("content") == "user_message"
# First call should NOT have previous_response_id
assert model.first_turn_args.get("previous_response_id") is None
# Check the input from the second turn (after function execution)
last_input = model.last_turn_args["input"]
# Without passing previous_response_id and auto_previous_response_id,
# the second turn should contain all items (no chaining):
# user message, assistant response, function call, and tool result
assert isinstance(last_input, list)
assert len(last_input) == 4 # User message, assistant message, function call, and tool result
# Second call should also NOT have previous_response_id (no chaining)
assert model.last_turn_args.get("previous_response_id") is None
@pytest.mark.asyncio
async def test_dynamic_tool_addition_run() -> None:
"""Test that tools can be added to an agent during a run."""
model = FakeModel()
executed: dict[str, bool] = {"called": False}
agent = Agent(name="test", model=model, tool_use_behavior="run_llm_again")
@function_tool(name_override="tool2")
def tool2() -> str:
executed["called"] = True
return "result2"
@function_tool(name_override="add_tool")
async def add_tool() -> str:
agent.tools.append(tool2)
return "added"
agent.tools.append(add_tool)
model.add_multiple_turn_outputs(
[
[get_function_tool_call("add_tool", json.dumps({}), call_id="call-add-tool")],
[get_function_tool_call("tool2", json.dumps({}), call_id="call-tool-two")],
[get_text_message("done")],
]
)
result = await Runner.run(agent, input="start")
assert executed["called"] is True
assert result.final_output == "done"
@pytest.mark.asyncio
async def test_tool_not_found_behavior_returns_error_to_model() -> None:
model = FakeModel()
agent = Agent(name="test", model=model, tool_use_behavior="run_llm_again")
model.add_multiple_turn_outputs(
[
[get_function_tool_call("missing_tool", "{}", call_id="call_missing")],
[get_text_message("recovered")],
]
)
result = await Runner.run(
agent,
input="start",
run_config=RunConfig(tool_not_found_behavior="return_error_to_model"),
)
assert result.final_output == "recovered"
second_turn_input = model.last_turn_args["input"]
assert isinstance(second_turn_input, list)
tool_outputs = [
item
for item in second_turn_input
if isinstance(item, dict) and item.get("type") == "function_call_output"
]
assert tool_outputs == [
{
"call_id": "call_missing",
"output": "Tool 'missing_tool' not found.",
"type": "function_call_output",
}
]
@pytest.mark.asyncio
async def test_tool_not_found_behavior_uses_tool_error_formatter() -> None:
model = FakeModel()
agent = Agent(name="test", model=model, tool_use_behavior="run_llm_again")
model.add_multiple_turn_outputs(
[
[get_function_tool_call("missing_tool", "{}", call_id="call_missing")],
[get_text_message("recovered")],
]
)
seen_kinds: list[str] = []
async def formatter(args: Any) -> str | None:
seen_kinds.append(args.kind)
if args.kind != "tool_not_found":
return None
return f"{args.tool_name} unavailable for {args.call_id}"
result = await Runner.run(
agent,
input="start",
run_config=RunConfig(
tool_not_found_behavior="return_error_to_model",
tool_error_formatter=formatter,
),
)
assert result.final_output == "recovered"
assert seen_kinds == ["tool_not_found"]
second_turn_input = model.last_turn_args["input"]
assert isinstance(second_turn_input, list)
tool_outputs = [
item
for item in second_turn_input
if isinstance(item, dict) and item.get("type") == "function_call_output"
]
assert tool_outputs == [
{
"call_id": "call_missing",
"output": "missing_tool unavailable for call_missing",
"type": "function_call_output",
}
]
@pytest.mark.asyncio
async def test_tool_not_found_behavior_handles_mixed_function_tool_calls() -> None:
model = FakeModel()
calls: list[str] = []
@function_tool(name_override="known_tool")
async def known_tool() -> str:
calls.append("known_tool")
return "known result"
agent = Agent(
name="test",
model=model,
tools=[known_tool],
tool_use_behavior="run_llm_again",
)
model.add_multiple_turn_outputs(
[
[
get_function_tool_call("missing_tool", "{}", call_id="call_missing"),
get_function_tool_call("known_tool", "{}", call_id="call_known"),
],
[get_text_message("done")],
]
)
result = await Runner.run(
agent,
input="start",
run_config=RunConfig(tool_not_found_behavior="return_error_to_model"),
)
assert calls == ["known_tool"]
assert result.final_output == "done"
second_turn_input = model.last_turn_args["input"]
assert isinstance(second_turn_input, list)
tool_outputs = {
item.get("call_id"): item.get("output")
for item in second_turn_input
if isinstance(item, dict) and item.get("type") == "function_call_output"
}
assert tool_outputs == {
"call_known": "known result",
"call_missing": "Tool 'missing_tool' not found.",
}
@pytest.mark.asyncio
async def test_session_add_items_called_multiple_times_for_multi_turn_completion():
"""Test that SQLiteSession.add_items is called multiple times
during a multi-turn agent completion.
"""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_agent_runner_session_multi_turn_calls.db"
session_id = "runner_session_multi_turn_calls"
session = SQLiteSession(session_id, db_path)
# Define a tool that will be called by the orchestrator agent
@function_tool
async def echo_tool(text: str) -> str:
return f"Echo: {text}"
# Orchestrator agent that calls the tool multiple times in one completion
orchestrator_agent = Agent(
name="orchestrator_agent",
instructions=(
"Call echo_tool twice with inputs of 'foo' and 'bar', then return a summary."
),
tools=[echo_tool],
)
# Patch the model to simulate two tool calls and a final message
model = FakeModel()
orchestrator_agent.model = model
model.add_multiple_turn_outputs(
[
# First turn: tool call
[get_function_tool_call("echo_tool", json.dumps({"text": "foo"}), call_id="1")],
# Second turn: tool call
[get_function_tool_call("echo_tool", json.dumps({"text": "bar"}), call_id="2")],
# Third turn: final output
[get_final_output_message("Summary: Echoed foo and bar")],
]
)
# Patch add_items to count calls
with patch.object(SQLiteSession, "add_items", wraps=session.add_items) as mock_add_items:
result = await Runner.run(orchestrator_agent, input="foo and bar", session=session)
expected_items = [
{"content": "foo and bar", "role": "user"},
{
"arguments": '{"text": "foo"}',
"call_id": "1",
"name": "echo_tool",
"type": "function_call",
"id": "1",
},
{"call_id": "1", "output": "Echo: foo", "type": "function_call_output"},
{
"arguments": '{"text": "bar"}',
"call_id": "2",
"name": "echo_tool",
"type": "function_call",
"id": "1",
},
{"call_id": "2", "output": "Echo: bar", "type": "function_call_output"},
{
"id": "1",
"content": [
{
"annotations": [],
"logprobs": [],
"text": "Summary: Echoed foo and bar",
"type": "output_text",
}
],
"role": "assistant",
"status": "completed",
"type": "message",
},
]
expected_calls = [
# First call is the initial input
(([expected_items[0]],),),
# Second call is the first tool call and its result
(([expected_items[1], expected_items[2]],),),
# Third call is the second tool call and its result
(([expected_items[3], expected_items[4]],),),
# Fourth call is the final output
(([expected_items[5]],),),
]
assert mock_add_items.call_args_list == expected_calls
assert result.final_output == "Summary: Echoed foo and bar"
assert (await session.get_items()) == expected_items
session.close()
@pytest.mark.asyncio
async def test_execute_approved_tools_with_non_function_tool():
"""Test _execute_approved_tools handles non-FunctionTool."""
model = FakeModel()
# Create a computer tool (not a FunctionTool)
class MockComputer(Computer):
@property
def environment(self) -> str: # type: ignore[override]
return "mac"
@property
def dimensions(self) -> tuple[int, int]:
return (1920, 1080)
def screenshot(self) -> str:
return "screenshot"
def click(self, x: int, y: int, button: str) -> None:
pass
def double_click(self, x: int, y: int) -> None:
pass
def drag(self, path: list[tuple[int, int]]) -> None:
pass
def keypress(self, keys: list[str]) -> None:
pass
def move(self, x: int, y: int) -> None:
pass
def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
pass
def type(self, text: str) -> None:
pass
def wait(self) -> None:
pass
computer = MockComputer()
computer_tool = ComputerTool(computer=computer)
agent = Agent(name="TestAgent", model=model, tools=[computer_tool])
# Create an approved tool call for the computer tool
# ComputerTool is not a function tool and should still fail approval execution cleanly.
tool_call = get_function_tool_call(computer_tool.name, "{}")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
# Should add error message about tool not being a function tool
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert "not a function tool" in generated_items[0].output.lower()
@pytest.mark.asyncio
async def test_execute_approved_tools_with_rejected_tool():
"""Test _execute_approved_tools handles rejected tools."""
tool_called = False
async def test_tool() -> str:
nonlocal tool_called
tool_called = True
return "tool_result"
tool = function_tool(test_tool, name_override="test_tool")
_, agent = make_model_and_agent(tools=[tool])
# Create a rejected tool call
tool_call = get_function_tool_call("test_tool", "{}")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=False,
)
# Should add rejection message
assert len(generated_items) == 1
assert "not approved" in generated_items[0].output.lower()
assert not tool_called # Tool should not have been executed
@pytest.mark.asyncio
async def test_execute_approved_tools_rejects_changed_pending_invocation() -> None:
"""A decision for one payload must not authorize a changed interruption."""
tool_called = False
async def test_tool(value: str) -> str:
nonlocal tool_called
tool_called = True
return value
tool = function_tool(test_tool, name_override="test_tool")
_, agent = make_model_and_agent(tools=[tool])
approved_call = get_function_tool_call(
"test_tool",
'{"value":"safe"}',
call_id="call-shared",
)
changed_call = get_function_tool_call(
"test_tool",
'{"value":"changed"}',
call_id="call-shared",
)
assert isinstance(approved_call, ResponseFunctionToolCall)
assert isinstance(changed_call, ResponseFunctionToolCall)
approved_item = ToolApprovalItem(agent=agent, raw_item=approved_call)
changed_item = ToolApprovalItem(agent=agent, raw_item=changed_call)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await run_execute_approved_tools(
agent=agent,
approval_item=changed_item,
approve=None,
mutate_state=lambda state, _item: state.approve(approved_item),
)
assert tool_called is False
@pytest.mark.asyncio
async def test_execute_approved_tools_with_rejected_tool_uses_run_level_formatter():
"""Rejected tools should prefer RunConfig tool error formatter output."""
async def test_tool() -> str:
return "tool_result"
tool = function_tool(test_tool, name_override="test_tool")
_, agent = make_model_and_agent(tools=[tool])
tool_call = get_function_tool_call("test_tool", "{}")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=False,
run_config=RunConfig(
tool_error_formatter=lambda args: f"run-level {args.tool_name} denied ({args.call_id})"
),
)
assert len(generated_items) == 1
assert generated_items[0].output == "run-level test_tool denied (2)"
@pytest.mark.asyncio
async def test_execute_approved_tools_with_rejected_tool_prefers_explicit_message():
"""Rejected tools should prefer explicit rejection messages over the formatter."""
async def test_tool() -> str:
return "tool_result"
tool = function_tool(test_tool, name_override="test_tool")
_, agent = make_model_and_agent(tools=[tool])
tool_call = get_function_tool_call("test_tool", "{}")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=False,
run_config=RunConfig(
tool_error_formatter=lambda args: f"run-level {args.tool_name} denied ({args.call_id})"
),
mutate_state=lambda state, item: state.reject(
item, rejection_message="explicit rejection message"
),
)
assert len(generated_items) == 1
assert generated_items[0].output == "explicit rejection message"
@pytest.mark.asyncio
async def test_execute_approved_tools_with_rejected_deferred_tool_uses_display_name():
"""Rejected deferred tools should collapse synthetic namespaces in formatter output."""
async def get_weather() -> str:
return "sunny"
tool = function_tool(get_weather, name_override="get_weather", defer_loading=True)
_, agent = make_model_and_agent(tools=[tool])
tool_call = get_function_tool_call("get_weather", "{}", namespace="get_weather")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(
agent=agent,
raw_item=tool_call,
tool_name="get_weather",
tool_namespace="get_weather",
)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=False,
run_config=RunConfig(
tool_error_formatter=lambda args: f"run-level {args.tool_name} denied ({args.call_id})"
),
)
assert len(generated_items) == 1
assert generated_items[0].output == "run-level get_weather denied (2)"
@pytest.mark.asyncio
async def test_execute_approved_tools_with_rejected_tool_formatter_none_uses_default():
"""Rejected tools should use default message when formatter returns None."""
async def test_tool() -> str:
return "tool_result"
tool = function_tool(test_tool, name_override="test_tool")
_, agent = make_model_and_agent(tools=[tool])
tool_call = get_function_tool_call("test_tool", "{}")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=False,
run_config=RunConfig(tool_error_formatter=lambda _args: None),
)
assert len(generated_items) == 1
assert generated_items[0].output == "Tool execution was not approved."
@pytest.mark.asyncio
async def test_execute_approved_tools_with_unclear_status():
"""Test _execute_approved_tools handles unclear approval status."""
tool_called = False
async def test_tool() -> str:
nonlocal tool_called
tool_called = True
return "tool_result"
tool = function_tool(test_tool, name_override="test_tool")
_, agent = make_model_and_agent(tools=[tool])
# Create a tool call with unclear status (neither approved nor rejected)
tool_call = get_function_tool_call("test_tool", "{}")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=None,
)
# Should add unclear status message
assert len(generated_items) == 1
assert "unclear" in generated_items[0].output.lower()
assert not tool_called # Tool should not have been executed
@pytest.mark.asyncio
async def test_execute_approved_tools_with_missing_tool():
"""Test _execute_approved_tools handles missing tools."""
_, agent = make_model_and_agent()
# Agent has no tools
# Create an approved tool call for a tool that doesn't exist
tool_call = get_function_tool_call("nonexistent_tool", "{}")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
# Should add error message about tool not found
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert "not found" in generated_items[0].output.lower()
@pytest.mark.asyncio
async def test_execute_approved_tools_does_not_resolve_explicit_namespaced_tool_by_bare_name():
crm_calls: list[str] = []
billing_calls: list[str] = []
async def crm_lookup() -> str:
crm_calls.append("crm")
return "crm"
async def billing_lookup() -> str:
billing_calls.append("billing")
return "billing"
crm_tool = tool_namespace(
name="crm",
description="CRM tools",
tools=[function_tool(crm_lookup, name_override="lookup_account")],
)[0]
billing_tool = tool_namespace(
name="billing",
description="Billing tools",
tools=[function_tool(billing_lookup, name_override="lookup_account")],
)[0]
agent = Agent(name="TestAgent", model=FakeModel(), tools=[crm_tool, billing_tool])
tool_call = get_function_tool_call("lookup_account", "{}", call_id="call-ambiguous")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert "not found" in generated_items[0].output.lower()
assert crm_calls == []
assert billing_calls == []
@pytest.mark.asyncio
async def test_execute_approved_tools_does_not_fallback_from_namespaced_approval_to_bare_tool():
bare_calls: list[str] = []
async def bare_lookup() -> str:
bare_calls.append("bare")
return "bare"
bare_tool = function_tool(bare_lookup, name_override="lookup_account")
agent = Agent(name="TestAgent", model=FakeModel(), tools=[bare_tool])
tool_call = get_function_tool_call(
"lookup_account",
"{}",
call_id="call-billing",
namespace="billing",
)
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert "billing.lookup_account" in generated_items[0].output
assert "not found" in generated_items[0].output.lower()
assert bare_calls == []
@pytest.mark.asyncio
async def test_execute_approved_tools_prefers_visible_top_level_function_over_deferred_same_name_tool( # noqa: E501
):
visible_calls: list[str] = []
deferred_calls: list[str] = []
async def visible_lookup() -> str:
visible_calls.append("visible")
return "visible"
async def deferred_lookup() -> str:
deferred_calls.append("deferred")
return "deferred"
visible_tool = function_tool(visible_lookup, name_override="lookup_account")
deferred_tool = function_tool(
deferred_lookup,
name_override="lookup_account",
defer_loading=True,
)
agent = Agent(name="TestAgent", model=FakeModel(), tools=[visible_tool, deferred_tool])
tool_call = get_function_tool_call("lookup_account", "{}", call_id="call-visible")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert generated_items[0].output == "visible"
assert visible_calls == ["visible"]
assert deferred_calls == []
@pytest.mark.asyncio
async def test_execute_approved_tools_uses_internal_lookup_key_for_deferred_top_level_calls() -> (
None
):
visible_calls: list[str] = []
deferred_calls: list[str] = []
async def visible_lookup() -> str:
visible_calls.append("visible")
return "visible"
async def deferred_lookup() -> str:
deferred_calls.append("deferred")
return "deferred"
visible_tool = function_tool(
visible_lookup,
name_override="lookup_account.lookup_account",
)
deferred_tool = function_tool(
deferred_lookup,
name_override="lookup_account",
defer_loading=True,
)
agent = Agent(name="TestAgent", model=FakeModel(), tools=[visible_tool, deferred_tool])
tool_call = get_function_tool_call(
"lookup_account",
"{}",
call_id="call-deferred",
namespace="lookup_account",
)
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert generated_items[0].output == "deferred"
assert visible_calls == []
assert deferred_calls == ["deferred"]
@pytest.mark.asyncio
async def test_deferred_collision_rejection_prefers_explicit_message() -> None:
async def visible_lookup() -> str:
return "visible"
async def deferred_lookup() -> str:
return "deferred"
visible_tool = function_tool(
visible_lookup,
name_override="lookup_account.lookup_account",
)
deferred_tool = function_tool(
deferred_lookup,
name_override="lookup_account",
defer_loading=True,
)
agent = Agent(name="TestAgent", model=FakeModel(), tools=[visible_tool, deferred_tool])
tool_call = get_function_tool_call(
"lookup_account",
"{}",
call_id="call-deferred",
namespace="lookup_account",
)
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(
agent=agent,
raw_item=tool_call,
tool_name="lookup_account",
tool_namespace="lookup_account",
tool_lookup_key=("deferred_top_level", "lookup_account"),
)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=False,
run_config=RunConfig(
tool_error_formatter=lambda args: f"run-level {args.tool_name} denied ({args.call_id})"
),
mutate_state=lambda state, item: state.reject(
item, rejection_message="explicit rejection message"
),
)
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert generated_items[0].output == "explicit rejection message"
@pytest.mark.asyncio
async def test_execute_approved_tools_uses_last_duplicate_top_level_function():
first_calls: list[str] = []
second_calls: list[str] = []
async def first_lookup() -> str:
first_calls.append("first")
return "first"
async def second_lookup() -> str:
second_calls.append("second")
return "second"
first_tool = function_tool(first_lookup, name_override="lookup_account")
second_tool = function_tool(second_lookup, name_override="lookup_account")
agent = Agent(name="TestAgent", model=FakeModel(), tools=[first_tool, second_tool])
tool_call = get_function_tool_call("lookup_account", "{}", call_id="call-shadow")
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert generated_items[0].output == "second"
assert first_calls == []
assert second_calls == ["second"]
@pytest.mark.asyncio
async def test_execute_approved_tools_rejects_missing_call_id():
"""Test _execute_approved_tools rejects tool approvals without call IDs."""
_, agent = make_model_and_agent()
tool_call = {"type": "function_call", "name": "test_tool"}
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
with pytest.raises(ModelBehaviorError, match="non-empty call ID"):
await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
@pytest.mark.asyncio
async def test_execute_approved_tools_with_invalid_raw_item_type():
"""Test _execute_approved_tools handles approvals with unsupported raw_item types."""
async def test_tool() -> str:
return "tool_result"
tool = function_tool(test_tool, name_override="test_tool")
_, agent = make_model_and_agent(tools=[tool])
tool_call = {
"type": "function_call",
"name": "test_tool",
"call_id": "call-1",
"arguments": "{}",
}
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert "invalid raw_item type" in generated_items[0].output.lower()
@pytest.mark.asyncio
async def test_execute_approved_tools_instance_method():
"""Ensure execute_approved_tools runs approved tools as expected."""
tool_called = False
async def test_tool() -> str:
nonlocal tool_called
tool_called = True
return "tool_result"
tool = function_tool(test_tool, name_override="test_tool")
_, agent = make_model_and_agent(tools=[tool])
tool_call = get_function_tool_call("test_tool", json.dumps({}))
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
# Tool should have been called
assert tool_called is True
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert generated_items[0].output == "tool_result"
@pytest.mark.asyncio
async def test_execute_approved_tools_timeout_returns_error_as_result() -> None:
async def slow_tool() -> str:
await asyncio.sleep(0.2)
return "tool_result"
tool = function_tool(slow_tool, name_override="test_tool", timeout=0.01)
_, agent = make_model_and_agent(tools=[tool])
tool_call = get_function_tool_call("test_tool", json.dumps({}))
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
generated_items = await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)
assert len(generated_items) == 1
assert isinstance(generated_items[0], ToolCallOutputItem)
assert "timed out" in generated_items[0].output.lower()
@pytest.mark.asyncio
async def test_execute_approved_tools_timeout_can_raise_exception() -> None:
async def slow_tool() -> str:
await asyncio.sleep(0.2)
return "tool_result"
tool = function_tool(
slow_tool,
name_override="test_tool",
timeout=0.01,
timeout_behavior="raise_exception",
)
_, agent = make_model_and_agent(tools=[tool])
tool_call = get_function_tool_call("test_tool", json.dumps({}))
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
with pytest.raises(ToolTimeoutError, match="timed out"):
await run_execute_approved_tools(
agent=agent,
approval_item=approval_item,
approve=True,
)