google--adk-python/tests/unittests/cli/plugins/test_replay_plugin.py

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for the replay plugin's load / replay / cleanup lifecycle."""

from typing import Any
from typing import Optional

from google.adk.agents.callback_context import CallbackContext
from google.adk.cli.plugins.recordings_schema import Recording
from google.adk.cli.plugins.recordings_schema import Recordings
from google.adk.cli.plugins.recordings_schema import ToolRecording
from google.adk.cli.plugins.replay_plugin import ReplayConfigError
from google.adk.cli.plugins.replay_plugin import ReplayPlugin
from google.adk.cli.plugins.replay_plugin import ReplayVerificationError
from google.adk.tools.base_tool import BaseTool
from google.adk.utils.yaml_utils import dump_pydantic_to_yaml
from google.genai import types
import pytest

from ... import testing_utils

_NON_STREAMING_FILE = 'generated-recordings.yaml'
_STREAMING_FILE = 'generated-recordings-sse.yaml'


class _SpyTool(BaseTool):
  """Tool that records the args it was actually executed with."""

  def __init__(self, name: str = 'roll_die', live_result: Any = None):
    super().__init__(name=name, description='test tool')
    self.live_calls: list[dict[str, Any]] = []
    self._live_result = (
        {'result': 'live'} if live_result is None else live_result
    )

  async def run_async(self, *, args, tool_context):
    self.live_calls.append(args)
    return self._live_result


def _recording(
    *,
    agent_name: str = 'agent_a',
    user_message_index: int = 0,
    tool_name: str = 'roll_die',
    args: Optional[dict[str, Any]] = None,
    response: Optional[dict[str, Any]] = None,
    call_id: str = 'fc-1',
) -> Recording:
  return Recording(
      user_message_index=user_message_index,
      agent_name=agent_name,
      tool_recording=ToolRecording(
          tool_call=types.FunctionCall(
              id=call_id, name=tool_name, args=args or {'sides': 6}
          ),
          tool_response=types.FunctionResponse(
              id=call_id, name=tool_name, response=response or {'result': 4}
          ),
      ),
  )


def _write_recordings(case_dir, recordings, *, file_name=_NON_STREAMING_FILE):
  dump_pydantic_to_yaml(
      Recordings(recordings=recordings),
      case_dir / file_name,
      sort_keys=False,
  )


async def _make_invocation(
    *,
    case_dir=None,
    user_message_index: int = 0,
    streaming_mode: Optional[str] = 'none',
    agent_names: tuple[str, ...] = ('agent_a',),
):
  """Builds one invocation plus a per-agent context sharing its session."""
  invocation_context = await testing_utils.create_invocation_context(
      testing_utils.create_test_agent(name=agent_names[0])
  )
  if case_dir is not None:
    config: dict[str, Any] = {
        'dir': str(case_dir),
        'user_message_index': user_message_index,
    }
    if streaming_mode is not None:
      config['streaming_mode'] = streaming_mode
    invocation_context.session.state['_adk_replay_config'] = config

  contexts = {agent_names[0]: CallbackContext(invocation_context)}
  for name in agent_names[1:]:
    contexts[name] = CallbackContext(
        invocation_context.model_copy(
            update={'agent': testing_utils.create_test_agent(name=name)}
        )
    )
  return invocation_context, contexts


async def test_before_run_without_replay_config_leaves_plugin_inert(tmp_path):
  """No replay config means the plugin must not intercept anything."""
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(case_dir=None)
  tool = _SpyTool()

  before_run_result = await plugin.before_run_callback(
      invocation_context=invocation_context
  )
  replayed = await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
  )

  # None tells the runtime to execute the tool itself; the plugin neither ran
  # the tool nor consumed a recording.
  assert before_run_result is None
  assert replayed is None
  assert tool.live_calls == []


async def test_before_run_with_partial_replay_config_leaves_plugin_inert(
    tmp_path,
):
  """A config missing user_message_index must not half-enable replay."""
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(case_dir=tmp_path)
  invocation_context.session.state['_adk_replay_config'] = {
      'dir': str(tmp_path),
      'streaming_mode': 'none',
  }
  tool = _SpyTool()

  await plugin.before_run_callback(invocation_context=invocation_context)
  replayed = await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
  )

  assert replayed is None
  assert tool.live_calls == []


async def test_before_tool_returns_recorded_response_not_live_result(tmp_path):
  """The recorded response wins over whatever the live tool returns."""
  _write_recordings(tmp_path, [_recording(response={'result': 4})])
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(case_dir=tmp_path)
  tool = _SpyTool(live_result={'result': 'live'})

  await plugin.before_run_callback(invocation_context=invocation_context)
  replayed = await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
  )

  assert replayed == {'result': 4}


async def test_before_tool_still_executes_the_underlying_tool(tmp_path):
  """Replay verifies the tool runs; only its response is substituted."""
  _write_recordings(tmp_path, [_recording(args={'sides': 6})])
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(case_dir=tmp_path)
  tool = _SpyTool()

  await plugin.before_run_callback(invocation_context=invocation_context)
  await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
  )

  assert tool.live_calls == [{'sides': 6}]


async def test_before_run_reads_the_sse_file_in_sse_streaming_mode(tmp_path):
  """streaming_mode selects which recordings file is authoritative."""
  _write_recordings(
      tmp_path,
      [_recording(response={'result': 'non-streaming'})],
      file_name=_NON_STREAMING_FILE,
  )
  _write_recordings(
      tmp_path,
      [_recording(response={'result': 'streaming'})],
      file_name=_STREAMING_FILE,
  )
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(
      case_dir=tmp_path, streaming_mode='sse'
  )

  await plugin.before_run_callback(invocation_context=invocation_context)
  replayed = await plugin.before_tool_callback(
      tool=_SpyTool(),
      tool_args={'sides': 6},
      tool_context=contexts['agent_a'],
  )

  assert replayed == {'result': 'streaming'}


async def test_before_run_reads_the_plain_file_in_non_streaming_mode(tmp_path):
  """The mirror of the sse case, so a swapped file name cannot pass both."""
  _write_recordings(
      tmp_path,
      [_recording(response={'result': 'non-streaming'})],
      file_name=_NON_STREAMING_FILE,
  )
  _write_recordings(
      tmp_path,
      [_recording(response={'result': 'streaming'})],
      file_name=_STREAMING_FILE,
  )
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(
      case_dir=tmp_path, streaming_mode='none'
  )

  await plugin.before_run_callback(invocation_context=invocation_context)
  replayed = await plugin.before_tool_callback(
      tool=_SpyTool(),
      tool_args={'sides': 6},
      tool_context=contexts['agent_a'],
  )

  assert replayed == {'result': 'non-streaming'}


async def test_before_run_unsupported_streaming_mode_raises_value_error(
    tmp_path,
):
  """An unknown streaming mode must fail loudly, not pick a default file."""
  _write_recordings(tmp_path, [_recording()])
  plugin = ReplayPlugin()
  invocation_context, _ = await _make_invocation(
      case_dir=tmp_path, streaming_mode='bidi'
  )

  with pytest.raises(ValueError, match='Unsupported streaming mode: bidi'):
    await plugin.before_run_callback(invocation_context=invocation_context)


async def test_before_run_missing_recordings_file_raises_config_error(
    tmp_path,
):
  """A missing file is a configuration problem, reported with its path."""
  plugin = ReplayPlugin()
  invocation_context, _ = await _make_invocation(case_dir=tmp_path)

  with pytest.raises(ReplayConfigError, match='Recordings file not found'):
    await plugin.before_run_callback(invocation_context=invocation_context)


async def test_before_run_unparsable_recordings_raise_config_error(tmp_path):
  """Schema violations surface as ReplayConfigError, not a pydantic error."""
  (tmp_path / _NON_STREAMING_FILE).write_text(
      'recordings:\n  - user_message_index: 0\n    agent_name: a\n'
      '    tool_recordings: {}\n',
      encoding='utf-8',
  )
  plugin = ReplayPlugin()
  invocation_context, _ = await _make_invocation(case_dir=tmp_path)

  with pytest.raises(ReplayConfigError, match='Failed to load recordings'):
    await plugin.before_run_callback(invocation_context=invocation_context)


async def test_before_tool_without_loaded_state_raises_config_error(tmp_path):
  """Replaying without a preceding before_run is a misuse, not a silent pass."""
  _write_recordings(tmp_path, [_recording()])
  plugin = ReplayPlugin()
  _, contexts = await _make_invocation(case_dir=tmp_path)

  with pytest.raises(ReplayConfigError, match='Replay state not initialized'):
    await plugin.before_tool_callback(
        tool=_SpyTool(),
        tool_args={'sides': 6},
        tool_context=contexts['agent_a'],
    )


async def test_before_tool_tool_name_mismatch_raises_verification_error(
    tmp_path,
):
  """Calling a different tool than recorded fails verification."""
  _write_recordings(tmp_path, [_recording(tool_name='roll_die')])
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(case_dir=tmp_path)

  await plugin.before_run_callback(invocation_context=invocation_context)
  with pytest.raises(ReplayVerificationError) as exc_info:
    await plugin.before_tool_callback(
        tool=_SpyTool(name='flip_coin'),
        tool_args={'sides': 6},
        tool_context=contexts['agent_a'],
    )

  message = str(exc_info.value)
  assert 'Tool name mismatch' in message
  assert 'roll_die' in message
  assert 'flip_coin' in message


async def test_before_tool_args_mismatch_raises_verification_error(tmp_path):
  """The recorded args must match exactly, not just the tool name."""
  _write_recordings(tmp_path, [_recording(args={'sides': 6})])
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(case_dir=tmp_path)

  await plugin.before_run_callback(invocation_context=invocation_context)
  with pytest.raises(ReplayVerificationError) as exc_info:
    await plugin.before_tool_callback(
        tool=_SpyTool(),
        tool_args={'sides': 20},
        tool_context=contexts['agent_a'],
    )

  message = str(exc_info.value)
  assert 'Tool args mismatch' in message
  assert "'sides': 20" in message


async def test_before_tool_beyond_recorded_calls_raises_verification_error(
    tmp_path,
):
  """An extra tool call past the end of the recordings is a replay failure."""
  _write_recordings(tmp_path, [_recording()])
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(case_dir=tmp_path)
  tool = _SpyTool()

  await plugin.before_run_callback(invocation_context=invocation_context)
  await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
  )

  with pytest.raises(ReplayVerificationError) as exc_info:
    await plugin.before_tool_callback(
        tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
    )

  message = str(exc_info.value)
  assert 'more tool requests than expected' in message
  assert 'Expected 1' in message


async def test_before_tool_advances_a_separate_index_per_agent(tmp_path):
  """Each agent has its own replay index; a sibling's call must not shift it."""
  _write_recordings(
      tmp_path,
      [
          _recording(
              agent_name='agent_a', args={'sides': 6}, response={'result': 4}
          ),
          _recording(
              agent_name='agent_b', args={'sides': 8}, response={'result': 7}
          ),
          _recording(
              agent_name='agent_a', args={'sides': 20}, response={'result': 17}
          ),
      ],
  )
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(
      case_dir=tmp_path, agent_names=('agent_a', 'agent_b')
  )
  tool = _SpyTool()

  await plugin.before_run_callback(invocation_context=invocation_context)
  first_a = await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
  )
  first_b = await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 8}, tool_context=contexts['agent_b']
  )
  second_a = await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 20}, tool_context=contexts['agent_a']
  )

  assert [first_a, first_b, second_a] == [
      {'result': 4},
      {'result': 7},
      {'result': 17},
  ]


async def test_before_tool_ignores_recordings_for_other_user_messages(
    tmp_path,
):
  """Only the recordings for the configured user message are replayable."""
  _write_recordings(
      tmp_path,
      [
          _recording(
              user_message_index=0,
              args={'sides': 6},
              response={'result': 'first turn'},
          ),
          _recording(
              user_message_index=1,
              args={'sides': 20},
              response={'result': 'second turn'},
          ),
      ],
  )
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(
      case_dir=tmp_path, user_message_index=1
  )
  tool = _SpyTool()

  await plugin.before_run_callback(invocation_context=invocation_context)
  replayed = await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 20}, tool_context=contexts['agent_a']
  )

  assert replayed == {'result': 'second turn'}
  # The turn-0 recording is not available to this invocation.
  with pytest.raises(ReplayVerificationError, match='Expected 1'):
    await plugin.before_tool_callback(
        tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
    )


async def test_after_run_discards_the_invocation_state(tmp_path):
  """Cleanup is observable: a later tool call no longer finds replay state."""
  _write_recordings(tmp_path, [_recording(), _recording(call_id='fc-2')])
  plugin = ReplayPlugin()
  invocation_context, contexts = await _make_invocation(case_dir=tmp_path)
  tool = _SpyTool()

  await plugin.before_run_callback(invocation_context=invocation_context)
  await plugin.before_tool_callback(
      tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
  )
  await plugin.after_run_callback(invocation_context=invocation_context)

  with pytest.raises(ReplayConfigError, match='Replay state not initialized'):
    await plugin.before_tool_callback(
        tool=tool, tool_args={'sides': 6}, tool_context=contexts['agent_a']
    )