From 703cf43f6b03550ec2b02def526f3f4ba98f7abd Mon Sep 17 00:00:00 2001 From: George Weale Date: Thu, 13 Aug 2026 14:52:11 -0700 Subject: [PATCH] fix: send media attached to a tool response to non-Gemini models Co-authored-by: George Weale PiperOrigin-RevId: 964308319 --- .../adk/integrations/oci/_oci_genai_llm.py | 25 ++++++ src/google/adk/models/anthropic_llm.py | 73 +++++++++++++++- src/google/adk/models/apigee_llm.py | 30 ++++++- src/google/adk/models/lite_llm.py | 21 +++++ .../integrations/oci/test_oci_genai_llm.py | 25 ++++++ tests/unittests/models/test_anthropic_llm.py | 83 +++++++++++++++++++ tests/unittests/models/test_apigee_llm.py | 51 ++++++++++++ tests/unittests/models/test_litellm.py | 60 ++++++++++++++ 8 files changed, 366 insertions(+), 2 deletions(-) diff --git a/src/google/adk/integrations/oci/_oci_genai_llm.py b/src/google/adk/integrations/oci/_oci_genai_llm.py index 93090bf5..5f2668a8 100644 --- a/src/google/adk/integrations/oci/_oci_genai_llm.py +++ b/src/google/adk/integrations/oci/_oci_genai_llm.py @@ -156,6 +156,25 @@ def _media_blocks_for_part(part: types.Part) -> list[Any]: ] +def _function_response_media_blocks( + function_response: types.FunctionResponse, +) -> list[Any]: + """Converts media a tool attached to its response into OCI content blocks.""" + blocks: list[Any] = [] + for response_part in function_response.parts or []: + blob = response_part.inline_data + if blob is None or blob.data is None or not blob.mime_type: + continue + blocks.extend( + _media_blocks_for_part( + types.Part( + inline_data=types.Blob(data=blob.data, mime_type=blob.mime_type) + ) + ) + ) + return blocks + + def _content_to_oci_message(content: types.Content) -> list[Any]: """Convert an ADK Content object to an OCI GenAI message. @@ -191,6 +210,12 @@ def _content_to_oci_message(content: types.Content) -> list[Any]: part.function_response.id or "", json.dumps(result) if isinstance(result, dict) else str(result), )) + # A tool can attach media alongside the serializable part of its result. + # A tool message carries text only, so the media has to follow the tool + # results as its own message. + media_blocks.extend( + _function_response_media_blocks(part.function_response) + ) elif part.inline_data or part.file_data: media_blocks.extend(_media_blocks_for_part(part)) diff --git a/src/google/adk/models/anthropic_llm.py b/src/google/adk/models/anthropic_llm.py index 6db4fdee..a5459434 100644 --- a/src/google/adk/models/anthropic_llm.py +++ b/src/google/adk/models/anthropic_llm.py @@ -78,6 +78,13 @@ _MessageBlockParam: TypeAlias = Union[ anthropic_types.ToolResultBlockParam, ] +# The subset of block types Claude accepts inside a tool result. +_ToolResultContentBlockParam: TypeAlias = Union[ + anthropic_types.TextBlockParam, + anthropic_types.ImageBlockParam, + anthropic_types.DocumentBlockParam, +] + # Attributes an Anthropic client exposes once it has resolved a credential, # whichever source it came from: a static API key, a static bearer token, or a # credential provider discovered from the environment or from the on-disk @@ -330,6 +337,55 @@ def _normalize_image_media_type(mime_type: str) -> _ImageMediaType: return cast(_ImageMediaType, normalized) +def _function_response_media_blocks( + function_response: types.FunctionResponse, +) -> list[_ToolResultContentBlockParam]: + """Converts media a tool attached to its response into tool result blocks. + + Media Claude cannot carry in a tool result is dropped with a warning rather + than raised on, because the tool that produced it is often third-party code + the caller cannot change, and losing one image is better than losing the + conversation. + """ + blocks: list[_ToolResultContentBlockParam] = [] + for response_part in function_response.parts or []: + blob = response_part.inline_data + if blob is None or blob.data is None or not blob.mime_type: + continue + media_type = blob.mime_type.split(";", 1)[0].strip().lower() + data = base64.b64encode(blob.data).decode() + if media_type in _ANTHROPIC_IMAGE_MEDIA_TYPES: + blocks.append( + anthropic_types.ImageBlockParam( + type="image", + source=anthropic_types.Base64ImageSourceParam( + type="base64", + # Narrowed by the membership test above. + media_type=cast(_ImageMediaType, media_type), + data=data, + ), + ) + ) + elif media_type == "application/pdf": + blocks.append( + anthropic_types.DocumentBlockParam( + type="document", + source=anthropic_types.Base64PDFSourceParam( + type="base64", + media_type="application/pdf", + data=data, + ), + ) + ) + else: + logger.warning( + "Dropping tool result media of type %s, which Claude cannot receive" + " in a tool result.", + media_type, + ) + return blocks + + class _ToolUseIdSanitizer: """Maps invalid tool_use IDs to deterministic fallbacks. @@ -426,10 +482,25 @@ def _part_to_message_block( # dropped. content = json.dumps(response_data) + # A tool can attach media alongside the serializable part of its result. + # It travels in a dedicated field, so it has to be mapped over explicitly + # or the model never sees it. + media_blocks = _function_response_media_blocks(function_response) + tool_result_content: Union[str, list[_ToolResultContentBlockParam]] + if media_blocks: + leading_text: list[_ToolResultContentBlockParam] = ( + [anthropic_types.TextBlockParam(type="text", text=content)] + if content + else [] + ) + tool_result_content = leading_text + media_blocks + else: + tool_result_content = content + return anthropic_types.ToolResultBlockParam( tool_use_id=sanitizer.sanitize(function_response.id), type="tool_result", - content=content, + content=tool_result_content, is_error=False, ) elif _is_image_part(part): diff --git a/src/google/adk/models/apigee_llm.py b/src/google/adk/models/apigee_llm.py index 2ca185b2..169f4c16 100644 --- a/src/google/adk/models/apigee_llm.py +++ b/src/google/adk/models/apigee_llm.py @@ -382,6 +382,23 @@ def _parse_logprobs( ) +def _function_response_media_content_parts( + function_response: types.FunctionResponse, +) -> list[dict[str, Any]]: + """Converts media a tool attached to its response into content parts.""" + media_content_parts: list[dict[str, Any]] = [] + for response_part in function_response.parts or []: + blob = response_part.inline_data + if blob is None or blob.data is None or not blob.mime_type: + continue + data = base64.b64encode(blob.data).decode('utf-8') + media_content_parts.append({ + 'type': 'image_url', + 'image_url': {'url': f'data:{blob.mime_type};base64,{data}'}, + }) + return media_content_parts + + def _validate_model_string(model: str) -> bool: """Validates the model string for Apigee LLM. @@ -736,7 +753,11 @@ class CompletionsHTTPClient: content_parts: list[dict[str, Any]] = [] refusals: list[str] = [] - function_responses = [] + function_responses: list[dict[str, Any]] = [] + # A tool can attach media alongside the serializable part of its result. + # A tool-role message carries text only, so the media has to follow the + # tool results as its own message. + response_media_parts: list[dict[str, Any]] = [] for part in content.parts or []: self._process_content_part( @@ -748,7 +769,14 @@ class CompletionsHTTPClient: 'tool_call_id': part.function_response.id, 'content': json.dumps(part.function_response.response), }) + response_media_parts.extend( + _function_response_media_content_parts(part.function_response) + ) if function_responses: + if response_media_parts: + function_responses.append( + {'role': 'user', 'content': response_media_parts} + ) return function_responses message: dict[str, Any] = {'role': role} diff --git a/src/google/adk/models/lite_llm.py b/src/google/adk/models/lite_llm.py index 2a33ba70..be81e1df 100644 --- a/src/google/adk/models/lite_llm.py +++ b/src/google/adk/models/lite_llm.py @@ -1121,6 +1121,23 @@ def _extract_thought_signature_from_tool_call( return None +def _function_response_media_parts( + function_response: types.FunctionResponse, +) -> list[types.Part]: + """Converts media a tool attached to its response into content parts.""" + media_parts: list[types.Part] = [] + for response_part in function_response.parts or []: + blob = response_part.inline_data + if blob is None or blob.data is None or not blob.mime_type: + continue + media_parts.append( + types.Part( + inline_data=types.Blob(data=blob.data, mime_type=blob.mime_type) + ) + ) + return media_parts + + async def _content_to_message_param( content: types.Content, *, @@ -1168,6 +1185,10 @@ async def _content_to_message_param( content=response_content, ) ) + # A tool can attach media alongside the serializable part of its + # result. A tool-role message carries text only, so the media has to + # follow the tool result as its own message. + non_tool_parts.extend(_function_response_media_parts(function_response)) else: non_tool_parts.append(part) diff --git a/tests/unittests/integrations/oci/test_oci_genai_llm.py b/tests/unittests/integrations/oci/test_oci_genai_llm.py index a6aeae5c..f3ca88a5 100644 --- a/tests/unittests/integrations/oci/test_oci_genai_llm.py +++ b/tests/unittests/integrations/oci/test_oci_genai_llm.py @@ -242,6 +242,31 @@ def test_content_to_oci_message_function_response(): assert msg.content[0].text +def test_content_to_oci_message_function_response_with_media(): + """Media a tool attached to its response follows as its own message.""" + import oci.generative_ai_inference.models as oci_models + + part = Part.from_function_response( + name="draw_chart", + response={"title": "Revenue"}, + parts=[ + types.FunctionResponsePart.from_bytes( + data=b"chart", mime_type="image/png" + ) + ], + ) + part.function_response.id = "call_xyz" + content = Content(role="user", parts=[part]) + + msgs = _content_to_oci_message(content) + + assert len(msgs) == 2 + assert isinstance(msgs[0], oci_models.ToolMessage) + assert msgs[0].tool_call_id == "call_xyz" + assert isinstance(msgs[1], oci_models.UserMessage) + assert msgs[1].content[0].image_url.url.startswith("data:image/png;base64,") + + def test_content_to_oci_message_multiple_function_responses(): import oci.generative_ai_inference.models as oci_models diff --git a/tests/unittests/models/test_anthropic_llm.py b/tests/unittests/models/test_anthropic_llm.py index 813e0357..b03abdcd 100644 --- a/tests/unittests/models/test_anthropic_llm.py +++ b/tests/unittests/models/test_anthropic_llm.py @@ -838,6 +838,89 @@ def test_part_to_message_block_with_multiple_content_items(): assert result["content"] == "First part\nSecond part" +def test_part_to_message_block_tool_result_with_image(): + """Media a tool attached to its response reaches Claude as an image block.""" + image_data = b"chart-bytes" + part = types.Part.from_function_response( + name="draw_chart", + response={"title": "Revenue"}, + parts=[ + types.FunctionResponsePart.from_bytes( + data=image_data, mime_type="image/png" + ) + ], + ) + part.function_response.id = "call_1" + + result = part_to_message_block(part) + + assert result["type"] == "tool_result" + text_block, image_block = result["content"] + assert text_block["type"] == "text" + assert "Revenue" in text_block["text"] + assert image_block["type"] == "image" + assert image_block["source"]["media_type"] == "image/png" + assert image_block["source"]["data"] == base64.b64encode(image_data).decode() + + +def test_part_to_message_block_tool_result_with_only_image(): + """A tool that returns nothing but media produces no empty text block.""" + part = types.Part.from_function_response( + name="screenshot", + response={}, + parts=[ + types.FunctionResponsePart.from_bytes( + data=b"png-bytes", mime_type="image/png" + ) + ], + ) + part.function_response.id = "call_2" + + result = part_to_message_block(part) + + assert [block["type"] for block in result["content"]] == ["image"] + + +def test_part_to_message_block_tool_result_with_pdf(): + """A PDF a tool attaches reaches Claude as a document block.""" + pdf_data = b"%PDF-1.4 report" + part = types.Part.from_function_response( + name="build_report", + response={}, + parts=[ + types.FunctionResponsePart.from_bytes( + data=pdf_data, mime_type="application/pdf" + ) + ], + ) + part.function_response.id = "call_3" + + result = part_to_message_block(part) + + document_block = result["content"][0] + assert document_block["type"] == "document" + assert document_block["source"]["media_type"] == "application/pdf" + assert document_block["source"]["data"] == base64.b64encode(pdf_data).decode() + + +def test_part_to_message_block_tool_result_drops_unsupported_media(): + """Media Claude cannot accept is dropped rather than failing the turn.""" + part = types.Part.from_function_response( + name="record_audio", + response={"duration": 3}, + parts=[ + types.FunctionResponsePart.from_bytes( + data=b"wav-bytes", mime_type="audio/wav" + ) + ], + ) + part.function_response.id = "call_4" + + result = part_to_message_block(part) + + assert result["content"] == json.dumps({"duration": 3}) + + def test_part_to_message_block_with_pdf_document(): """Test that part_to_message_block handles PDF document parts.""" pdf_data = b"%PDF-1.4 fake pdf content" diff --git a/tests/unittests/models/test_apigee_llm.py b/tests/unittests/models/test_apigee_llm.py index 274eae3c..aa906f52 100644 --- a/tests/unittests/models/test_apigee_llm.py +++ b/tests/unittests/models/test_apigee_llm.py @@ -1102,3 +1102,54 @@ def test_content_conversion_rejects_incomplete_inline_data( client = CompletionsHTTPClient(base_url='http://test') with pytest.raises(ValueError, match='Inline data must include'): client._content_to_messages(content) + + +def test_content_conversion_carries_function_response_media() -> None: + """Media a tool attached to its response follows as its own message.""" + part = types.Part.from_function_response( + name='draw_chart', + response={'title': 'Revenue'}, + parts=[ + types.FunctionResponsePart.from_bytes( + data=b'chart', mime_type='image/png' + ) + ], + ) + part.function_response.id = 'call_1' + content = types.Content(role='user', parts=[part]) + + client = CompletionsHTTPClient(base_url='http://test') + messages = client._content_to_messages(content) + + assert messages == [ + { + 'role': 'tool', + 'tool_call_id': 'call_1', + 'content': '{"title": "Revenue"}', + }, + { + 'role': 'user', + 'content': [{ + 'type': 'image_url', + 'image_url': {'url': 'data:image/png;base64,Y2hhcnQ='}, + }], + }, + ] + + +def test_content_conversion_without_function_response_media() -> None: + """A response carrying no media still converts to a lone tool message.""" + part = types.Part.from_function_response( + name='lookup', response={'status': 'ok'} + ) + part.function_response.id = 'call_1' + content = types.Content(role='user', parts=[part]) + + client = CompletionsHTTPClient(base_url='http://test') + messages = client._content_to_messages(content) + + assert messages == [{ + 'role': 'tool', + 'tool_call_id': 'call_1', + 'content': '{"status": "ok"}', + }] diff --git a/tests/unittests/models/test_litellm.py b/tests/unittests/models/test_litellm.py index bd1729c6..a63032d1 100644 --- a/tests/unittests/models/test_litellm.py +++ b/tests/unittests/models/test_litellm.py @@ -2243,6 +2243,63 @@ async def test_content_to_message_param_function_response_with_extra_parts(): ] +@pytest.mark.asyncio +async def test_content_to_message_param_function_response_with_media(): + """Media a tool attached to its response follows as its own message.""" + image_bytes = b"test_image_data" + tool_part = types.Part.from_function_response( + name="draw_chart", + response={"title": "Revenue"}, + parts=[ + types.FunctionResponsePart.from_bytes( + data=image_bytes, mime_type="image/png" + ) + ], + ) + tool_part.function_response.id = "tool_call_1" + + content = types.Content(role="user", parts=[tool_part]) + + messages = await _content_to_message_param(content) + + assert messages == [ + { + "role": "tool", + "tool_call_id": "tool_call_1", + "content": '{"title": "Revenue"}', + }, + { + "role": "user", + "content": [{ + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,dGVzdF9pbWFnZV9kYXRh" + }, + }], + }, + ] + + +@pytest.mark.asyncio +async def test_content_to_message_param_function_response_without_media(): + """A response carrying no media still converts to a lone tool message.""" + tool_part = types.Part.from_function_response( + name="lookup", + response={"status": "success"}, + ) + tool_part.function_response.id = "tool_call_1" + + content = types.Content(role="user", parts=[tool_part]) + + message = await _content_to_message_param(content) + + assert message == { + "role": "tool", + "tool_call_id": "tool_call_1", + "content": '{"status": "success"}', + } + + @pytest.mark.asyncio async def test_content_to_message_param_function_response_preserves_string(): """Tests that string responses are used directly without double-serialization. @@ -2267,6 +2324,9 @@ async def test_content_to_message_param_function_response_preserves_string(): mock_function_response = Mock(spec=types.FunctionResponse) mock_function_response.response = response_payload mock_function_response.id = "tool_call_1" + # Mock(spec=...) exposes none of a Pydantic model's fields, so every field + # the converter reads has to be set explicitly. + mock_function_response.parts = None part.function_response = mock_function_response content = types.Content(