fix: send media attached to a tool response to non-Gemini models

Co-authored-by: George Weale <gweale@google.com>
PiperOrigin-RevId: 964308319

src/google/adk/integrations/oci/_oci_genai_llm.py

@@ -156,6 +156,25 @@ def _media_blocks_for_part(part: types.Part) -> list[Any]:
   ]
 
 
+def _function_response_media_blocks(
+    function_response: types.FunctionResponse,
+) -> list[Any]:
+  """Converts media a tool attached to its response into OCI content blocks."""
+  blocks: list[Any] = []
+  for response_part in function_response.parts or []:
+    blob = response_part.inline_data
+    if blob is None or blob.data is None or not blob.mime_type:
+      continue
+    blocks.extend(
+        _media_blocks_for_part(
+            types.Part(
+                inline_data=types.Blob(data=blob.data, mime_type=blob.mime_type)
+            )
+        )
+    )
+  return blocks
+
+
 def _content_to_oci_message(content: types.Content) -> list[Any]:
   """Convert an ADK Content object to an OCI GenAI message.
 
@@ -191,6 +210,12 @@ def _content_to_oci_message(content: types.Content) -> list[Any]:
           part.function_response.id or "",
           json.dumps(result) if isinstance(result, dict) else str(result),
       ))
+      # A tool can attach media alongside the serializable part of its result.
+      # A tool message carries text only, so the media has to follow the tool
+      # results as its own message.
+      media_blocks.extend(
+          _function_response_media_blocks(part.function_response)
+      )
     elif part.inline_data or part.file_data:
       media_blocks.extend(_media_blocks_for_part(part))
 

src/google/adk/models/anthropic_llm.py

@@ -78,6 +78,13 @@ _MessageBlockParam: TypeAlias = Union[
     anthropic_types.ToolResultBlockParam,
 ]
 
+# The subset of block types Claude accepts inside a tool result.
+_ToolResultContentBlockParam: TypeAlias = Union[
+    anthropic_types.TextBlockParam,
+    anthropic_types.ImageBlockParam,
+    anthropic_types.DocumentBlockParam,
+]
+
 # Attributes an Anthropic client exposes once it has resolved a credential,
 # whichever source it came from: a static API key, a static bearer token, or a
 # credential provider discovered from the environment or from the on-disk
@@ -330,6 +337,55 @@ def _normalize_image_media_type(mime_type: str) -> _ImageMediaType:
   return cast(_ImageMediaType, normalized)
 
 
+def _function_response_media_blocks(
+    function_response: types.FunctionResponse,
+) -> list[_ToolResultContentBlockParam]:
+  """Converts media a tool attached to its response into tool result blocks.
+
+  Media Claude cannot carry in a tool result is dropped with a warning rather
+  than raised on, because the tool that produced it is often third-party code
+  the caller cannot change, and losing one image is better than losing the
+  conversation.
+  """
+  blocks: list[_ToolResultContentBlockParam] = []
+  for response_part in function_response.parts or []:
+    blob = response_part.inline_data
+    if blob is None or blob.data is None or not blob.mime_type:
+      continue
+    media_type = blob.mime_type.split(";", 1)[0].strip().lower()
+    data = base64.b64encode(blob.data).decode()
+    if media_type in _ANTHROPIC_IMAGE_MEDIA_TYPES:
+      blocks.append(
+          anthropic_types.ImageBlockParam(
+              type="image",
+              source=anthropic_types.Base64ImageSourceParam(
+                  type="base64",
+                  # Narrowed by the membership test above.
+                  media_type=cast(_ImageMediaType, media_type),
+                  data=data,
+              ),
+          )
+      )
+    elif media_type == "application/pdf":
+      blocks.append(
+          anthropic_types.DocumentBlockParam(
+              type="document",
+              source=anthropic_types.Base64PDFSourceParam(
+                  type="base64",
+                  media_type="application/pdf",
+                  data=data,
+              ),
+          )
+      )
+    else:
+      logger.warning(
+          "Dropping tool result media of type %s, which Claude cannot receive"
+          " in a tool result.",
+          media_type,
+      )
+  return blocks
+
+
 class _ToolUseIdSanitizer:
   """Maps invalid tool_use IDs to deterministic fallbacks.
 
@@ -426,10 +482,25 @@ def _part_to_message_block(
       # dropped.
       content = json.dumps(response_data)
 
+    # A tool can attach media alongside the serializable part of its result.
+    # It travels in a dedicated field, so it has to be mapped over explicitly
+    # or the model never sees it.
+    media_blocks = _function_response_media_blocks(function_response)
+    tool_result_content: Union[str, list[_ToolResultContentBlockParam]]
+    if media_blocks:
+      leading_text: list[_ToolResultContentBlockParam] = (
+          [anthropic_types.TextBlockParam(type="text", text=content)]
+          if content
+          else []
+      )
+      tool_result_content = leading_text + media_blocks
+    else:
+      tool_result_content = content
+
     return anthropic_types.ToolResultBlockParam(
         tool_use_id=sanitizer.sanitize(function_response.id),
         type="tool_result",
-        content=content,
+        content=tool_result_content,
         is_error=False,
     )
   elif _is_image_part(part):

src/google/adk/models/apigee_llm.py

@@ -382,6 +382,23 @@ def _parse_logprobs(
   )
 
 
+def _function_response_media_content_parts(
+    function_response: types.FunctionResponse,
+) -> list[dict[str, Any]]:
+  """Converts media a tool attached to its response into content parts."""
+  media_content_parts: list[dict[str, Any]] = []
+  for response_part in function_response.parts or []:
+    blob = response_part.inline_data
+    if blob is None or blob.data is None or not blob.mime_type:
+      continue
+    data = base64.b64encode(blob.data).decode('utf-8')
+    media_content_parts.append({
+        'type': 'image_url',
+        'image_url': {'url': f'data:{blob.mime_type};base64,{data}'},
+    })
+  return media_content_parts
+
+
 def _validate_model_string(model: str) -> bool:
   """Validates the model string for Apigee LLM.
 
@@ -736,7 +753,11 @@ class CompletionsHTTPClient:
     content_parts: list[dict[str, Any]] = []
     refusals: list[str] = []
 
-    function_responses = []
+    function_responses: list[dict[str, Any]] = []
+    # A tool can attach media alongside the serializable part of its result.
+    # A tool-role message carries text only, so the media has to follow the
+    # tool results as its own message.
+    response_media_parts: list[dict[str, Any]] = []
 
     for part in content.parts or []:
       self._process_content_part(
@@ -748,7 +769,14 @@ class CompletionsHTTPClient:
             'tool_call_id': part.function_response.id,
             'content': json.dumps(part.function_response.response),
         })
+        response_media_parts.extend(
+            _function_response_media_content_parts(part.function_response)
+        )
     if function_responses:
+      if response_media_parts:
+        function_responses.append(
+            {'role': 'user', 'content': response_media_parts}
+        )
       return function_responses
 
     message: dict[str, Any] = {'role': role}

src/google/adk/models/lite_llm.py

@@ -1121,6 +1121,23 @@ def _extract_thought_signature_from_tool_call(
   return None
 
 
+def _function_response_media_parts(
+    function_response: types.FunctionResponse,
+) -> list[types.Part]:
+  """Converts media a tool attached to its response into content parts."""
+  media_parts: list[types.Part] = []
+  for response_part in function_response.parts or []:
+    blob = response_part.inline_data
+    if blob is None or blob.data is None or not blob.mime_type:
+      continue
+    media_parts.append(
+        types.Part(
+            inline_data=types.Blob(data=blob.data, mime_type=blob.mime_type)
+        )
+    )
+  return media_parts
+
+
 async def _content_to_message_param(
     content: types.Content,
     *,
@@ -1168,6 +1185,10 @@ async def _content_to_message_param(
               content=response_content,
           )
       )
+      # A tool can attach media alongside the serializable part of its
+      # result. A tool-role message carries text only, so the media has to
+      # follow the tool result as its own message.
+      non_tool_parts.extend(_function_response_media_parts(function_response))
     else:
       non_tool_parts.append(part)
 

tests/unittests/integrations/oci/test_oci_genai_llm.py

@@ -242,6 +242,31 @@ def test_content_to_oci_message_function_response():
   assert msg.content[0].text
 
 
+def test_content_to_oci_message_function_response_with_media():
+  """Media a tool attached to its response follows as its own message."""
+  import oci.generative_ai_inference.models as oci_models
+
+  part = Part.from_function_response(
+      name="draw_chart",
+      response={"title": "Revenue"},
+      parts=[
+          types.FunctionResponsePart.from_bytes(
+              data=b"chart", mime_type="image/png"
+          )
+      ],
+  )
+  part.function_response.id = "call_xyz"
+  content = Content(role="user", parts=[part])
+
+  msgs = _content_to_oci_message(content)
+
+  assert len(msgs) == 2
+  assert isinstance(msgs[0], oci_models.ToolMessage)
+  assert msgs[0].tool_call_id == "call_xyz"
+  assert isinstance(msgs[1], oci_models.UserMessage)
+  assert msgs[1].content[0].image_url.url.startswith("data:image/png;base64,")
+
+
 def test_content_to_oci_message_multiple_function_responses():
   import oci.generative_ai_inference.models as oci_models
 

tests/unittests/models/test_anthropic_llm.py

@@ -838,6 +838,89 @@ def test_part_to_message_block_with_multiple_content_items():
   assert result["content"] == "First part\nSecond part"
 
 
+def test_part_to_message_block_tool_result_with_image():
+  """Media a tool attached to its response reaches Claude as an image block."""
+  image_data = b"chart-bytes"
+  part = types.Part.from_function_response(
+      name="draw_chart",
+      response={"title": "Revenue"},
+      parts=[
+          types.FunctionResponsePart.from_bytes(
+              data=image_data, mime_type="image/png"
+          )
+      ],
+  )
+  part.function_response.id = "call_1"
+
+  result = part_to_message_block(part)
+
+  assert result["type"] == "tool_result"
+  text_block, image_block = result["content"]
+  assert text_block["type"] == "text"
+  assert "Revenue" in text_block["text"]
+  assert image_block["type"] == "image"
+  assert image_block["source"]["media_type"] == "image/png"
+  assert image_block["source"]["data"] == base64.b64encode(image_data).decode()
+
+
+def test_part_to_message_block_tool_result_with_only_image():
+  """A tool that returns nothing but media produces no empty text block."""
+  part = types.Part.from_function_response(
+      name="screenshot",
+      response={},
+      parts=[
+          types.FunctionResponsePart.from_bytes(
+              data=b"png-bytes", mime_type="image/png"
+          )
+      ],
+  )
+  part.function_response.id = "call_2"
+
+  result = part_to_message_block(part)
+
+  assert [block["type"] for block in result["content"]] == ["image"]
+
+
+def test_part_to_message_block_tool_result_with_pdf():
+  """A PDF a tool attaches reaches Claude as a document block."""
+  pdf_data = b"%PDF-1.4 report"
+  part = types.Part.from_function_response(
+      name="build_report",
+      response={},
+      parts=[
+          types.FunctionResponsePart.from_bytes(
+              data=pdf_data, mime_type="application/pdf"
+          )
+      ],
+  )
+  part.function_response.id = "call_3"
+
+  result = part_to_message_block(part)
+
+  document_block = result["content"][0]
+  assert document_block["type"] == "document"
+  assert document_block["source"]["media_type"] == "application/pdf"
+  assert document_block["source"]["data"] == base64.b64encode(pdf_data).decode()
+
+
+def test_part_to_message_block_tool_result_drops_unsupported_media():
+  """Media Claude cannot accept is dropped rather than failing the turn."""
+  part = types.Part.from_function_response(
+      name="record_audio",
+      response={"duration": 3},
+      parts=[
+          types.FunctionResponsePart.from_bytes(
+              data=b"wav-bytes", mime_type="audio/wav"
+          )
+      ],
+  )
+  part.function_response.id = "call_4"
+
+  result = part_to_message_block(part)
+
+  assert result["content"] == json.dumps({"duration": 3})
+
+
 def test_part_to_message_block_with_pdf_document():
   """Test that part_to_message_block handles PDF document parts."""
   pdf_data = b"%PDF-1.4 fake pdf content"

tests/unittests/models/test_apigee_llm.py

@@ -1102,3 +1102,54 @@ def test_content_conversion_rejects_incomplete_inline_data(
   client = CompletionsHTTPClient(base_url='http://test')
   with pytest.raises(ValueError, match='Inline data must include'):
     client._content_to_messages(content)
+
+
+def test_content_conversion_carries_function_response_media() -> None:
+  """Media a tool attached to its response follows as its own message."""
+  part = types.Part.from_function_response(
+      name='draw_chart',
+      response={'title': 'Revenue'},
+      parts=[
+          types.FunctionResponsePart.from_bytes(
+              data=b'chart', mime_type='image/png'
+          )
+      ],
+  )
+  part.function_response.id = 'call_1'
+  content = types.Content(role='user', parts=[part])
+
+  client = CompletionsHTTPClient(base_url='http://test')
+  messages = client._content_to_messages(content)
+
+  assert messages == [
+      {
+          'role': 'tool',
+          'tool_call_id': 'call_1',
+          'content': '{"title": "Revenue"}',
+      },
+      {
+          'role': 'user',
+          'content': [{
+              'type': 'image_url',
+              'image_url': {'url': 'data:image/png;base64,Y2hhcnQ='},
+          }],
+      },
+  ]
+
+
+def test_content_conversion_without_function_response_media() -> None:
+  """A response carrying no media still converts to a lone tool message."""
+  part = types.Part.from_function_response(
+      name='lookup', response={'status': 'ok'}
+  )
+  part.function_response.id = 'call_1'
+  content = types.Content(role='user', parts=[part])
+
+  client = CompletionsHTTPClient(base_url='http://test')
+  messages = client._content_to_messages(content)
+
+  assert messages == [{
+      'role': 'tool',
+      'tool_call_id': 'call_1',
+      'content': '{"status": "ok"}',
+  }]

tests/unittests/models/test_litellm.py

@@ -2243,6 +2243,63 @@ async def test_content_to_message_param_function_response_with_extra_parts():
   ]
 
 
+@pytest.mark.asyncio
+async def test_content_to_message_param_function_response_with_media():
+  """Media a tool attached to its response follows as its own message."""
+  image_bytes = b"test_image_data"
+  tool_part = types.Part.from_function_response(
+      name="draw_chart",
+      response={"title": "Revenue"},
+      parts=[
+          types.FunctionResponsePart.from_bytes(
+              data=image_bytes, mime_type="image/png"
+          )
+      ],
+  )
+  tool_part.function_response.id = "tool_call_1"
+
+  content = types.Content(role="user", parts=[tool_part])
+
+  messages = await _content_to_message_param(content)
+
+  assert messages == [
+      {
+          "role": "tool",
+          "tool_call_id": "tool_call_1",
+          "content": '{"title": "Revenue"}',
+      },
+      {
+          "role": "user",
+          "content": [{
+              "type": "image_url",
+              "image_url": {
+                  "url": "data:image/png;base64,dGVzdF9pbWFnZV9kYXRh"
+              },
+          }],
+      },
+  ]
+
+
+@pytest.mark.asyncio
+async def test_content_to_message_param_function_response_without_media():
+  """A response carrying no media still converts to a lone tool message."""
+  tool_part = types.Part.from_function_response(
+      name="lookup",
+      response={"status": "success"},
+  )
+  tool_part.function_response.id = "tool_call_1"
+
+  content = types.Content(role="user", parts=[tool_part])
+
+  message = await _content_to_message_param(content)
+
+  assert message == {
+      "role": "tool",
+      "tool_call_id": "tool_call_1",
+      "content": '{"status": "success"}',
+  }
+
+
 @pytest.mark.asyncio
 async def test_content_to_message_param_function_response_preserves_string():
   """Tests that string responses are used directly without double-serialization.
@@ -2267,6 +2324,9 @@ async def test_content_to_message_param_function_response_preserves_string():
   mock_function_response = Mock(spec=types.FunctionResponse)
   mock_function_response.response = response_payload
   mock_function_response.id = "tool_call_1"
+  # Mock(spec=...) exposes none of a Pydantic model's fields, so every field
+  # the converter reads has to be set explicitly.
+  mock_function_response.parts = None
   part.function_response = mock_function_response
 
   content = types.Content(