From dadeea18fb273f21d7dca4cb258d703b8028b25e Mon Sep 17 00:00:00 2001 From: George Weale Date: Mon, 17 Aug 2026 23:04:51 +0000 Subject: [PATCH] fix(telemetry): Port inline binary data summarization on spans to v1 Serializing a content part in JSON mode base64-encodes its inline_data, so the model response span and the trace_send_data span carried the bytes themselves. A live session's audio chunks and any image or document a user uploaded were copied wholesale onto an exported span attribute, which sent the payload to the trace backend and grew the span with it. Now a new private helper replaces every part that has inline_data with a text part reading "", and both call sites route their content through it. The request side already dropped inline parts and is unchanged. Behaviour change: anyone who was reading audio or image bytes back out of a span now gets that description string instead. The model still receives the real bytes; only the span is summarized. --- src/google/adk/telemetry/tracing.py | 40 +++++++- tests/unittests/telemetry/test_spans.py | 117 ++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 2 deletions(-) diff --git a/src/google/adk/telemetry/tracing.py b/src/google/adk/telemetry/tracing.py index 158bef1c..2b2c08b8 100644 --- a/src/google/adk/telemetry/tracing.py +++ b/src/google/adk/telemetry/tracing.py @@ -360,7 +360,12 @@ def trace_call_llm( if _should_add_request_response_to_spans(): try: - llm_response_json = llm_response.model_dump_json(exclude_none=True) + response_for_trace = llm_response + if llm_response.content is not None: + response_for_trace = llm_response.model_copy( + update={'content': _summarize_inline_data(llm_response.content)} + ) + llm_response_json = response_for_trace.model_dump_json(exclude_none=True) except Exception: # pylint: disable=broad-exception-caught llm_response_json = '' @@ -409,6 +414,37 @@ def trace_call_llm( ) +def _summarize_inline_data(content: types.Content) -> types.Content: + """Returns ``content`` with inline binary parts reduced to a description. + + Serializing a part in JSON mode base64-encodes its ``inline_data``, so a + live session's audio chunks would otherwise be copied wholesale onto a span + attribute. Only the mime type and byte count are kept. + + Args: + content: The content to summarize. + + Returns: + A copy of ``content`` whose inline binary parts carry a text description + instead of the bytes. + """ + parts: list[types.Part] = [] + for part in content.parts or []: + blob = part.inline_data + if blob is None: + parts.append(part) + continue + parts.append( + types.Part( + text=( + f"" + ) + ) + ) + return types.Content(role=content.role, parts=parts) + + def trace_send_data( invocation_context: InvocationContext, event_id: str, @@ -435,7 +471,7 @@ def trace_send_data( span.set_attribute( 'gcp.vertex.agent.data', _safe_json_serialize([ - types.Content(role=content.role, parts=content.parts).model_dump( + _summarize_inline_data(content).model_dump( exclude_none=True, mode='json' ) for content in data diff --git a/tests/unittests/telemetry/test_spans.py b/tests/unittests/telemetry/test_spans.py index c2fa6dfc..a5aad74f 100644 --- a/tests/unittests/telemetry/test_spans.py +++ b/tests/unittests/telemetry/test_spans.py @@ -802,6 +802,123 @@ async def test_trace_send_data_disabling_request_response_content( ) +@pytest.mark.asyncio +async def test_trace_call_llm_summarizes_response_inline_data( + monkeypatch, mock_span_fixture +): + """Inline binary data in the response is described, not copied to the span.""" + monkeypatch.setattr( + 'opentelemetry.trace.get_current_span', lambda: mock_span_fixture + ) + + agent = LlmAgent(name='test_agent') + invocation_context = await _create_invocation_context(agent) + llm_request = LlmRequest( + model='gemini-pro', config=types.GenerateContentConfig() + ) + llm_response = LlmResponse( + content=types.Content( + role='model', + parts=[ + types.Part(text='hi'), + types.Part.from_bytes(data=b'test_data', mime_type='audio/pcm'), + ], + ) + ) + + trace_call_llm(invocation_context, 'test_event_id', llm_request, llm_response) + + llm_response_json = next( + call_obj.args[1] + for call_obj in mock_span_fixture.set_attribute.call_args_list + if call_obj.args[0] == 'gcp.vertex.agent.llm_response' + ) + + # b'test_data' base64-encodes to 'dGVzdF9kYXRh'. + assert 'dGVzdF9kYXRh' not in llm_response_json + assert 'hi' in llm_response_json + assert '' in llm_response_json + + +@pytest.mark.asyncio +async def test_trace_send_data_summarizes_inline_data( + monkeypatch, mock_span_fixture +): + """Inline binary data is described on the span, never copied onto it.""" + monkeypatch.setenv(ADK_CAPTURE_MESSAGE_CONTENT_IN_SPANS, 'true') + monkeypatch.setattr( + 'opentelemetry.trace.get_current_span', lambda: mock_span_fixture + ) + + agent = LlmAgent(name='test_agent') + invocation_context = await _create_invocation_context(agent) + + trace_send_data( + invocation_context=invocation_context, + event_id='test_event_id', + data=[ + types.Content( + role='user', + parts=[ + types.Part(text='hi'), + types.Part.from_bytes( + data=b'test_data', mime_type='audio/pcm' + ), + ], + ) + ], + ) + + data_json = next( + call_obj.args[1] + for call_obj in mock_span_fixture.set_attribute.call_args_list + if call_obj.args[0] == 'gcp.vertex.agent.data' + ) + + # b'test_data' base64-encodes to 'dGVzdF9kYXRh'. + assert 'dGVzdF9kYXRh' not in data_json + assert 'hi' in data_json + assert '' in data_json + + +@pytest.mark.asyncio +async def test_trace_send_data_summarizes_blob_without_mime_type( + monkeypatch, mock_span_fixture +): + """A blob is described even when its mime type and bytes are unset. + + The parts-less content in the same call pins that summarizing tolerates + ``Content.parts`` being unset. + """ + monkeypatch.setenv(ADK_CAPTURE_MESSAGE_CONTENT_IN_SPANS, 'true') + monkeypatch.setattr( + 'opentelemetry.trace.get_current_span', lambda: mock_span_fixture + ) + + agent = LlmAgent(name='test_agent') + invocation_context = await _create_invocation_context(agent) + + trace_send_data( + invocation_context=invocation_context, + event_id='test_event_id', + data=[ + types.Content(role='user'), + types.Content( + role='user', parts=[types.Part(inline_data=types.Blob())] + ), + ], + ) + + data_json = next( + call_obj.args[1] + for call_obj in mock_span_fixture.set_attribute.call_args_list + if call_obj.args[0] == 'gcp.vertex.agent.data' + ) + + assert '' in data_json + assert 'inlineData' not in data_json + + @pytest.mark.asyncio @mock.patch('google.adk.telemetry.tracing.otel_logger') @mock.patch('google.adk.telemetry.tracing.tracer')