fix(telemetry): Port inline binary data summarization on spans to v1

Serializing a content part in JSON mode base64-encodes its inline_data, so the
model response span and the trace_send_data span carried the bytes themselves.
A live session's audio chunks and any image or document a user uploaded were
copied wholesale onto an exported span attribute, which sent the payload to the
trace backend and grew the span with it.

Now a new private helper replaces every part that has inline_data with a text
part reading "<inline_data: {mime}, {n} bytes>", and both call sites route
their content through it. The request side already dropped inline parts and is
unchanged.

Behaviour change: anyone who was reading audio or image bytes back out of a
span now gets that description string instead. The model still receives the
real bytes; only the span is summarized.
This commit is contained in:
George Weale
2026-08-17 23:04:51 +00:00
parent 8c1ae1459d
commit dadeea18fb
2 changed files with 155 additions and 2 deletions
+38 -2
View File
@@ -360,7 +360,12 @@ def trace_call_llm(
if _should_add_request_response_to_spans():
try:
llm_response_json = llm_response.model_dump_json(exclude_none=True)
response_for_trace = llm_response
if llm_response.content is not None:
response_for_trace = llm_response.model_copy(
update={'content': _summarize_inline_data(llm_response.content)}
)
llm_response_json = response_for_trace.model_dump_json(exclude_none=True)
except Exception: # pylint: disable=broad-exception-caught
llm_response_json = '<not serializable>'
@@ -409,6 +414,37 @@ def trace_call_llm(
)
def _summarize_inline_data(content: types.Content) -> types.Content:
"""Returns ``content`` with inline binary parts reduced to a description.
Serializing a part in JSON mode base64-encodes its ``inline_data``, so a
live session's audio chunks would otherwise be copied wholesale onto a span
attribute. Only the mime type and byte count are kept.
Args:
content: The content to summarize.
Returns:
A copy of ``content`` whose inline binary parts carry a text description
instead of the bytes.
"""
parts: list[types.Part] = []
for part in content.parts or []:
blob = part.inline_data
if blob is None:
parts.append(part)
continue
parts.append(
types.Part(
text=(
f"<inline_data: {blob.mime_type or 'unknown'},"
f" {len(blob.data or b'')} bytes>"
)
)
)
return types.Content(role=content.role, parts=parts)
def trace_send_data(
invocation_context: InvocationContext,
event_id: str,
@@ -435,7 +471,7 @@ def trace_send_data(
span.set_attribute(
'gcp.vertex.agent.data',
_safe_json_serialize([
types.Content(role=content.role, parts=content.parts).model_dump(
_summarize_inline_data(content).model_dump(
exclude_none=True, mode='json'
)
for content in data
+117
View File
@@ -802,6 +802,123 @@ async def test_trace_send_data_disabling_request_response_content(
)
@pytest.mark.asyncio
async def test_trace_call_llm_summarizes_response_inline_data(
monkeypatch, mock_span_fixture
):
"""Inline binary data in the response is described, not copied to the span."""
monkeypatch.setattr(
'opentelemetry.trace.get_current_span', lambda: mock_span_fixture
)
agent = LlmAgent(name='test_agent')
invocation_context = await _create_invocation_context(agent)
llm_request = LlmRequest(
model='gemini-pro', config=types.GenerateContentConfig()
)
llm_response = LlmResponse(
content=types.Content(
role='model',
parts=[
types.Part(text='hi'),
types.Part.from_bytes(data=b'test_data', mime_type='audio/pcm'),
],
)
)
trace_call_llm(invocation_context, 'test_event_id', llm_request, llm_response)
llm_response_json = next(
call_obj.args[1]
for call_obj in mock_span_fixture.set_attribute.call_args_list
if call_obj.args[0] == 'gcp.vertex.agent.llm_response'
)
# b'test_data' base64-encodes to 'dGVzdF9kYXRh'.
assert 'dGVzdF9kYXRh' not in llm_response_json
assert 'hi' in llm_response_json
assert '<inline_data: audio/pcm, 9 bytes>' in llm_response_json
@pytest.mark.asyncio
async def test_trace_send_data_summarizes_inline_data(
monkeypatch, mock_span_fixture
):
"""Inline binary data is described on the span, never copied onto it."""
monkeypatch.setenv(ADK_CAPTURE_MESSAGE_CONTENT_IN_SPANS, 'true')
monkeypatch.setattr(
'opentelemetry.trace.get_current_span', lambda: mock_span_fixture
)
agent = LlmAgent(name='test_agent')
invocation_context = await _create_invocation_context(agent)
trace_send_data(
invocation_context=invocation_context,
event_id='test_event_id',
data=[
types.Content(
role='user',
parts=[
types.Part(text='hi'),
types.Part.from_bytes(
data=b'test_data', mime_type='audio/pcm'
),
],
)
],
)
data_json = next(
call_obj.args[1]
for call_obj in mock_span_fixture.set_attribute.call_args_list
if call_obj.args[0] == 'gcp.vertex.agent.data'
)
# b'test_data' base64-encodes to 'dGVzdF9kYXRh'.
assert 'dGVzdF9kYXRh' not in data_json
assert 'hi' in data_json
assert '<inline_data: audio/pcm, 9 bytes>' in data_json
@pytest.mark.asyncio
async def test_trace_send_data_summarizes_blob_without_mime_type(
monkeypatch, mock_span_fixture
):
"""A blob is described even when its mime type and bytes are unset.
The parts-less content in the same call pins that summarizing tolerates
``Content.parts`` being unset.
"""
monkeypatch.setenv(ADK_CAPTURE_MESSAGE_CONTENT_IN_SPANS, 'true')
monkeypatch.setattr(
'opentelemetry.trace.get_current_span', lambda: mock_span_fixture
)
agent = LlmAgent(name='test_agent')
invocation_context = await _create_invocation_context(agent)
trace_send_data(
invocation_context=invocation_context,
event_id='test_event_id',
data=[
types.Content(role='user'),
types.Content(
role='user', parts=[types.Part(inline_data=types.Blob())]
),
],
)
data_json = next(
call_obj.args[1]
for call_obj in mock_span_fixture.set_attribute.call_args_list
if call_obj.args[0] == 'gcp.vertex.agent.data'
)
assert '<inline_data: unknown, 0 bytes>' in data_json
assert 'inlineData' not in data_json
@pytest.mark.asyncio
@mock.patch('google.adk.telemetry.tracing.otel_logger')
@mock.patch('google.adk.telemetry.tracing.tracer')