github-featured/openai--openai-agents-python
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

fix: #2009 #2010 Improve PCM duration calculation and handle VAD truncation (#2059)

Browse Source 
Co-authored-by: Kazuhiro Sera <seratch@openai.com>
This commit is contained in:
Ruskaruma
2026-01-08 12:27:33 +05:30
committed by GitHub
parent b4c80b8870
commit b5d9152a27
5 changed files with 67 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agents/realtime/_util.py
+14 -3
View File
@@ -2,8 +2,19 @@ from __future__ import annotations
from .config import RealtimeAudioFormat
PCM16_SAMPLE_RATE_HZ = 24_000
PCM16_SAMPLE_WIDTH_BYTES = 2
G711_SAMPLE_RATE_HZ = 8_000
def calculate_audio_length_ms(format: RealtimeAudioFormat | None, audio_bytes: bytes) -> float:
if format and isinstance(format, str) and format.startswith("g711"):
return (len(audio_bytes) / 8000) * 1000
return (len(audio_bytes) / 24 / 2) * 1000
if not audio_bytes:
return 0.0
normalized_format = format.lower() if isinstance(format, str) else None
if normalized_format and normalized_format.startswith("g711"):
return (len(audio_bytes) / G711_SAMPLE_RATE_HZ) * 1000
samples = len(audio_bytes) / PCM16_SAMPLE_WIDTH_BYTES
return (samples / PCM16_SAMPLE_RATE_HZ) * 1000
src/agents/realtime/openai_realtime.py
+20 -3
View File
@@ -691,10 +691,30 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
last_audio = self._audio_state_tracker.get_last_audio_item()
if last_audio is not None:
item_id, content_index = last_audio
playback_state = self._get_playback_state()
playback_item_id = playback_state.get("current_item_id")
playback_content_index = playback_state.get("current_item_content_index") or 0
playback_elapsed_ms = playback_state.get("elapsed_ms")
await self._emit_event(
RealtimeModelAudioInterruptedEvent(item_id=item_id, content_index=content_index)
)
elapsed_override = getattr(parsed, "audio_end_ms", None)
if elapsed_override is None or elapsed_override <= 0:
effective_elapsed_ms = playback_elapsed_ms
else:
effective_elapsed_ms = float(elapsed_override)
if playback_item_id and effective_elapsed_ms is not None:
truncated_ms = max(int(round(effective_elapsed_ms)), 0)
await self._send_raw_message(
_ConversionHelper.convert_interrupt(
playback_item_id,
playback_content_index,
truncated_ms,
)
)
# Reset trackers so subsequent playback state queries don't
# reference audio that has been interrupted client‑side.
self._audio_state_tracker.on_interrupted()
@@ -713,9 +733,6 @@ class OpenAIRealtimeWebSocketModel(RealtimeModel):
)
if not automatic_response_cancellation_enabled:
await self._cancel_response()
# Avoid sending conversation.item.truncate here. When the session's
# turn_detection.interrupt_response is enabled (GA default), the server emits
# conversation.item.truncated after the VAD start and takes care of history updates.
elif parsed.type == "response.created":
self._ongoing_response = True
await self._emit_event(RealtimeModelTurnStartedEvent())
tests/realtime/test_openai_realtime.py
+24 -8
View File
@@ -516,7 +516,7 @@ class TestEventHandlingRobustness(TestOpenAIRealtimeWebSocketModel):
# Prepare tracker state to simulate ongoing audio
model._audio_state_tracker.set_audio_format("pcm16")
model._audio_state_tracker.on_audio_delta("i1", 0, b"aaaa")
model._audio_state_tracker.on_audio_delta("i1", 0, b"a" * 48)
model._ongoing_response = True
# Patch sending to avoid websocket dependency
@@ -537,6 +537,17 @@ class TestEventHandlingRobustness(TestOpenAIRealtimeWebSocketModel):
}
)
truncate_events = [
call.args[0]
for call in model._send_raw_message.await_args_list
if getattr(call.args[0], "type", None) == "conversation.item.truncate"
]
assert truncate_events
truncate_event = truncate_events[0]
assert truncate_event.item_id == "i1"
assert truncate_event.content_index == 0
assert truncate_event.audio_end_ms == 1
# Output transcript delta
await model._handle_ws_event(
{
@@ -836,23 +847,27 @@ class TestSendEventAndConfig(TestOpenAIRealtimeWebSocketModel):
for event in audio_deltas:
await model._handle_ws_event(event)
# Should accumulate audio length: 8 bytes / 24 / 2 * 1000 = milliseconds
# Total: 8 bytes / 24 / 2 * 1000
expected_length = (8 / 24 / 2) * 1000
# Should accumulate audio length: 8 bytes -> 4 samples -> (4 / 24000) * 1000 ≈ 0.167 ms
expected_length = (8 / (24_000 * 2)) * 1000
# Test through the actual audio state tracker
audio_state = model._audio_state_tracker.get_state("item_1", 0)
assert audio_state is not None
assert abs(audio_state.audio_length_ms - expected_length) < 0.001
assert audio_state.audio_length_ms == pytest.approx(expected_length, rel=0, abs=1e-6)
def test_calculate_audio_length_ms_pure_function(self, model):
"""Test the pure audio length calculation function."""
from agents.realtime._util import calculate_audio_length_ms
# Test various audio buffer sizes for pcm16 format
assert calculate_audio_length_ms("pcm16", b"test") == (4 / 24 / 2) * 1000 # 4 bytes
expected_pcm = (len(b"test") / (24_000 * 2)) * 1000
assert calculate_audio_length_ms("pcm16", b"test") == pytest.approx(
expected_pcm, rel=0, abs=1e-6
) # 4 bytes
assert calculate_audio_length_ms("pcm16", b"") == 0 # empty
assert calculate_audio_length_ms("pcm16", b"a" * 48) == 1000.0 # exactly 1000ms worth
assert calculate_audio_length_ms("pcm16", b"a" * 48) == pytest.approx(
(48 / (24_000 * 2)) * 1000, rel=0, abs=1e-6
) # exactly 1ms worth
# Test g711 format
assert calculate_audio_length_ms("g711_ulaw", b"test") == (4 / 8000) * 1000 # 4 bytes
@@ -879,7 +894,8 @@ class TestSendEventAndConfig(TestOpenAIRealtimeWebSocketModel):
# Test that audio state is tracked correctly
audio_state = model._audio_state_tracker.get_state("test_item", 5)
assert audio_state is not None
assert audio_state.audio_length_ms == (4 / 24 / 2) * 1000 # 4 bytes in milliseconds
expected_ms = (len(b"test") / (24_000 * 2)) * 1000
assert audio_state.audio_length_ms == pytest.approx(expected_ms, rel=0, abs=1e-6)
# Test that last audio item is tracked
last_item = model._audio_state_tracker.get_last_audio_item()
tests/realtime/test_playback_tracker.py
+6 -5
View File
@@ -64,9 +64,9 @@ class TestPlaybackTracker:
state = tracker.get_state("item_1", 0)
assert state is not None
# Should accumulate: 8 bytes / 24 / 2 * 1000 = 166.67ms
expected_length = (8 / 24 / 2) * 1000
assert abs(state.audio_length_ms - expected_length) < 0.01
# Should accumulate: 8 bytes -> 4 samples -> (4 / 24000) * 1000 ≈ 0.167ms
expected_length = (8 / (24_000 * 2)) * 1000
assert state.audio_length_ms == pytest.approx(expected_length, rel=0, abs=1e-6)
def test_state_cleanup_on_interruption(self):
"""Test both trackers properly reset state on interruption."""
@@ -105,8 +105,9 @@ class TestPlaybackTracker:
# Test PCM format (24kHz, default)
pcm_bytes = b"test" # 4 bytes
pcm_length = calculate_audio_length_ms("pcm16", pcm_bytes)
assert pcm_length == (4 / 24 / 2) * 1000 # ~83.33ms
expected_pcm = (len(pcm_bytes) / (24_000 * 2)) * 1000
assert pcm_length == pytest.approx(expected_pcm, rel=0, abs=1e-6)
# Test None format (defaults to PCM)
none_length = calculate_audio_length_ms(None, pcm_bytes)
assert none_length == pcm_length
assert none_length == pytest.approx(expected_pcm, rel=0, abs=1e-6)
tests/realtime/test_playback_tracker_manual_unit.py
+3 -3
View File
@@ -5,16 +5,16 @@ def test_playback_tracker_on_play_bytes_and_state():
tr = RealtimePlaybackTracker()
tr.set_audio_format("pcm16") # PCM path
# 48k bytes -> (48000 / 24 / 2) * 1000 = 1,000,000ms per current util
# 48k bytes -> (48000 / (24000 * 2)) * 1000 = 1_000ms
tr.on_play_bytes("item1", 0, b"x" * 48000)
st = tr.get_state()
assert st["current_item_id"] == "item1"
assert st["elapsed_ms"] and abs(st["elapsed_ms"] - 1_000_000.0) < 1e-6
assert st["elapsed_ms"] and abs(st["elapsed_ms"] - 1_000.0) < 1e-6
# Subsequent play on same item accumulates
tr.on_play_ms("item1", 0, 500.0)
st2 = tr.get_state()
assert st2["elapsed_ms"] and abs(st2["elapsed_ms"] - 1_000_500.0) < 1e-6
assert st2["elapsed_ms"] and abs(st2["elapsed_ms"] - 1_500.0) < 1e-6
# Interruption clears state
tr.on_interrupted()