Files
omnigent-ai--omnigent/tests/e2e_ui/chat/test_failure_error_card.py
T
Pat Sukprasert 1ed6b49671 fix(codex-native): surface standalone turn errors (#5060)
* fix(codex-native): surface standalone turn errors

Signed-off-by: Pat Sukprasert <pattara.sk127@gmail.com>

* fix(codex-native): dedupe terminal error edges

Signed-off-by: Pat Sukprasert <pattara.sk127@gmail.com>

---------

Signed-off-by: Pat Sukprasert <pattara.sk127@gmail.com>
2026-08-20 14:07:36 +08:00

304 lines
11 KiB
Python

"""Failure error pills render as clear English, not a raw code + log blob.
A harness launch/turn failure persists an ``error`` transcript item. Before,
the chat rendered it as ``Error · <code>`` over the raw message. Now the
failure renders as a centered pill leading with a human headline: a
classified failure shows its friendly title, and an unclassified one still
maps its ``code`` to a plain-English sentence (mirror of
``describe_failure_code`` / ``FAILURE_CODE_DESCRIPTIONS``) rather than
exposing the enum. The pill expands in place to the message and diagnostics.
Seeds the error item straight into the store (like ``seed_committed_turn``)
so the assertion is on transcript hydration + rendering, deterministic and
independent of any live turn.
"""
from __future__ import annotations
import json
import httpx
import pytest
from playwright.sync_api import Page, Route, expect
from tests.e2e_ui.conftest import _server_state, seed_committed_turn
def _publish_native_status(
base_url: str,
session_id: str,
status: str,
*,
response_id: str,
output: str | None = None,
) -> None:
"""Publish the status payload used by native harness forwarders."""
data: dict[str, object] = {"status": status, "response_id": response_id}
if output is not None:
data["output"] = output
response = httpx.post(
f"{base_url}/v1/sessions/{session_id}/events",
json={"type": "external_session_status", "data": data},
timeout=10.0,
)
response.raise_for_status()
def _seed_error_item(session_id: str, *, code: str, message: str) -> None:
"""Append a committed ``error`` transcript item to the session's store.
Mirrors ``seed_committed_turn`` but writes an error banner item, so the
chat hydrates and renders it through the same path a real
``response.error`` persists.
:param session_id: Session to append to, e.g. ``"conv_abc123"``.
:param code: Error classifier, e.g. ``"required_terminal_exited"``.
:param message: Raw error message stored alongside the code.
:raises RuntimeError: If the server under test isn't one we spawned.
"""
from omnigent.entities import ErrorData, NewConversationItem
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
database_uri = _server_state.get("database_uri")
if not database_uri:
raise RuntimeError(
"seeding an error item needs the spawned server's database; it is "
"unavailable when running against --ui-base-url."
)
SqlAlchemyConversationStore(str(database_uri)).append(
session_id,
[
NewConversationItem(
type="error",
response_id="resp_seeded_error",
data=ErrorData(source="execution", code=code, message=message),
),
],
)
def test_unclassified_failure_renders_english_headline_not_raw_code(
page: Page,
seeded_session: tuple[str, str],
) -> None:
"""A known failure code reads as a sentence, never the bare enum.
:param page: Playwright page fixture.
:param seeded_session: ``(base_url, session_id)`` from the local server.
:returns: None.
"""
base_url, session_id = seeded_session
_seed_error_item(
session_id,
code="required_terminal_exited",
message="Required terminal exited unexpectedly; the runtime is no longer available.",
)
page.goto(f"{base_url}/c/{session_id}")
pill = page.get_by_test_id("error-pill")
# The friendly, code-derived headline is shown...
expect(pill).to_contain_text("The agent's terminal exited unexpectedly", timeout=15_000)
# ...and the raw enum is not surfaced as the headline.
expect(pill).not_to_contain_text("Error · required_terminal_exited", timeout=15_000)
def test_live_native_failure_status_surfaces_each_turn(
page: Page,
seeded_session: tuple[str, str],
) -> None:
"""Each failed native response renders its status-carried error message."""
base_url, session_id = seeded_session
message = "You've hit your usage limit."
page.goto(f"{base_url}/c/{session_id}")
expect(page.get_by_role("textbox", name="Message the agent")).to_be_visible(timeout=15_000)
_publish_native_status(base_url, session_id, "running", response_id="codex_turn_1")
_publish_native_status(
base_url,
session_id,
"failed",
response_id="codex_turn_1",
output=message,
)
pills = page.get_by_test_id("error-pill")
expect(pills).to_have_count(1, timeout=15_000)
_publish_native_status(base_url, session_id, "running", response_id="codex_turn_2")
_publish_native_status(
base_url,
session_id,
"failed",
response_id="codex_turn_2",
output=message,
)
expect(pills).to_have_count(2, timeout=15_000)
second_pill = pills.nth(1)
second_pill.locator('button[aria-expanded="false"]').click()
expect(second_pill.get_by_test_id("error-message-content")).to_contain_text(message)
def test_persisted_failure_expands_retries_and_dismisses_locally(
page: Page,
seeded_session: tuple[str, str],
) -> None:
"""A persisted recoverable error exposes details and local controls.
:param page: Playwright page fixture.
:param seeded_session: ``(base_url, session_id)`` from the local server.
:returns: None.
"""
base_url, session_id = seeded_session
_seed_error_item(
session_id,
code="required_terminal_exited",
message=(
"Required terminal exited unexpectedly; the runtime is no longer available.\n\n"
"Terminal diagnostics:\nexit_code: 1\nsignal: SIGTERM\n\n"
"Last captured output:\nfatal: runner unavailable"
),
)
retry_payloads: list[dict[str, object]] = []
def _recover(route: Route) -> None:
payload = route.request.post_data_json
if payload.get("type") != "retry_session":
route.continue_()
return
retry_payloads.append(payload)
route.fulfill(
status=200,
content_type="application/json",
body=json.dumps(
{
"queued": False,
"recovered": True,
"recovery": "runner_relaunched",
}
),
)
page.route(f"**/v1/sessions/{session_id}/events", _recover)
page.goto(f"{base_url}/c/{session_id}")
pill = page.get_by_test_id("error-pill")
expect(pill).to_be_visible(timeout=15_000)
headline = pill.get_by_role(
"button", name="The agent's terminal exited unexpectedly", exact=False
)
expect(headline).to_have_attribute("aria-expanded", "false")
expect(pill.get_by_test_id("error-message-content")).to_have_count(0)
headline.focus()
page.keyboard.press("Enter")
expect(headline).to_have_attribute("aria-expanded", "true")
expect(pill.get_by_role("heading", name="Message")).to_be_visible()
expect(pill.get_by_test_id("error-message-content")).to_contain_text(
"Required terminal exited unexpectedly"
)
# Clicking inside the expanded message body must not collapse the pill.
pill.get_by_test_id("error-message-content").click()
expect(headline).to_have_attribute("aria-expanded", "true")
# Clicking pill padding (outside any button) toggles expansion.
pill.click(position={"x": 3, "y": 3})
expect(headline).to_have_attribute("aria-expanded", "false")
pill.click(position={"x": 3, "y": 3})
expect(headline).to_have_attribute("aria-expanded", "true")
pill.get_by_role("button", name="View diagnostics").click()
tabs = pill.get_by_role("tablist", name="Diagnostic sections")
expect(tabs).to_be_visible()
terminal_tab = tabs.get_by_role("tab", name="Terminal")
expect(terminal_tab).to_have_attribute("aria-selected", "true")
expect(pill.get_by_test_id("error-diagnostics-content")).to_contain_text("exit_code: 1")
tabs.get_by_role("tab", name="Last captured output").click()
expect(pill.get_by_test_id("error-diagnostics-content")).to_contain_text(
"fatal: runner unavailable"
)
# Retry triggers recovery and removes the pill rather than expanding it.
pill.get_by_role("button", name="Retry").click()
expect(pill).to_have_count(0)
assert retry_payloads == [{"type": "retry_session", "data": {}}]
page.reload()
pill = page.get_by_test_id("error-pill")
expect(pill).to_be_visible(timeout=15_000)
pill.get_by_role("button", name="Dismiss error message").click()
expect(pill).to_have_count(0)
@pytest.mark.parametrize("viewport_width", [1440, 2400])
def test_error_row_divider_spans_the_chat_column(
page: Page,
seeded_session: tuple[str, str],
viewport_width: int,
) -> None:
"""The banner's dashed rule spans the full chat column, not just the pill.
Regression net for the error-only shrink-wrap: ``MessageContent``
defaults to ``w-fit``, so an error-only bubble once clipped the rule to
the 560px pill (~592px) instead of the column. Widths are compared
against a long-text assistant turn measured in the same viewport — no
hardcoded pixel values. Runs at two widths since the column is
responsive below its ``max-w-3xl`` cap: 2400px lets the column reach
the cap (where the shrink-wrap shows — the pill fits inside it), while
1440px squeezes the column below the pill's width (``max-w-full`` caps
the pill either way) and locks the invariant against a wrapper that
narrows the row below the column.
:param page: Playwright page fixture.
:param seeded_session: ``(base_url, session_id)`` from the local server.
:param viewport_width: Browser width in CSS pixels for this run.
:returns: None.
"""
base_url, session_id = seeded_session
seed_committed_turn(
session_id,
prompt="Write something long.",
reply=(
"A long assistant answer that stretches the chat column to its full "
"width regardless of the viewport size. " * 12
),
response_id="resp_geometry_long",
)
_seed_error_item(
session_id,
code="required_terminal_exited",
message="Required terminal exited unexpectedly; the runtime is no longer available.",
)
page.set_viewport_size({"width": viewport_width, "height": 1080})
page.goto(f"{base_url}/c/{session_id}")
pill = page.get_by_test_id("error-pill")
expect(pill).to_be_visible(timeout=15_000)
widths = page.evaluate(
"""() => {
const pill = document.querySelector('[data-testid="error-pill"]');
const row = pill.parentElement;
const divider = row.querySelector('span[aria-hidden="true"]');
const bubbles = [...document.querySelectorAll('[data-testid="message-bubble"]')];
const textBubble = bubbles.find((b) =>
b.textContent.includes('stretches the chat column'));
return {
row: row.getBoundingClientRect().width,
divider: divider.getBoundingClientRect().width,
textTurn: textBubble.firstElementChild.getBoundingClientRect().width,
};
}"""
)
assert abs(widths["row"] - widths["textTurn"]) <= 2, (
f"error row spans {widths['row']:.0f}px but the chat column spans "
f"{widths['textTurn']:.0f}px — the banner shrink-wrapped the pill"
)
assert abs(widths["divider"] - widths["row"]) <= 1, (
f"dashed rule spans {widths['divider']:.0f}px of its {widths['row']:.0f}px row"
)