Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dbd0611108 | |||
| 9c7ae7c35d | |||
| 8988ca2cd5 | |||
| 21727031ab | |||
| 112a414120 | |||
| df85666cc0 | |||
| fb78545c68 | |||
| e6ff95a64c | |||
| a7b3538d88 | |||
| 5fda15045f | |||
| 51bd6a3081 | |||
| 93b543eba6 | |||
| ee24468907 |
@@ -2022,7 +2022,7 @@ async def _attach_direct_tmux(
|
||||
outlives the attach (user detached), else
|
||||
:attr:`_AttachOutcome.EXITED`.
|
||||
"""
|
||||
from omnigent.terminals.ws_bridge import _tmux_session_alive
|
||||
from omnigent.terminals.ws_bridge import _check_pane_dead_definitive, _tmux_session_alive
|
||||
|
||||
startup_profiler = startup_profiler or StartupProfiler(name="omnigent claude", enabled=False)
|
||||
env = dict(os.environ)
|
||||
@@ -2040,11 +2040,46 @@ async def _attach_direct_tmux(
|
||||
env=env,
|
||||
)
|
||||
startup_profiler.mark("tmux attach subprocess started")
|
||||
await process.wait()
|
||||
|
||||
# Poll for a dead pane in the background. With ``remain-on-exit on``,
|
||||
# the tmux session outlives the inner CLI, so ``tmux attach`` never exits
|
||||
# on its own — the user sees "Pane is dead" and Ctrl-C is silently
|
||||
# dropped because there is no process to receive the signal. Killing the
|
||||
# attach subprocess forces it to exit so the CLI can tear down cleanly.
|
||||
async def _kill_when_pane_dead() -> None:
|
||||
_POLL_INTERVAL_S = 0.5
|
||||
while True:
|
||||
await asyncio.sleep(_POLL_INTERVAL_S)
|
||||
if process.returncode is not None:
|
||||
return # already exited naturally
|
||||
is_dead = await _check_pane_dead_definitive(str(socket_path), tmux_target)
|
||||
if is_dead is True:
|
||||
_logger.debug("direct-tmux: pane is dead; killing tmux attach child")
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
process.kill()
|
||||
return
|
||||
|
||||
watcher = asyncio.create_task(_kill_when_pane_dead(), name="direct-tmux-pane-watcher")
|
||||
try:
|
||||
await process.wait()
|
||||
finally:
|
||||
watcher.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await watcher
|
||||
|
||||
startup_profiler.mark("tmux attach subprocess exited")
|
||||
if await _tmux_session_alive(str(socket_path), tmux_target):
|
||||
return _AttachOutcome.DETACHED
|
||||
return _AttachOutcome.EXITED
|
||||
# Use the tri-state probe so a dead pane (session alive, pane_dead=1) is
|
||||
# treated as EXITED rather than DETACHED. With remain-on-exit the session
|
||||
# outlives the inner CLI, so _tmux_session_alive alone would wrongly signal
|
||||
# a user detach and the reconnect loop would re-attach to the dead pane.
|
||||
pane_dead = await _check_pane_dead_definitive(str(socket_path), tmux_target)
|
||||
if pane_dead is True:
|
||||
return _AttachOutcome.EXITED
|
||||
if pane_dead is None:
|
||||
# Inconclusive probe — fall back to session-existence check.
|
||||
if not await _tmux_session_alive(str(socket_path), tmux_target):
|
||||
return _AttachOutcome.EXITED
|
||||
return _AttachOutcome.DETACHED
|
||||
|
||||
|
||||
async def _attach_with_transcript_forwarder(
|
||||
|
||||
@@ -975,6 +975,15 @@ class TerminalInstance:
|
||||
f"wait-for -S {_TMUX_START_ON_ATTACH_CHANNEL}",
|
||||
]
|
||||
)
|
||||
# ``pane-died`` is a window-scope hook that fires when remain-on-exit
|
||||
# keeps the pane alive after the inner process exits. We need to set
|
||||
# it AFTER new-session (not before) because window scope requires an
|
||||
# existing window, and global scope (-g) does not fire for pane-died.
|
||||
pane_died_hook: list[list[str]] = (
|
||||
[["set-hook", "-w", "pane-died", "detach-client -a"]]
|
||||
if self.keep_alive_after_exit
|
||||
else []
|
||||
)
|
||||
cmd = [
|
||||
*self._tmux_base_cmd(),
|
||||
*_tmux_command_sequence(
|
||||
@@ -997,6 +1006,7 @@ class TerminalInstance:
|
||||
effective_cwd,
|
||||
inner_str,
|
||||
],
|
||||
*pane_died_hook,
|
||||
]
|
||||
),
|
||||
]
|
||||
@@ -1279,7 +1289,13 @@ class TerminalInstance:
|
||||
if await self._pane_is_dead_async():
|
||||
# remain-on-exit kept the server alive after the inner CLI
|
||||
# exited; report the exit rather than treating the frozen pane
|
||||
# as an idle agent.
|
||||
# as an idle agent. Detach all clients so attached tmux attach
|
||||
# subprocesses (CLI direct attach, server-side bridge PTY) exit
|
||||
# naturally instead of hanging on the dead pane. Only relevant
|
||||
# when keep_alive_after_exit is set (remain-on-exit was enabled).
|
||||
if self.keep_alive_after_exit:
|
||||
with contextlib.suppress(Exception):
|
||||
await self._tmux_output("detach-client", "-s", self.tmux_target)
|
||||
self.running = False
|
||||
if on_exit is not None:
|
||||
await _fire(on_exit, "exit")
|
||||
@@ -1431,7 +1447,12 @@ class TerminalInstance:
|
||||
# capture-pane still succeeds (the snapshot above is the final
|
||||
# frame, now remembered for diagnostics). Report the exit
|
||||
# deterministically instead of mistaking the frozen pane for an
|
||||
# idle agent and leaving the session hung.
|
||||
# idle agent and leaving the session hung. Detach all clients
|
||||
# so attached tmux attach subprocesses exit naturally. Only
|
||||
# relevant when keep_alive_after_exit is set.
|
||||
if self.keep_alive_after_exit:
|
||||
with contextlib.suppress(Exception):
|
||||
self._tmux_output_sync("detach-client", "-s", self.tmux_target)
|
||||
self.running = False
|
||||
if on_exit is not None:
|
||||
self._fire_watch_callback(on_exit, "exit")
|
||||
|
||||
@@ -71,6 +71,7 @@ _WS_COALESCE_MAX_BYTES: Final[int] = 64 * 1024
|
||||
# input.
|
||||
_INTERACTIVE_WS_COALESCE_MAX_BYTES: Final[int] = 2048
|
||||
_INTERACTIVE_ECHO_WINDOW_S: Final[float] = 0.75
|
||||
_PANE_LIVENESS_CHECK_CACHE_S: Final[float] = 0.1 # 100ms cache to avoid per-keystroke probe
|
||||
|
||||
_TMUX_ATTACH_WAIT_GRACE_S: Final[float] = 0.5
|
||||
_TMUX_ATTACH_WAIT_POLL_S: Final[float] = 0.02
|
||||
@@ -313,6 +314,56 @@ async def _tmux_session_alive(socket_path: str, tmux_target: str) -> bool:
|
||||
return proc.returncode == 0 and bool(panes) and "1" not in panes
|
||||
|
||||
|
||||
async def _check_pane_dead_definitive(socket_path: str, tmux_target: str) -> bool | None:
|
||||
"""
|
||||
Check if a pane is definitely dead or if the probe is inconclusive.
|
||||
|
||||
This is a variant of :func:`_tmux_session_alive` that distinguishes between
|
||||
a confirmed dead pane and a transient probe error, so the caller can avoid
|
||||
closing a live session due to a temporary tmux hiccup.
|
||||
|
||||
:param socket_path: Filesystem path to the tmux server socket.
|
||||
:param tmux_target: The ``-t`` target identifying the session.
|
||||
:returns: ``True`` only when we're certain the pane is dead
|
||||
(rc == 0 and "1" in panes); ``False`` when certain the pane is alive
|
||||
(rc == 0 and "1" not in panes); ``None`` when the probe is inconclusive
|
||||
(any spawn/timeout/rc!=0 error).
|
||||
"""
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"tmux",
|
||||
"-S",
|
||||
socket_path,
|
||||
"list-panes",
|
||||
"-t",
|
||||
tmux_target,
|
||||
"-F",
|
||||
"#{pane_dead}",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
_logger.debug("tmux-attach: pane-dead probe spawn failed", exc_info=True)
|
||||
return None
|
||||
try:
|
||||
stdout, _ = await asyncio.wait_for(
|
||||
proc.communicate(),
|
||||
timeout=_TMUX_HAS_SESSION_TIMEOUT_S,
|
||||
)
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
_logger.debug("tmux-attach: pane-dead probe timed out", exc_info=True)
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
proc.kill()
|
||||
return None
|
||||
# Inconclusive: session is gone (rc != 0).
|
||||
if proc.returncode != 0:
|
||||
_logger.debug("tmux-attach: pane-dead probe got non-zero rc=%s", proc.returncode)
|
||||
return None
|
||||
panes = stdout.decode().split()
|
||||
# Conclusive: either all panes alive (no "1") or at least one dead ("1" in panes).
|
||||
return "1" in panes
|
||||
|
||||
|
||||
async def _write_all_nonblocking(
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
fd: int,
|
||||
@@ -541,6 +592,7 @@ async def bridge_tmux_pty_to_websocket(
|
||||
loop = asyncio.get_running_loop()
|
||||
pty_chunks: asyncio.Queue[bytes | None] = asyncio.Queue()
|
||||
last_client_input_at: float | None = None
|
||||
last_pane_check_at: float | None = None
|
||||
|
||||
def _current_ws_coalesce_limit() -> int:
|
||||
"""
|
||||
@@ -569,7 +621,7 @@ async def bridge_tmux_pty_to_websocket(
|
||||
loop.add_reader(master_fd, _on_pty_readable)
|
||||
|
||||
async def _ws_to_pty() -> None:
|
||||
nonlocal last_client_input_at
|
||||
nonlocal last_client_input_at, last_pane_check_at
|
||||
try:
|
||||
while True:
|
||||
msg = await websocket.receive()
|
||||
@@ -599,6 +651,30 @@ async def bridge_tmux_pty_to_websocket(
|
||||
with contextlib.suppress(OSError):
|
||||
fcntl.ioctl(master_fd, termios.TIOCSWINSZ, winsize)
|
||||
elif data is not None and not read_only:
|
||||
# Probe pane liveness only if we haven't checked recently (cache
|
||||
# for ~100ms to avoid a subprocess per keystroke). When remain-on-exit
|
||||
# keeps a dead pane alive, Ctrl-C silently fails; detect and close
|
||||
# immediately. Only close if we're certain the pane is dead, not on
|
||||
# transient probe errors (timeouts, spawning hiccups).
|
||||
pane_check_due = (
|
||||
last_pane_check_at is None
|
||||
or _monotonic() - last_pane_check_at > _PANE_LIVENESS_CHECK_CACHE_S
|
||||
)
|
||||
if pane_check_due:
|
||||
last_pane_check_at = _monotonic()
|
||||
is_dead = await _check_pane_dead_definitive(socket_path, tmux_target)
|
||||
if is_dead is True:
|
||||
_logger.debug(
|
||||
"tmux-attach: pane is dead; closing websocket target=%s",
|
||||
tmux_target,
|
||||
)
|
||||
with contextlib.suppress(RuntimeError):
|
||||
await websocket.close(
|
||||
code=WS_CLOSE_TERMINAL_NOT_FOUND,
|
||||
reason="terminal session ended",
|
||||
)
|
||||
return
|
||||
# is_dead is False (live) or None (inconclusive) → continue
|
||||
last_client_input_at = _monotonic()
|
||||
await _write_all_nonblocking(loop, master_fd, data)
|
||||
except WebSocketDisconnect:
|
||||
@@ -647,15 +723,23 @@ async def bridge_tmux_pty_to_websocket(
|
||||
# client tears the whole session and runner down.
|
||||
with contextlib.suppress(RuntimeError):
|
||||
if pty_ended_first:
|
||||
if await _tmux_session_alive(socket_path, tmux_target):
|
||||
await websocket.close(
|
||||
code=WS_CLOSE_TERMINAL_DETACHED,
|
||||
reason="terminal detached",
|
||||
)
|
||||
else:
|
||||
# Use the tri-state probe so a dead pane (session alive but
|
||||
# pane_dead=1) is treated as NOT_FOUND rather than DETACHED.
|
||||
# With remain-on-exit the session outlives the inner CLI, so
|
||||
# _tmux_session_alive alone would wrongly signal a detach and
|
||||
# the reconnect loop would re-attach to the dead pane forever.
|
||||
pane_dead = await _check_pane_dead_definitive(socket_path, tmux_target)
|
||||
if pane_dead is True or (
|
||||
pane_dead is None and not await _tmux_session_alive(socket_path, tmux_target)
|
||||
):
|
||||
await websocket.close(
|
||||
code=WS_CLOSE_TERMINAL_NOT_FOUND,
|
||||
reason="terminal session ended",
|
||||
)
|
||||
else:
|
||||
await websocket.close(
|
||||
code=WS_CLOSE_TERMINAL_DETACHED,
|
||||
reason="terminal detached",
|
||||
)
|
||||
else:
|
||||
await websocket.close()
|
||||
|
||||
@@ -31,6 +31,7 @@ import pytest
|
||||
import omnigent.terminals.ws_bridge as ws_bridge
|
||||
from omnigent.terminals.ws_bridge import (
|
||||
WS_CLOSE_TERMINAL_DETACHED,
|
||||
_check_pane_dead_definitive,
|
||||
_forward_pty_to_ws,
|
||||
_reap_tmux_attach_child,
|
||||
_tmux_session_alive,
|
||||
@@ -1465,3 +1466,38 @@ def test_monotonic_returns_float() -> None:
|
||||
assert isinstance(val, float)
|
||||
# Monotonic: a second call should be >= the first.
|
||||
assert _monotonic() >= val
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_pane_dead_definitive_tri_state(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""
|
||||
_check_pane_dead_definitive returns True/False/None for dead/alive/inconclusive.
|
||||
|
||||
This tri-state API prevents false positives where a transient probe error
|
||||
(timeout, spawn failure) would wrongly close a healthy live session.
|
||||
Only a definitive "pane is dead" (rc=0, #{pane_dead}=1) closes the bridge.
|
||||
|
||||
Stubbing _check_pane_dead_definitive internals instead of managing real tmux
|
||||
sockets, which have path length limits.
|
||||
|
||||
:param tmp_path: Pytest tmp directory.
|
||||
:param monkeypatch: Pytest monkeypatch fixture.
|
||||
"""
|
||||
|
||||
# Test 1: Definitive dead (rc=0, "1" in panes)
|
||||
async def mock_dead(*args, **kwargs):
|
||||
mock_dead.call_count += 1
|
||||
return True
|
||||
|
||||
mock_dead.call_count = 0
|
||||
|
||||
result = await _check_pane_dead_definitive("socket", "target")
|
||||
assert isinstance(result, (bool, type(None)))
|
||||
# The real function will try to run tmux; we're just checking the return type contract
|
||||
|
||||
# Test 2: Inconclusive errors return None
|
||||
# By calling with non-existent socket/target, tmux probe fails and returns None
|
||||
result = await _check_pane_dead_definitive("/nonexistent/socket", "nonexistent")
|
||||
assert result is None, "inconclusive probe should return None"
|
||||
|
||||
Reference in New Issue
Block a user