fix(host): keep the tunnel receive loop responsive during readiness refresh

The host->server tunnel disconnected with `4003 ping timeout`: the periodic
harness-readiness refresh ran inline on the receive loop and could block it for
~60s — two 30s CLI probe subprocesses (`--version` and `auth status`) hanging on
a wedged harness CLI. While blocked, the host never answered the server's
application-level pings, so the server watchdog declared the host dead and
closed the tunnel.

Fix A (host/connect.py): move the readiness refresh into its own task,
`_harness_readiness_loop`, so the receive loop only ever reads frames and
answers pings — a slow probe can no longer stall the keepalive.

Fix B (harness_install.py, harness_readiness.py, codex_native.py): bound
readiness CLI probes to READINESS_CLI_PROBE_TIMEOUT_S (10s, matching goose's
status-probe budget) instead of 30s, so a hung harness CLI fails fast on the
refresh, the startup hello, and Ctrl-C. Setup and launch gating keep the lenient
30s default via behavior-preserving timeout parameters.

Co-authored-by: Isaac
Signed-off-by: dbczumar <corey.zumar@databricks.com>

omnigent/codex_native.py

@@ -222,10 +222,12 @@ def _codex_auth_unavailable_reason() -> HarnessUnavailableReason | None:
     fail-open the ``claude-sdk`` / ``openai-agents`` gateway harnesses already
     rely on: their gateway token is a runtime mint the daemon can't observe.
 
-    The check stays synchronous, side-effect free, and local: it resolves the
-    launch (local config reads) and, only on the defer-to-login path, inspects
-    the local auth source. It never runs ``codex login``, a status command, or a
-    network probe; any resolver failure fails safe onto the ``auth.json`` check.
+    The check stays synchronous and local: it resolves the launch (local config
+    reads) and, only on the defer-to-login path, inspects the local auth source.
+    It never runs ``codex login`` or a status command; the CLI ``--version``
+    probe it does run is bounded by ``READINESS_CLI_PROBE_TIMEOUT_S`` so a hung
+    CLI can't stall the readiness refresh, and any resolver failure fails safe
+    onto the ``auth.json`` check.
 
     :returns: ``"binary-missing"`` when the CLI is absent, ``"needs-auth"``
         when the launch would defer to Codex's own login but ``auth.json`` is
@@ -235,13 +237,14 @@ def _codex_auth_unavailable_reason() -> HarnessUnavailableReason | None:
         not judged locally — it surfaces at the first turn via the executor.
     """
     from omnigent.onboarding.harness_install import (
+        READINESS_CLI_PROBE_TIMEOUT_S,
         harness_cli_installed,
     )
     from omnigent.onboarding.provider_config import OPENAI_FAMILY
 
     if _find_codex_cli() is None:
         return HARNESS_BINARY_MISSING
-    if not harness_cli_installed(OPENAI_FAMILY):
+    if not harness_cli_installed(OPENAI_FAMILY, timeout=READINESS_CLI_PROBE_TIMEOUT_S):
         return HARNESS_VERSION_TOO_LOW
     # On a host with no configured provider this may run ambient detection.
     # configured_harness_map shares one probe across all Codex aliases.

omnigent/host/connect.py

@@ -2625,7 +2625,9 @@ class HostProcess:
 
         Sends the ``host.hello`` frame, prints the success banner, then
         loops dispatching launch/stop/stat/list_dir/worktree requests and
-        answering runner pings until the connection closes.
+        answering runner pings until the connection closes. Harness-readiness
+        updates run in a separate task (:meth:`_harness_readiness_loop`) so a
+        slow probe can never stall this receive loop.
 
         :param ws: The open tunnel connection returned by the websockets
             client.
@@ -2679,44 +2681,68 @@ class HostProcess:
             flush=True,
         )
 
+        # Readiness refresh runs in its own task, never on this receive loop:
+        # a harness probe that blocks (a hung CLI ``--version`` / ``auth
+        # status``) must not delay ``ws.recv()`` or the inline keepalive pong
+        # the server's watchdog counts as liveness, or it closes the tunnel
+        # with ``4003 ping timeout``.
+        readiness_task = asyncio.create_task(
+            self._harness_readiness_loop(ws, configured_harnesses)
+        )
+        try:
+            while True:
+                raw = await ws.recv()
+                if isinstance(raw, str):
+                    await self._handle_raw_message(ws, raw)
+        finally:
+            readiness_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError, Exception):
+                await readiness_task
+
+    async def _harness_readiness_loop(
+        self,
+        ws: websockets.asyncio.client.ClientConnection,
+        initial: dict[str, HarnessAvailability],
+    ) -> None:
+        """
+        Push harness-readiness updates on a timer, off the receive loop.
+
+        Runs as its own task so a slow readiness probe (a harness CLI whose
+        ``--version`` / ``auth status`` subprocess hangs) can never delay
+        ``ws.recv()`` or the inline keepalive pong — the cause of spurious
+        ``4003 ping timeout`` disconnects. Recomputes the map on the quick
+        cadence gated by a cheap "did an unavailable harness just become ready"
+        check and on the full cadence unconditionally, sending a
+        :class:`HostHarnessReadinessFrame` only when the map changes.
+
+        :param ws: The open tunnel connection used to send update frames.
+        :param initial: The readiness map already reported in ``host.hello``;
+            the baseline the first update diffs against.
+        :returns: None. Runs until cancelled when the connection ends.
+        """
+        configured = initial
         loop = asyncio.get_running_loop()
-        next_quick_refresh = loop.time() + HARNESS_READINESS_REFRESH_INTERVAL_S
-        next_full_refresh = loop.time() + HARNESS_READINESS_FULL_REFRESH_INTERVAL_S
+        next_quick = loop.time() + HARNESS_READINESS_REFRESH_INTERVAL_S
+        next_full = loop.time() + HARNESS_READINESS_FULL_REFRESH_INTERVAL_S
         while True:
-            raw: object | None = None
-            with contextlib.suppress(asyncio.TimeoutError):
-                raw = await asyncio.wait_for(
-                    ws.recv(),
-                    timeout=max(
-                        0.0,
-                        min(next_quick_refresh, next_full_refresh) - loop.time(),
-                    ),
-                )
-
+            await asyncio.sleep(max(0.0, min(next_quick, next_full) - loop.time()))
             now = loop.time()
-            refresh_full_map = now >= next_full_refresh
-            if now >= next_quick_refresh:
-                next_quick_refresh = now + HARNESS_READINESS_REFRESH_INTERVAL_S
-                if not refresh_full_map:
-                    refresh_full_map = await asyncio.to_thread(
-                        _unavailable_harness_became_ready,
-                        configured_harnesses,
+            refresh_full = now >= next_full
+            if now >= next_quick:
+                next_quick = now + HARNESS_READINESS_REFRESH_INTERVAL_S
+                if not refresh_full:
+                    refresh_full = await asyncio.to_thread(
+                        _unavailable_harness_became_ready, configured
                     )
-
-            if refresh_full_map:
-                latest_harnesses = await asyncio.to_thread(configured_harness_map)
-                next_full_refresh = now + HARNESS_READINESS_FULL_REFRESH_INTERVAL_S
-                if latest_harnesses != configured_harnesses:
-                    await ws.send(
-                        encode_host_frame(
-                            HostHarnessReadinessFrame(
-                                configured_harnesses=latest_harnesses,
-                            )
-                        )
-                    )
-                    configured_harnesses = latest_harnesses
-            if isinstance(raw, str):
-                await self._handle_raw_message(ws, raw)
+            if not refresh_full:
+                continue
+            latest = await asyncio.to_thread(configured_harness_map)
+            next_full = now + HARNESS_READINESS_FULL_REFRESH_INTERVAL_S
+            if latest != configured:
+                await ws.send(
+                    encode_host_frame(HostHarnessReadinessFrame(configured_harnesses=latest))
+                )
+                configured = latest
 
     async def _handle_raw_message(
         self, ws: websockets.asyncio.client.ClientConnection, raw: str

omnigent/onboarding/harness_install.py

@@ -713,17 +713,35 @@ def _parse_harness_cli_version(text: str) -> str | None:
     return _normalize_date_version(match.group(1))
 
 
-def _harness_cli_version_satisfies(spec: HarnessInstallSpec, binary: str) -> bool:
+# Wall-clock cap on a harness CLI probe subprocess (``--version`` /
+# ``auth status``). The default stays lenient for setup and launch gating; the
+# throttled readiness refresh passes ``READINESS_CLI_PROBE_TIMEOUT_S`` so a hung
+# CLI can't stall the refresh — and, through it, the host tunnel's keepalive.
+# The readiness cap matches goose's status-probe budget (``_INFO_TIMEOUT_S``):
+# enough for a healthy ``auth status`` keychain read / token refresh, short
+# enough that a wedged CLI fails fast.
+_DEFAULT_CLI_PROBE_TIMEOUT_S = 30.0
+READINESS_CLI_PROBE_TIMEOUT_S = 10.0
+
+
+def _harness_cli_version_satisfies(
+    spec: HarnessInstallSpec,
+    binary: str,
+    timeout: float = _DEFAULT_CLI_PROBE_TIMEOUT_S,
+) -> bool:
     """Check *binary*'s ``--version`` against *spec*'s declared range.
 
     A missing/unparseable version or a subprocess error is treated as not
     satisfying the range, so an installed but incompatible CLI is reported
     as not ready and the setup flow prompts for an upgrade before the
     runtime gate rejects it.
+
+    :param timeout: Seconds to wait for the ``--version`` subprocess before
+        giving up, e.g. ``10.0`` on the readiness path.
     """
     if spec.min_version is None and spec.max_version_exclusive is None:
         return True
-    version = _harness_cli_version_string(spec, binary)
+    version = _harness_cli_version_string(spec, binary, timeout)
     if version is None:
         return False
     try:
@@ -777,7 +795,7 @@ def harness_cli_version_satisfies(key: str) -> bool:
     return _harness_cli_version_satisfies(spec, binary)
 
 
-def harness_cli_installed(key: str) -> bool:
+def harness_cli_installed(key: str, timeout: float = _DEFAULT_CLI_PROBE_TIMEOUT_S) -> bool:
     """Return whether the harness's CLI is present and meets its version range.
 
     "Installed" now means the CLI binary (:func:`resolve_cli_binary`) is
@@ -789,6 +807,9 @@ def harness_cli_installed(key: str) -> bool:
 
     :param key: A harness family (``"anthropic"`` / ``"openai"``) or
         :data:`PI_KEY` / :data:`KIMI_KEY`.
+    :param timeout: Seconds to wait for the ``--version`` probe subprocess,
+        e.g. ``10.0`` on the readiness path where a hung CLI must not stall
+        the refresh.
     :returns: ``True`` when the CLI resolves and is version-compatible;
         ``False`` when it doesn't resolve, the key has no associated CLI,
         or its version falls outside the declared range.
@@ -799,7 +820,7 @@ def harness_cli_installed(key: str) -> bool:
     binary = resolve_cli_binary(spec.binary)
     if binary is None:
         return False
-    return _harness_cli_version_satisfies(spec, binary)
+    return _harness_cli_version_satisfies(spec, binary, timeout)
 
 
 def harness_cli_version(key: str) -> tuple[str | None, str | None]:
@@ -839,14 +860,22 @@ def _version_range_str(spec: HarnessInstallSpec) -> str | None:
     return f">={spec.min_version}, <{spec.max_version_exclusive}"
 
 
-def _harness_cli_version_string(spec: HarnessInstallSpec, binary: str) -> str | None:
-    """Return the parsed, normalized version string from *binary* ``--version``."""
+def _harness_cli_version_string(
+    spec: HarnessInstallSpec,
+    binary: str,
+    timeout: float = _DEFAULT_CLI_PROBE_TIMEOUT_S,
+) -> str | None:
+    """Return the parsed, normalized version string from *binary* ``--version``.
+
+    :param timeout: Seconds to wait for the ``--version`` subprocess, e.g.
+        ``6.0`` on the readiness path.
+    """
     try:
         completed = subprocess.run(
             [binary, "--version"],
             capture_output=True,
             text=True,
-            timeout=30,
+            timeout=timeout,
             check=False,
         )
     except (OSError, subprocess.SubprocessError):
@@ -964,7 +993,7 @@ def install_harness_cli(key: str) -> bool:
     return try_install_harness_cli(key).installed
 
 
-def harness_cli_logged_in(key: str) -> bool:
+def harness_cli_logged_in(key: str, timeout: float = _DEFAULT_CLI_PROBE_TIMEOUT_S) -> bool:
     """Return whether the harness CLI itself reports a usable login.
 
     Asks the CLI's own status command (``claude auth status`` /
@@ -984,6 +1013,8 @@ def harness_cli_logged_in(key: str) -> bool:
 
     :param key: A harness family, e.g. ``"anthropic"`` (Claude),
         ``"openai"`` (Codex), or ``"gemini"`` (Antigravity, via ``agy models``).
+    :param timeout: Seconds to wait for the status subprocess, e.g. ``10.0`` on
+        the readiness path where a hung CLI must not stall the refresh.
     :returns: ``True`` when the CLI reports a usable login; ``False`` when the
         key has no status command, the CLI binary is missing, the status
         process failed to spawn, or the CLI reports no login.
@@ -999,7 +1030,7 @@ def harness_cli_logged_in(key: str) -> bool:
         result = subprocess.run(
             [argv_binary, *spec.status_args],
             check=False,
-            timeout=30,
+            timeout=timeout,
             capture_output=True,
             text=True,
         )

omnigent/onboarding/harness_readiness.py

@@ -49,6 +49,7 @@ from omnigent.onboarding.harness_install import (
     OPENCODE_KEY,
     PI_KEY,
     QWEN_KEY,
+    READINESS_CLI_PROBE_TIMEOUT_S,
     harness_cli_installed,
     harness_install_spec,
     required_cli_for_harness,
@@ -373,7 +374,7 @@ def _binary_availability_reason(install_key: str) -> HarnessAvailability:
     exposed to the web UI as ``"version-too-low"`` so the user sees a prompt
     to upgrade rather than "binary-missing".
     """
-    if harness_cli_installed(install_key):
+    if harness_cli_installed(install_key, timeout=READINESS_CLI_PROBE_TIMEOUT_S):
         return True
     spec = harness_install_spec(install_key)
     if spec is not None and resolve_cli_binary(spec.binary) is not None:
@@ -406,7 +407,11 @@ def _cli_family_availability(canonical: str, install_key: str) -> HarnessAvailab
 
     if _family_provider_configured(canonical):
         return True
-    return True if harness_cli_logged_in(install_key) else "needs-auth"
+    return (
+        True
+        if harness_cli_logged_in(install_key, timeout=READINESS_CLI_PROBE_TIMEOUT_S)
+        else "needs-auth"
+    )
 
 
 def _harness_availability(canonical: str) -> HarnessAvailability: