perf(host): open daemon readiness polls tight, then back off (#5190)

The host-online, runner-online and Claude-terminal-ready waits all polled
on a flat 0.5s cadence. Those waits gate every native-harness launch and
usually resolve on the first probe or two — a warm host is already online,
a fresh runner connects in a second or two — so a flat cadence spends up
to a full interval doing nothing after the thing is already ready.

Replace the fixed sleep with `daemon_poll_intervals()`: open at 0.1s, grow
geometrically, hold at the existing 0.5s for the long tail. Fast launches
notice readiness sooner without a long wait hammering the server.

`DAEMON_POLL_INTERVAL_S` keeps its name and value as the steady-state cap,
so `connect.py`'s runner-exit watcher still matches it; its comment now
notes the client's opening probes are tighter.

Signed-off-by: harry-yao_data <harry.yao@databricks.com>
Co-authored-by: harry-yao_data <harry.yao@databricks.com>
Co-authored-by: Isaac <no-reply@databricks.com>

omnigent/claude_native.py

@@ -108,7 +108,7 @@ from omnigent.claude_native_state import (
 from omnigent.conversation_browser import conversation_url, open_conversation_link_if_enabled
 from omnigent.entities.session_resources import terminal_resource_id
 from omnigent.host.daemon_launch import (
-    DAEMON_POLL_INTERVAL_S,
+    daemon_poll_intervals,
     error_text,
     launch_or_reuse_daemon_runner,
     open_daemon_client,
@@ -3824,11 +3824,12 @@ async def _wait_for_claude_terminal_ready(
     :raises click.ClickException: If no terminal appears in time.
     """
     deadline = asyncio.get_event_loop().time() + timeout_s
+    intervals = daemon_poll_intervals()
     while asyncio.get_event_loop().time() < deadline:
         terminal_id = await _find_running_claude_terminal(client, session_id)
         if terminal_id is not None:
             return terminal_id
-        await asyncio.sleep(DAEMON_POLL_INTERVAL_S)
+        await asyncio.sleep(next(intervals))
     raise click.ClickException(
         f"The runner did not create the Claude terminal for {session_id!r} "
         f"within {timeout_s:.0f}s."

omnigent/host/connect.py

@@ -185,8 +185,8 @@ _LOG_TAIL_MAX_BYTES = 4096
 # the error summary above it remains visible.
 _LOG_TAIL_MAX_LINES = 15
 
-# Poll cadence for the per-runner exit watcher. 0.5s matches the
-# client's online-poll cadence (daemon_launch.DAEMON_POLL_INTERVAL_S),
+# Poll cadence for the per-runner exit watcher. 0.5s matches the client's
+# steady-state online-poll cadence (daemon_launch.DAEMON_POLL_INTERVAL_S),
 # so a crashed runner is reported within about one client poll.
 _RUNNER_WATCH_INTERVAL_S = 0.5
 

omnigent/host/daemon_launch.py

@@ -15,6 +15,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+from collections.abc import Iterator
 
 import click
 import httpx
@@ -22,10 +23,32 @@ import httpx
 from omnigent.claude_native_bridge import url_component
 from omnigent.process_logging import display_log_path, process_log_dir
 
-# Poll cadence while waiting for a daemon-spawned runner to connect its
-# tunnel or for a resource to appear.
+# Steady-state poll cadence while waiting for a daemon-spawned runner to
+# connect its tunnel or for a resource to appear.
 DAEMON_POLL_INTERVAL_S = 0.5
 
+# Readiness usually lands on the first probe or two, so open tight and ease
+# off to the steady cadence for the long tail.
+DAEMON_POLL_INITIAL_INTERVAL_S = 0.1
+DAEMON_POLL_BACKOFF_FACTOR = 1.5
+
+
+def daemon_poll_intervals() -> Iterator[float]:
+    """
+    Yield successive sleeps between daemon readiness probes.
+
+    Starts at :data:`DAEMON_POLL_INITIAL_INTERVAL_S` and grows
+    geometrically to :data:`DAEMON_POLL_INTERVAL_S`, then holds there.
+    Infinite: callers stop on their own deadline.
+
+    :returns: Iterator of sleep durations in seconds, e.g.
+        ``0.1, 0.15, 0.225, ... 0.5, 0.5``.
+    """
+    interval = DAEMON_POLL_INITIAL_INTERVAL_S
+    while True:
+        yield interval
+        interval = min(interval * DAEMON_POLL_BACKOFF_FACTOR, DAEMON_POLL_INTERVAL_S)
+
 
 def _json_body(resp: httpx.Response) -> dict[str, object]:
     """Decode a host/runner status response body, tolerating non-JSON.
@@ -120,6 +143,7 @@ async def wait_for_host_online(
     :raises click.ClickException: If the host is not online in time.
     """
     deadline = asyncio.get_event_loop().time() + timeout_s
+    intervals = daemon_poll_intervals()
     last_error: httpx.TransportError | None = None
     while asyncio.get_event_loop().time() < deadline:
         try:
@@ -129,7 +153,7 @@ async def wait_for_host_online(
         else:
             if resp.status_code == 200 and _json_body(resp).get("status") == "online":
                 return
-        await asyncio.sleep(DAEMON_POLL_INTERVAL_S)
+        await asyncio.sleep(next(intervals))
     message = (
         f"The connect daemon for host {host_id!r} did not come online within {timeout_s:.0f}s."
     )
@@ -180,6 +204,7 @@ async def wait_for_runner_online(
         does not connect in time.
     """
     deadline = asyncio.get_event_loop().time() + timeout_s
+    intervals = daemon_poll_intervals()
     last_error: httpx.TransportError | None = None
     while asyncio.get_event_loop().time() < deadline:
         try:
@@ -199,7 +224,7 @@ async def wait_for_runner_online(
                     raise click.ClickException(
                         f"Runner {runner_id!r} failed to start: {exit_error}"
                     )
-        await asyncio.sleep(DAEMON_POLL_INTERVAL_S)
+        await asyncio.sleep(next(intervals))
     message = f"Runner {runner_id!r} did not connect within {timeout_s:.0f}s."
     if last_error is not None:
         message += f" Last connection error: {last_error!r}."

tests/host/test_daemon_launch.py

@@ -146,12 +146,15 @@ class _AlwaysHtml:
 
 @pytest.fixture
 def fast_poll(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Shrink the poll interval so the wait loops iterate in milliseconds.
+    """Shrink the poll intervals so the wait loops iterate in milliseconds.
 
-    Patches the module's own constant (read at call time inside the
-    loops), keeping each test well under 100ms instead of multiples
-    of the real 0.5s cadence.
+    Patches the module's own constants (read at call time by
+    ``daemon_poll_intervals``), keeping each test well under 100ms
+    instead of multiples of the real cadence. Both the opening interval
+    and the steady-state cap are shrunk — leaving the opening one at its
+    real value would dominate these short loops.
     """
+    monkeypatch.setattr(daemon_launch, "DAEMON_POLL_INITIAL_INTERVAL_S", 0.01)
     monkeypatch.setattr(daemon_launch, "DAEMON_POLL_INTERVAL_S", 0.01)
 
 
@@ -392,3 +395,26 @@ async def test_open_daemon_client_no_slice_key_without_host() -> None:
     """A hostless (local) session leaves routing to the default fallback."""
     async with open_daemon_client("https://ws.example.com/api/2.0/omnigent", {}, None) as client:
         assert OMNIGENT_SLICE_KEY_HEADER not in client.headers
+
+
+def test_daemon_poll_intervals_open_tight_then_hold_at_the_cadence() -> None:
+    """
+    Readiness probes start tight and ease off to the steady cadence.
+
+    These waits gate every native-harness launch and usually resolve in
+    the first probe or two, so the opening interval must be well under
+    the steady cadence — otherwise a resource that became ready
+    immediately still costs a full interval of dead time. The sequence
+    must also be monotonic and never exceed the cadence, so a long wait
+    does not hammer the server.
+    """
+    intervals = daemon_launch.daemon_poll_intervals()
+    first_ten = [next(intervals) for _ in range(10)]
+
+    assert first_ten[0] == daemon_launch.DAEMON_POLL_INITIAL_INTERVAL_S
+    assert first_ten[0] < daemon_launch.DAEMON_POLL_INTERVAL_S
+    assert first_ten == sorted(first_ten)
+    assert max(first_ten) == daemon_launch.DAEMON_POLL_INTERVAL_S
+    # Reaching "ready" on the third probe must cost less than the old flat
+    # cadence would have spent getting there.
+    assert sum(first_ten[:2]) < 2 * daemon_launch.DAEMON_POLL_INTERVAL_S