fix(host): run the clean shutdown path on SIGTERM

`omnigent host stop` signals a backgrounded daemon with SIGTERM, and the
daemon installed no handler for it — so Python exited where it stood and
`run()`'s cleanup never happened: tracked runners were not terminated, the
final orphan drain never ran, and the zygote's control socket was closed by
process death rather than `stop()`. The runners still went away, but only
because each noticed its dead parent and hard-exited after a 2s grace,
skipping its own graceful drain. Only the foreground `omnigent host` reached
the cleanup, via Ctrl-C's KeyboardInterrupt — despite `run()`'s docstring
promising "Ctrl-C / SIGTERM exit cleanly".

Install a SIGTERM handler that force-drops the live tunnel the same way the
resume-from-suspend watcher already does, so `_serve_frames`' recv raises at
once, and set a flag the reconnect loop checks so it breaks instead of
announcing a reconnect it will not make. A shutdown event cuts short an
in-flight backoff sleep so a SIGTERM landing mid-backoff doesn't wait it out.

SIGINT is deliberately left alone: it already reaches the cleanup as
KeyboardInterrupt, and `omnigent host` relies on that exception rather than a
clean 0 exit.

Co-authored-by: Isaac <no-reply@databricks.com>

omnigent/host/connect.py

@@ -16,6 +16,7 @@ import json
 import logging
 import os
 import re
+import signal
 import subprocess
 import sys
 from collections.abc import Callable, Iterator, Mapping
@@ -930,6 +931,11 @@ class HostProcess:
         # detected resume; read+cleared in run()'s reconnect handler to force a
         # prompt reconnect (skip the backoff).
         self._woke_from_suspend = False
+        # Set by a shutdown signal (see _request_shutdown): the flag stops
+        # run()'s loop reconnecting, the event cuts short a backoff sleep so a
+        # SIGTERM mid-backoff doesn't wait it out.
+        self._shutdown_requested = False
+        self._shutdown_event: asyncio.Event | None = None
 
     def _tracked_runner_pids(self) -> set[int]:
         """PIDs of runners this host spawned and still tracks directly.
@@ -2691,7 +2697,9 @@ class HostProcess:
 
         Connects to the server, sends hello, and enters the
         receive loop. Reconnects with exponential backoff on
-        disconnect. Ctrl-C / SIGTERM exit cleanly.
+        disconnect. Ctrl-C / SIGTERM exit cleanly — see
+        :meth:`_install_shutdown_signal_handlers` for why SIGTERM needs
+        a handler to get there.
 
         :returns: None. Runs until the process is terminated.
         :raises HostConnectError: On a permanent failure — auth /
@@ -2703,6 +2711,11 @@ class HostProcess:
         # harness is reported as unknown and must not prevent registration.
         await self._initialize_capabilities()
 
+        # Route SIGTERM into the same clean shutdown Ctrl-C takes, so the
+        # cleanup in this method's finally block runs for a backgrounded
+        # daemon too.
+        self._install_shutdown_signal_handlers()
+
         # Reap orphaned harness/tool grandchildren that reparent here when a
         # runner dies (this host is PID 1 in a container, or a subreaper
         # otherwise). Without this they pile up as <defunct> zombies and can
@@ -2730,6 +2743,8 @@ class HostProcess:
         backoff = _RECONNECT_BASE_S
         try:
             while True:
+                if self._shutdown_requested:
+                    break
                 try:
                     await self._connect_and_serve()
                     backoff = _RECONNECT_BASE_S
@@ -2741,6 +2756,11 @@ class HostProcess:
                     # ``run_host_process`` can fail loud.
                     raise
                 except Exception as exc:
+                    if self._shutdown_requested:
+                        # We aborted this tunnel ourselves (see
+                        # _request_shutdown) — don't classify it as a
+                        # disconnect or announce a reconnect we won't make.
+                        break
                     if not isinstance(exc, InvalidURI):
                         # Any non-redirect failure (5xx bounce, network
                         # blip, mid-serve drop) breaks a login-redirect
@@ -2857,7 +2877,9 @@ class HostProcess:
                         if woke
                         else (" (recycle — prompt reconnect)" if recycle else ""),
                     )
-                    await asyncio.sleep(wait_s)
+                    await self._sleep_before_reconnect(wait_s)
+                    if self._shutdown_requested:
+                        break
                     import random
 
                     if recycle:
@@ -2904,6 +2926,78 @@ class HostProcess:
                     self._zygote.stop()
                 self._zygote = None
 
+    def _install_shutdown_signal_handlers(self) -> None:
+        """Route SIGTERM into the clean shutdown path Ctrl-C already takes.
+
+        SIGINT arrives as ``KeyboardInterrupt``, which :meth:`run` catches on
+        its way into the cleanup that terminates tracked runners, drains
+        orphans and stops the zygote. SIGTERM has no such default: Python
+        exits where it stands, so a backgrounded daemon — the only kind
+        ``omnigent host stop`` can signal — never ran that cleanup. Its
+        runners were left to notice the dead parent themselves and hard-exit
+        after their 2s grace, skipping their own graceful drain.
+
+        Best-effort: a platform with no ``add_signal_handler`` (Windows) keeps
+        the previous behavior rather than failing daemon startup.
+
+        SIGINT is deliberately left alone: it already reaches the cleanup as
+        ``KeyboardInterrupt``, and ``omnigent host`` relies on that exception
+        to exit with the interrupt's status rather than a clean 0.
+
+        :returns: None.
+        """
+        loop = asyncio.get_running_loop()
+        self._shutdown_event = asyncio.Event()
+        with contextlib.suppress(NotImplementedError, RuntimeError, ValueError):
+            loop.add_signal_handler(signal.SIGTERM, self._request_shutdown, signal.SIGTERM)
+
+    def _request_shutdown(self, sig: signal.Signals) -> None:
+        """Unwind :meth:`run` into its cleanup after a shutdown signal.
+
+        Force-drops the live tunnel the same way :meth:`_on_resume_from_suspend`
+        does, so ``_serve_frames``' ``recv`` raises at once, and wakes any
+        reconnect backoff. :meth:`run`'s loop then sees the flag and breaks
+        instead of reconnecting, reaching the cleanup in its ``finally``.
+
+        Deliberately does not cancel ``run()``'s own task: that cleanup awaits
+        the reaper and watcher tasks, which a pending cancellation would cut
+        short. Runs synchronously on the event loop, so reading ``self._ws``
+        is atomic w.r.t. ``_serve_frames``.
+
+        With no live tunnel (a shutdown that lands mid-handshake) there is
+        nothing to abort; the loop breaks once the handshake settles, bounded
+        by its own connect timeout.
+
+        :param sig: The signal received, e.g. ``signal.SIGTERM``.
+        :returns: None.
+        """
+        if self._shutdown_requested:
+            # A second signal means the operator is impatient; the sender's own
+            # SIGKILL escalation is the backstop, so just note it.
+            _logger.info("Received %s again; shutdown already in progress", sig.name)
+            return
+        self._shutdown_requested = True
+        _logger.info("Received %s; shutting down host", sig.name)
+        if self._shutdown_event is not None:
+            self._shutdown_event.set()
+        ws = self._ws
+        transport = getattr(ws, "transport", None) if ws is not None else None
+        if transport is not None:
+            with contextlib.suppress(Exception):
+                transport.abort()
+
+    async def _sleep_before_reconnect(self, wait_s: float) -> None:
+        """Wait out the reconnect backoff, returning early on shutdown.
+
+        :param wait_s: Backoff to wait, in seconds, e.g. ``0.5``.
+        :returns: None.
+        """
+        if self._shutdown_event is None:
+            await asyncio.sleep(wait_s)
+            return
+        with contextlib.suppress(TimeoutError):
+            await asyncio.wait_for(self._shutdown_event.wait(), timeout=wait_s)
+
     def _on_resume_from_suspend(self, gap_s: float) -> None:
         """Force-drop the tunnel after a detected wake from system suspend.
 

tests/host/test_connect.py

@@ -6,6 +6,8 @@ import asyncio
 import contextlib
 import errno
 import logging
+import os
+import signal
 import subprocess
 import threading
 import time
@@ -18,6 +20,7 @@ from websockets.datastructures import Headers
 from websockets.exceptions import ConnectionClosedError, InvalidStatus, InvalidURI
 from websockets.http11 import Response
 
+from omnigent._platform import IS_POSIX
 from omnigent.host import HOST_FATAL_EXIT_CODE
 from omnigent.host.connect import (
     HostConnectError,
@@ -4746,3 +4749,69 @@ def test_post_connect_auth_rejection_escalates_without_going_fatal(
     assert "omnigent login http://localhost:8000" in escalated
     assert "no longer a transient network blip" in escalated
     assert host._auth_retry_streak == _AUTH_REJECT_ESCALATE_ATTEMPTS
+
+
+@pytest.mark.skipif(not IS_POSIX, reason="SIGTERM handling is POSIX-only")
+async def test_sigterm_runs_the_clean_shutdown_path(tmp_path: Path) -> None:
+    """Regression: SIGTERM must reach run()'s cleanup, not kill the daemon.
+
+    ``omnigent host stop`` signals a backgrounded daemon with SIGTERM. Without
+    a handler Python exits where it stands, so the cleanup that terminates
+    tracked runners never ran and they were left to their own parent-death
+    watchdog.
+    """
+    host = _make_host_process()
+    host._zygote = None
+    serving = asyncio.Event()
+    aborted = asyncio.Event()
+    # run() force-drops the live tunnel on a shutdown signal, so stand in a
+    # connection whose transport records the abort.
+    host._ws = SimpleNamespace(  # type: ignore[assignment]
+        transport=SimpleNamespace(abort=aborted.set)
+    )
+
+    async def _hang() -> None:
+        serving.set()
+        await aborted.wait()
+        raise ConnectionClosedError(None, None)
+
+    async def _noop() -> None:
+        return None
+
+    proc = subprocess.Popen(
+        ["sleep", "60"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    host._runners["runner_sigterm"] = _RunnerHandle(proc=proc, log_path=tmp_path / "runner.log")
+
+    with (
+        patch.object(host, "_connect_and_serve", _hang),
+        patch.object(host, "_initialize_capabilities", _noop),
+        patch.object(host, "_orphan_reaper_loop", _noop),
+        patch("omnigent.host.connect.watch_for_resume", lambda _cb: _noop()),
+    ):
+        run_task = asyncio.create_task(host.run())
+        await asyncio.wait_for(serving.wait(), timeout=5.0)
+        # The handler is installed by now; sending the real signal proves the
+        # wiring rather than just the helper it calls.
+        assert host._shutdown_event is not None
+        os.kill(os.getpid(), signal.SIGTERM)
+        await asyncio.wait_for(run_task, timeout=10.0)
+
+    assert proc.poll() is not None, "SIGTERM must terminate tracked runners"
+    assert host._runners == {}
+
+
+@pytest.mark.skipif(not IS_POSIX, reason="SIGTERM handling is POSIX-only")
+async def test_shutdown_signal_cuts_short_the_reconnect_backoff(tmp_path: Path) -> None:
+    """A SIGTERM during reconnect backoff must not wait out the sleep."""
+    host = _make_host_process()
+    host._shutdown_event = asyncio.Event()
+
+    host._request_shutdown(signal.SIGTERM)
+
+    start = time.monotonic()
+    await host._sleep_before_reconnect(30.0)
+    assert time.monotonic() - start < 5.0, "backoff should return early on shutdown"
+    assert host._shutdown_requested is True