Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5c938a1801 | |||
| e34d2426a4 | |||
| 8f1c268b44 | |||
| f2e7299643 | |||
| 2fe8bf0051 | |||
| dc5ce0ae7f | |||
| 796b3def20 | |||
| f198528373 | |||
| 94b7faa985 |
+71
-30
@@ -233,11 +233,14 @@ _DAEMON_RECONNECT_GRACE_S = 5.0
|
||||
_DAEMON_REUSE_MIN_AGE_S = 6.0
|
||||
|
||||
# How long uvicorn waits for active connections (WebSocket, SSE) after
|
||||
# SIGTERM before force-closing them. 30 s gives in-flight responses time
|
||||
# to drain while still guaranteeing the port is released promptly.
|
||||
# SIGTERM before force-closing them. SSE streams signal themselves via
|
||||
# session_stream.shutdown_all() in _ShutdownSignalingServer.shutdown(),
|
||||
# so the main remaining consumers of this window are WebSocket tunnels
|
||||
# that need a moment to drain. 5 s is enough for a clean tunnel teardown
|
||||
# while keeping Ctrl-C feeling instant.
|
||||
# Overridable via OMNIGENT_SERVER_SHUTDOWN_TIMEOUT_S for deployments that
|
||||
# need a longer drain window (e.g. large file uploads).
|
||||
_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_S_DEFAULT = 30
|
||||
_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_S_DEFAULT = 5
|
||||
_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_S = int(
|
||||
os.environ.get(
|
||||
"OMNIGENT_SERVER_SHUTDOWN_TIMEOUT_S",
|
||||
@@ -2972,6 +2975,7 @@ def server(
|
||||
port = _picked
|
||||
|
||||
import uvicorn
|
||||
import uvicorn.server
|
||||
|
||||
from omnigent.runner.transports.ws_tunnel.limits import (
|
||||
RUNNER_TUNNEL_MAX_MESSAGE_BYTES,
|
||||
@@ -3220,34 +3224,71 @@ def server(
|
||||
# this foreground server instead of tearing it down on a spurious
|
||||
# sig mismatch.
|
||||
register_local_server(port)
|
||||
|
||||
class _ShutdownSignalingServer(uvicorn.server.Server):
|
||||
"""uvicorn.Server that signals active SSE subscribers before the
|
||||
graceful-shutdown wait starts.
|
||||
|
||||
uvicorn calls ``Server.shutdown()`` in this order:
|
||||
1. close listening sockets / call connection.shutdown()
|
||||
2. ``asyncio.wait_for(_wait_tasks_to_complete(), timeout=…)``
|
||||
3. force-cancel remaining tasks on timeout
|
||||
4. run the ASGI lifespan shutdown handler
|
||||
|
||||
The ASGI lifespan ``finally`` block runs at step 4 — too late. SSE
|
||||
generators waiting on a heartbeat tick are already force-cancelled by
|
||||
step 3, which produces spurious ``CancelledError`` tracebacks.
|
||||
Overriding here lets us drain SSE streams before step 2 so they exit
|
||||
cleanly within the graceful window.
|
||||
"""
|
||||
|
||||
async def shutdown(self, sockets=None) -> None: # type: ignore[override]
|
||||
import asyncio as _asyncio
|
||||
|
||||
from omnigent.runtime import session_stream as _session_stream
|
||||
|
||||
_session_stream.shutdown_all()
|
||||
# Yield to the event loop so generators can consume _DONE,
|
||||
# flush their final "data: [DONE]\n\n" chunk, and exit before
|
||||
# super().shutdown() calls connection.shutdown() / transport.close().
|
||||
# Without this pause the generators write to an already-closing
|
||||
# transport, leaving connections open past the graceful window.
|
||||
await _asyncio.sleep(0)
|
||||
await super().shutdown(sockets)
|
||||
|
||||
_config = uvicorn.Config(
|
||||
app,
|
||||
host=host,
|
||||
port=port,
|
||||
log_config=_server_uvicorn_log_config(),
|
||||
ws_max_size=RUNNER_TUNNEL_MAX_MESSAGE_BYTES,
|
||||
# Server side of the runner/host tunnels' protocol keepalive, aligned
|
||||
# to the 90 s app-level budget instead of uvicorn's 20 s default that
|
||||
# drops a busy-but-healthy tunnel with 1011 — issue #1116.
|
||||
#
|
||||
# uvicorn's ws_ping_* is server-global (no per-route override), so this
|
||||
# 30 s/90 s budget also applies to the app's other WebSocket routes —
|
||||
# /v1/sessions/updates (browser stream) and .../terminals/{id}/attach.
|
||||
# Deliberate and acceptable: for an IDLE such socket the protocol
|
||||
# PING/PONG is the only half-open detector (the sessions-updates
|
||||
# heartbeat is a server->client send, and an idle terminal has no
|
||||
# traffic), so widening it means a dead idle browser/terminal socket is
|
||||
# reaped at worst ~120 s (30 s interval + 90 s timeout) instead of
|
||||
# ~40 s — a slightly later half-open cleanup (e.g. the out-of-process
|
||||
# terminal-attach proxy holds its runner socket + tmux child ~80 s
|
||||
# longer), bounded and eventually reaped, not a leak or correctness
|
||||
# change. The tunnels are the sockets that actually need the looser
|
||||
# budget (issue #1116).
|
||||
ws_ping_interval=TUNNEL_KEEPALIVE_PING_INTERVAL_S,
|
||||
ws_ping_timeout=TUNNEL_KEEPALIVE_PING_TIMEOUT_S,
|
||||
timeout_graceful_shutdown=_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_S,
|
||||
)
|
||||
try:
|
||||
uvicorn.run(
|
||||
app,
|
||||
host=host,
|
||||
port=port,
|
||||
log_config=_server_uvicorn_log_config(),
|
||||
ws_max_size=RUNNER_TUNNEL_MAX_MESSAGE_BYTES,
|
||||
# Server side of the runner/host tunnels' protocol keepalive, aligned
|
||||
# to the 90 s app-level budget instead of uvicorn's 20 s default that
|
||||
# drops a busy-but-healthy tunnel with 1011 — issue #1116.
|
||||
#
|
||||
# uvicorn's ws_ping_* is server-global (no per-route override), so this
|
||||
# 30 s/90 s budget also applies to the app's other WebSocket routes —
|
||||
# /v1/sessions/updates (browser stream) and .../terminals/{id}/attach.
|
||||
# Deliberate and acceptable: for an IDLE such socket the protocol
|
||||
# PING/PONG is the only half-open detector (the sessions-updates
|
||||
# heartbeat is a server->client send, and an idle terminal has no
|
||||
# traffic), so widening it means a dead idle browser/terminal socket is
|
||||
# reaped at worst ~120 s (30 s interval + 90 s timeout) instead of
|
||||
# ~40 s — a slightly later half-open cleanup (e.g. the out-of-process
|
||||
# terminal-attach proxy holds its runner socket + tmux child ~80 s
|
||||
# longer), bounded and eventually reaped, not a leak or correctness
|
||||
# change. The tunnels are the sockets that actually need the looser
|
||||
# budget (issue #1116).
|
||||
ws_ping_interval=TUNNEL_KEEPALIVE_PING_INTERVAL_S,
|
||||
ws_ping_timeout=TUNNEL_KEEPALIVE_PING_TIMEOUT_S,
|
||||
timeout_graceful_shutdown=_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_S,
|
||||
)
|
||||
_ShutdownSignalingServer(_config).run()
|
||||
except KeyboardInterrupt:
|
||||
# uvicorn.run() swallows KeyboardInterrupt; match that behaviour so
|
||||
# a Ctrl-C exit doesn't print Click's "Aborted!" or exit non-zero.
|
||||
pass
|
||||
finally:
|
||||
if _is_canonical_local_server:
|
||||
clear_local_server_record()
|
||||
|
||||
@@ -708,6 +708,11 @@ def pick_local_port(preferred: int = _DEFAULT_LOCAL_PORT) -> int:
|
||||
import socket
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
# SO_REUSEADDR mirrors what uvicorn sets when it binds. Without
|
||||
# it, a fast server restart sees EADDRINUSE on macOS/BSD because
|
||||
# recently closed connections are still in TIME_WAIT even though
|
||||
# the listening socket is gone and uvicorn could successfully bind.
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
s.bind(("127.0.0.1", preferred))
|
||||
except OSError:
|
||||
|
||||
@@ -125,6 +125,23 @@ def close(conversation_id: str) -> None:
|
||||
loop.call_soon_threadsafe(queue.put_nowait, _DONE)
|
||||
|
||||
|
||||
def shutdown_all() -> None:
|
||||
"""Signal all active subscribers across every conversation to exit.
|
||||
|
||||
Broadcasts the end-of-stream sentinel to every queued subscriber so
|
||||
SSE generators return at their next iteration without waiting for a
|
||||
heartbeat timeout or forced task cancellation. Called from the asyncio
|
||||
event loop (``_ShutdownSignalingServer.shutdown`` in ``cli.py``) before
|
||||
uvicorn's graceful-shutdown wait starts, so streams drain within the
|
||||
window rather than being force-cancelled. Sync callers should use
|
||||
:func:`close` per-conversation instead.
|
||||
"""
|
||||
with _lock:
|
||||
all_subs = [entry for subs in _subscribers.values() for entry in subs]
|
||||
for queue, _ in all_subs:
|
||||
queue.put_nowait(_DONE)
|
||||
|
||||
|
||||
async def subscribe(
|
||||
conversation_id: str,
|
||||
*,
|
||||
|
||||
+28
-25
@@ -1151,6 +1151,7 @@ def test_server_command_reads_tunnel_token_and_does_not_spawn_runner(
|
||||
:returns: None.
|
||||
"""
|
||||
import uvicorn
|
||||
import uvicorn.server
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
@@ -1165,22 +1166,27 @@ def test_server_command_reads_tunnel_token_and_does_not_spawn_runner(
|
||||
captured["create_app_kwargs"] = kwargs
|
||||
return _original_create_app(**kwargs)
|
||||
|
||||
def _fake_uvicorn_run(app: Any, **kwargs: Any) -> None:
|
||||
"""Skip the blocking server loop.
|
||||
def _fake_server_run(self: Any) -> None:
|
||||
"""Skip the blocking server loop; capture config as flat kwargs dict.
|
||||
|
||||
:param app: FastAPI app instance built by ``create_app``.
|
||||
:param kwargs: Uvicorn options (host, port).
|
||||
:param self: The uvicorn Server instance whose config holds all options.
|
||||
:returns: None.
|
||||
"""
|
||||
del app
|
||||
captured["uvicorn_kwargs"] = kwargs
|
||||
captured["uvicorn_kwargs"] = {
|
||||
"ws_max_size": self.config.ws_max_size,
|
||||
"ws_ping_interval": self.config.ws_ping_interval,
|
||||
"ws_ping_timeout": self.config.ws_ping_timeout,
|
||||
"log_config": self.config.log_config,
|
||||
"port": self.config.port,
|
||||
"host": self.config.host,
|
||||
}
|
||||
captured["uvicorn_called"] = True
|
||||
|
||||
from omnigent.server import app as app_module
|
||||
|
||||
_original_create_app = app_module.create_app
|
||||
monkeypatch.setattr(app_module, "create_app", _spy_create_app)
|
||||
monkeypatch.setattr(uvicorn, "run", _fake_uvicorn_run)
|
||||
monkeypatch.setattr(uvicorn.server.Server, "run", _fake_server_run)
|
||||
monkeypatch.setenv("OMNIGENT_RUNNER_TUNNEL_TOKEN", "test-tunnel-token-abc")
|
||||
|
||||
# On a loopback bind the `server` command reuses an already-running
|
||||
@@ -1248,6 +1254,7 @@ def test_server_with_explicit_db_does_not_reuse_canonical_server(
|
||||
shared pidfile.
|
||||
"""
|
||||
import uvicorn
|
||||
import uvicorn.server
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
_original_create_app = None
|
||||
@@ -1261,22 +1268,20 @@ def test_server_with_explicit_db_does_not_reuse_canonical_server(
|
||||
captured["create_app_kwargs"] = kwargs
|
||||
return _original_create_app(**kwargs)
|
||||
|
||||
def _fake_uvicorn_run(app: Any, **kwargs: Any) -> None:
|
||||
def _fake_server_run(self: Any) -> None:
|
||||
"""Skip the blocking server loop, record that it was called.
|
||||
|
||||
:param app: FastAPI app built by ``create_app``.
|
||||
:param kwargs: Uvicorn options (host, port, ...).
|
||||
:param self: The uvicorn Server instance.
|
||||
:returns: None.
|
||||
"""
|
||||
del app
|
||||
captured["uvicorn_kwargs"] = kwargs
|
||||
captured["uvicorn_kwargs"] = {"port": self.config.port}
|
||||
captured["uvicorn_called"] = True
|
||||
|
||||
from omnigent.server import app as app_module
|
||||
|
||||
_original_create_app = app_module.create_app
|
||||
monkeypatch.setattr(app_module, "create_app", _spy_create_app)
|
||||
monkeypatch.setattr(uvicorn, "run", _fake_uvicorn_run)
|
||||
monkeypatch.setattr(uvicorn.server.Server, "run", _fake_server_run)
|
||||
|
||||
# A healthy canonical server EXISTS. A bare `omnigent server` would
|
||||
# reuse it; an explicit-DB server must ignore it. register/clear must
|
||||
@@ -1334,19 +1339,18 @@ def test_server_with_explicit_port_does_not_check_canonical_server(
|
||||
:returns: None.
|
||||
"""
|
||||
import uvicorn
|
||||
import uvicorn.server
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _fake_uvicorn_run(app: Any, **kwargs: Any) -> None:
|
||||
def _fake_server_run(self: Any) -> None:
|
||||
"""
|
||||
Skip the blocking server loop.
|
||||
|
||||
:param app: FastAPI app instance built by ``create_app``.
|
||||
:param kwargs: Uvicorn options (host, port).
|
||||
:param self: The uvicorn Server instance.
|
||||
:returns: None.
|
||||
"""
|
||||
del app
|
||||
captured["uvicorn_kwargs"] = kwargs
|
||||
captured["uvicorn_kwargs"] = {"port": self.config.port}
|
||||
|
||||
def _must_not_check_existing() -> str | None:
|
||||
"""
|
||||
@@ -1367,7 +1371,7 @@ def test_server_with_explicit_port_does_not_check_canonical_server(
|
||||
|
||||
from omnigent.host import local_server as _local_server_mod
|
||||
|
||||
monkeypatch.setattr(uvicorn, "run", _fake_uvicorn_run)
|
||||
monkeypatch.setattr(uvicorn.server.Server, "run", _fake_server_run)
|
||||
monkeypatch.setattr(_local_server_mod, "local_server_url_if_healthy", _must_not_check_existing)
|
||||
monkeypatch.setattr(_local_server_mod, "register_local_server", _must_not_touch_pidfile)
|
||||
monkeypatch.setattr(_local_server_mod, "clear_local_server_record", _must_not_touch_pidfile)
|
||||
@@ -1447,19 +1451,18 @@ def test_server_command_explicit_port_uses_bind_probe_not_connect_probe(
|
||||
import socket
|
||||
|
||||
import uvicorn
|
||||
import uvicorn.server
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _fake_uvicorn_run(app: Any, **kwargs: Any) -> None:
|
||||
def _fake_server_run(self: Any) -> None:
|
||||
"""
|
||||
Skip the blocking server loop.
|
||||
|
||||
:param app: FastAPI app instance built by ``create_app``.
|
||||
:param kwargs: Uvicorn options (host, port).
|
||||
:param self: The uvicorn Server instance.
|
||||
:returns: None.
|
||||
"""
|
||||
del app
|
||||
captured["uvicorn_kwargs"] = kwargs
|
||||
captured["uvicorn_kwargs"] = {"port": self.config.port}
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
@@ -1468,7 +1471,7 @@ def test_server_command_explicit_port_uses_bind_probe_not_connect_probe(
|
||||
with pytest.raises(OSError):
|
||||
socket.create_connection(("127.0.0.1", port), timeout=0.01)
|
||||
|
||||
monkeypatch.setattr(uvicorn, "run", _fake_uvicorn_run)
|
||||
monkeypatch.setattr(uvicorn.server.Server, "run", _fake_server_run)
|
||||
monkeypatch.setenv("OMNIGENT_AUTH_ENABLED", "0")
|
||||
|
||||
db_path = tmp_path / "chat.db"
|
||||
|
||||
Reference in New Issue
Block a user