fix(test-infra): bind the smoke fixture server without a reverse-DNS lookup

The macOS Intel leg failed the fixture contract on every run while every
other platform passed. Removing the readiness fsync did not change it; the
hardened diagnostics named the real cause on the first remote run:
waited=30.0s, exit_status=alive, port_file=absent, staged_temp_files=none,
empty startup log -- the process was healthy but had never reached
publish_port.

http.server.HTTPServer.server_bind() resolves socket.getfqdn(host). On a
host whose resolver does not answer for the bind address that call blocks
for the resolver timeout, so the constructor never returns and no port is
ever published. The fixture now binds through a subclass that keeps the
threading server but skips the FQDN resolution, which only feeds CGI-style
variables this fixture never serves.

Proven both directions locally by forcing socket.getfqdn to hang: the
subclass publishes its port immediately, the stock server never does. That
forcing hook is kept as a permanent contract guard, so the reverse-DNS
dependency cannot return without turning the gate red on every platform
rather than on one runner nobody can reproduce.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel
2026-07-25 01:01:03 +02:00
parent 636dd3f92c
commit 0d46fffbaa
2 changed files with 79 additions and 3 deletions
+21 -1
View File
@@ -8,9 +8,29 @@ import functools
import http.server
import os
import pathlib
import socketserver
import tempfile
class FixtureHTTPServer(http.server.ThreadingHTTPServer):
"""Bind without the stdlib's reverse-DNS lookup.
http.server.HTTPServer.server_bind() resolves socket.getfqdn(host). On a
host whose resolver does not answer for the bind address that call blocks
for the resolver timeout, so the process stays alive and never reaches
publish_port -- observed on macOS Intel runners as waited=30.0s,
exit_status=alive, port_file=absent, empty startup log. The FQDN only
feeds CGI-style variables this fixture never serves, so binding without
it is both sufficient and immune to resolver behaviour.
"""
def server_bind(self) -> None:
socketserver.TCPServer.server_bind(self)
host, port = self.server_address[:2]
self.server_name = host
self.server_port = port
def publish_port(port_file: pathlib.Path, port: int) -> None:
"""Atomically publish the assigned port after the listening socket exists."""
port_file.parent.mkdir(parents=True, exist_ok=True)
@@ -46,7 +66,7 @@ def main() -> None:
http.server.SimpleHTTPRequestHandler,
directory=str(directory),
)
with http.server.ThreadingHTTPServer((args.bind, 0), handler) as server:
with FixtureHTTPServer((args.bind, 0), handler) as server:
publish_port(args.port_file, server.server_port)
print(
f"smoke fixture server: http://{args.bind}:{server.server_port} "
+58 -2
View File
@@ -51,8 +51,10 @@ helper_source = read(helper_relative)
# The server owns the ephemeral bind. A parent-side socket probe followed by
# python -m http.server would reintroduce the close/rebind race this guards.
require(
"ThreadingHTTPServer((args.bind, 0)" in helper_source,
"fixture server must bind port 0 itself and retain the listening socket",
"HTTPServer((args.bind, 0)" in helper_source
and "ThreadingHTTPServer)" in helper_source,
"fixture server must bind port 0 itself on a threading server and retain "
"the listening socket",
)
require(
"--port-file" in helper_source and "os.replace" in helper_source,
@@ -430,6 +432,60 @@ if helper.is_file():
process.wait(timeout=3)
failures.append("fixture server did not terminate promptly")
# Regression guard, by construction: binding must not depend on reverse DNS.
# http.server's default server_bind() resolves socket.getfqdn(), which blocks
# for the resolver timeout on hosts that do not answer for the bind address --
# the server stays alive and never publishes its port (macOS Intel). Force the
# resolver to hang and require the port anyway, so the dependency cannot
# return without turning this gate red on every platform.
if helper.is_file():
with tempfile.TemporaryDirectory(prefix="cbm-fixture-dns-") as dns_temp:
dns_root = pathlib.Path(dns_temp)
dns_fixture = dns_root / "fixture"
dns_fixture.mkdir()
(dns_fixture / "expected-artifact.txt").write_bytes(b"fixture-ok\n")
dns_port_file = dns_root / "port"
dns_wrapper = dns_root / "hang_reverse_dns.py"
dns_wrapper.write_text(
"import runpy, socket, sys, time\n"
"socket.getfqdn = lambda *a, **k: time.sleep(300)\n"
"sys.argv = ['smoke-fixture-server.py', '--directory', "
f"{str(dns_fixture)!r}, '--port-file', {str(dns_port_file)!r}]\n"
f"runpy.run_path({str(helper)!r}, run_name='__main__')\n",
encoding="utf-8",
)
dns_log = dns_root / "server.log"
with dns_log.open("wb") as dns_handle:
dns_process = subprocess.Popen(
[sys.executable, str(dns_wrapper)],
stdout=dns_handle,
stderr=subprocess.STDOUT,
)
try:
dns_port = 0
dns_deadline = time.monotonic() + 20
while time.monotonic() < dns_deadline:
if dns_port_file.is_file():
dns_text = dns_port_file.read_text(encoding="ascii").strip()
if dns_text:
dns_port = int(dns_text)
break
if dns_process.poll() is not None:
break
time.sleep(0.02)
require(
dns_port > 0,
"fixture server must bind without a reverse-DNS lookup "
"(hanging socket.getfqdn must not block port publication)",
)
finally:
dns_process.terminate()
try:
dns_process.wait(timeout=3)
except subprocess.TimeoutExpired:
dns_process.kill()
dns_process.wait(timeout=3)
if failures:
print("smoke fixture contract: FAIL", file=sys.stderr)
for failure in failures: