fix(render): two silent hangs in the turbo capture path (#132)
* fix(render): scope the /dev/shm scratch dir to the current user
The turbo path writes Chrome's raw BGRA tiles to a fixed
`/dev/shm/pixelrag_render/raw`. `/dev/shm` is shared by every user on the box,
so that directory belongs to whoever ran first — it lands at 0775 owned by them,
and every later user is denied write access to it.
Nothing about that surfaced. `mkdir(exist_ok=True)` still succeeds against the
existing directory; Chrome's write failure goes to its stderr, which this backend
sends to DEVNULL; and the resulting FileNotFoundError in every compression worker
was swallowed by a bare `except Exception: pass`. A second user on the same host
therefore got "Done: 13 tiles", a manifest listing 13 tiles with
`complete: true`, and an output directory holding nothing but `tiles.json`.
Scope the scratch path per uid, refuse to start when it isn't writable, and count
and log compression failures instead of dropping them.
Measured on a shared box, as the non-owning user:
before: Done: 13 tiles -> 0 images on disk
after: Done: 13 tiles -> 13 images on disk (2,611,261 bytes)
* fix(render): start the compression pool with spawn, not fork
The pool is created from a thread while the capture side already runs its own
threads. Forking a multi-threaded process can leave the child holding a lock
that was never released, and the child then blocks forever — CPython has warned
about this since 3.12.
A single-threaded caller forks cleanly, which is why this survived: `pixelshot`
from a shell works every time. Under a threaded host it hangs with no output and
no error. The full test suite is one such host: `tests/test_render.py` passes in
5.8s on its own and hangs indefinitely when the async tests run first.
Switching to spawn also means the pool initializer has to survive pickling, so
the core-affinity helper moves from a closure to a module-level function.
before: full suite exceeded a 400s timeout, no result
after: 77 passed, 2 skipped in 12.17s
* style: sort the local imports ruff I001 flagged
This commit is contained in:
@@ -99,6 +99,42 @@ _WAIT_FONTS_IMGS = (
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pin_to_cores(cores) -> None:
|
||||
"""Pool initializer: keep compression workers off the capture cores.
|
||||
|
||||
Module level, not a closure, because the pool is started with "spawn" — the
|
||||
initializer has to survive pickling, which a nested function does not.
|
||||
"""
|
||||
try:
|
||||
os.sched_setaffinity(0, cores)
|
||||
except (OSError, AttributeError):
|
||||
pass # not Linux, or the affinity call is unavailable — harmless
|
||||
|
||||
|
||||
def _raw_scratch_dir() -> Path:
|
||||
"""Per-user scratch directory on /dev/shm for Chrome's raw BGRA dumps.
|
||||
|
||||
`/dev/shm` is shared by every user on the box, so a fixed path belongs to
|
||||
whoever ran first: the directory lands at 0775 owned by them, and for every
|
||||
later user `mkdir(exist_ok=True)` still succeeds while Chrome's write into it
|
||||
is denied. That combination is silent — capture reports success, the manifest
|
||||
claims N tiles, and not one image is written. Scoping the path to the current
|
||||
uid keeps users out of each other's way.
|
||||
|
||||
The writability check is here rather than at first write because the failure
|
||||
it catches surfaces inside Chrome, whose stderr this backend discards.
|
||||
"""
|
||||
d = Path(f"/dev/shm/pixelrag_render-{os.getuid()}/raw")
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
if not os.access(d, os.W_OK):
|
||||
raise RuntimeError(
|
||||
f"{d} is not writable by uid {os.getuid()}. Chrome writes raw tiles "
|
||||
"there; without write access every capture is silently lost. Remove "
|
||||
"the directory and re-run, or set the turbo path aside with turbo=False."
|
||||
)
|
||||
return d
|
||||
|
||||
|
||||
def compress_tile(raw_path: str, out_path: str, quality: int = 85) -> None:
|
||||
"""Read raw BGRA file, compress to JPEG, delete raw file.
|
||||
|
||||
@@ -285,8 +321,7 @@ async def _run_render(
|
||||
jpeg_quality: int,
|
||||
n_compressors: int,
|
||||
) -> dict:
|
||||
raw_dir = Path("/dev/shm/pixelrag_render/raw")
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
raw_dir = _raw_scratch_dir()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Semaphore: limit concurrent captures (CPU-bound) to n_workers // 2
|
||||
@@ -296,9 +331,9 @@ async def _run_render(
|
||||
# Compression: dedicated thread with its own multiprocessing.Pool.
|
||||
# Tiles are pushed to a thread-safe queue from capture workers.
|
||||
# The thread runs pool.starmap in batches, fully independent of asyncio.
|
||||
import multiprocessing
|
||||
import queue as _queue
|
||||
import threading
|
||||
from multiprocessing import Pool as MPPool
|
||||
|
||||
metrics = {
|
||||
"total_tiles": 0,
|
||||
@@ -312,15 +347,20 @@ async def _run_render(
|
||||
n_cpus = os.cpu_count() or 128
|
||||
compress_cores = set(range(max(0, n_cpus - n_compressors), n_cpus))
|
||||
|
||||
def _pool_init():
|
||||
try:
|
||||
os.sched_setaffinity(0, compress_cores)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _compressor_thread():
|
||||
pool = MPPool(processes=n_compressors, initializer=_pool_init)
|
||||
# Warm up: ensure all workers are forked and idle before capture starts
|
||||
# "spawn", not the platform default. This pool is created from a thread while
|
||||
# the capture side already runs its own threads, and forking a multi-threaded
|
||||
# process can deadlock the child on a lock that was held at fork time — CPython
|
||||
# warns about exactly this since 3.12. In a single-threaded caller the fork
|
||||
# happens to work, which is why it survived so long; under any threaded host
|
||||
# (a test session that ran async tests first, a web server calling into the
|
||||
# renderer) it hangs with no output and no error.
|
||||
pool = multiprocessing.get_context("spawn").Pool(
|
||||
processes=n_compressors,
|
||||
initializer=_pin_to_cores,
|
||||
initargs=(compress_cores,),
|
||||
)
|
||||
# Warm up: ensure all workers are up and idle before capture starts
|
||||
pool.map(int, range(n_compressors))
|
||||
async_results = []
|
||||
while True:
|
||||
@@ -328,12 +368,24 @@ async def _run_render(
|
||||
if item is None:
|
||||
break
|
||||
async_results.append(pool.apply_async(compress_tile, item))
|
||||
# Wait for all remaining
|
||||
# Wait for all remaining. A compression failure means that tile never
|
||||
# reached disk, so it has to be counted and logged: swallowing it let a
|
||||
# run report "Done: 13 tiles" with an empty output directory.
|
||||
failed = 0
|
||||
for ar in async_results:
|
||||
try:
|
||||
ar.get(timeout=60)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
if failed <= 3: # one line per failure is enough to diagnose
|
||||
logger.error("tile compression failed: %s: %s", type(e).__name__, e)
|
||||
if failed:
|
||||
logger.error(
|
||||
"%d/%d tiles failed to compress and were NOT written to disk",
|
||||
failed,
|
||||
len(async_results),
|
||||
)
|
||||
metrics["errors"] += failed
|
||||
pool.close()
|
||||
pool.join()
|
||||
compress_done.set()
|
||||
|
||||
Reference in New Issue
Block a user