feat: background import queue, playlist import, and queue management (#350)

Imports run through an explicit serial queue: queue several tracks, a playlist, or a folder of files and keep using StemDeck while they extract. Adds a Queue view with per-job cancel and drag-to-reorder, and a restored queue waits for the user to start it.

Closes #344, #345, #346, #347, #348, #349, #351, #352, #353.
This commit is contained in:
Tha.Les
2026-08-11 21:31:29 +01:00
committed by GitHub
parent a70e9cd53e
commit afe871ce81
34 changed files with 3774 additions and 169 deletions
+8 -3
View File
@@ -215,7 +215,7 @@ Install prerequisites:
```powershell
git clone https://github.com/stemdeckapp/stemdeck stemdeck; cd stemdeck
uv sync
uv run uvicorn app.main:app --host 127.0.0.1 --port 8000
uv run uvicorn app.main:app --host 127.0.0.1 --port 8000 --timeout-graceful-shutdown 5
```
Open <http://localhost:8000>.
@@ -227,7 +227,7 @@ Open <http://localhost:8000>.
```powershell
uv pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
$env:STEMDECK_DEMUCS_DEVICE = "cuda"
uv run uvicorn app.main:app --host 127.0.0.1 --port 8000
uv run uvicorn app.main:app --host 127.0.0.1 --port 8000 --timeout-graceful-shutdown 5
```
---
@@ -237,9 +237,14 @@ uv run uvicorn app.main:app --host 127.0.0.1 --port 8000
```sh
git clone https://github.com/stemdeckapp/stemdeck stemdeck && cd stemdeck
uv sync
uv run uvicorn app.main:app --reload
uv run uvicorn app.main:app --reload --timeout-graceful-shutdown 5
```
> `--timeout-graceful-shutdown` bounds how long uvicorn waits for open
> connections when you stop it. StemDeck keeps a long-lived SSE stream open
> for the import queue while a browser tab is on the app, so without it
> Ctrl-C waits for that stream instead of exiting.
#### Docker
```sh
+17 -6
View File
@@ -23,22 +23,33 @@ _MAX_SSE_CONNECTIONS = 200
_sse_active = 0
def claim_sse_slot() -> None:
"""Reserve one of the shared connection slots, or 503. Split out so the
queue stream in app/api/queue.py shares one budget with this one rather
than each getting its own."""
global _sse_active
if _sse_active >= _MAX_SSE_CONNECTIONS:
raise HTTPException(status_code=503, detail="too many concurrent streams")
_sse_active += 1
def release_sse_slot() -> None:
global _sse_active
_sse_active -= 1
@router.get("/jobs/{job_id}/events")
async def job_events(job_id: str) -> StreamingResponse:
"""Server-Sent Events stream of job state updates. Closes when the job
reaches a terminal status (done, error, cancelled) or after 4 hours."""
global _sse_active
if not JOB_ID_RE.match(job_id):
raise HTTPException(status_code=404, detail="job not found")
job = registry_get(job_id)
if job is None:
raise HTTPException(status_code=404, detail="job not found")
if _sse_active >= _MAX_SSE_CONNECTIONS:
raise HTTPException(status_code=503, detail="too many concurrent streams")
_sse_active += 1
claim_sse_slot()
async def stream() -> AsyncIterator[str]:
global _sse_active
try:
last_v = -1
keepalive_at = 0
@@ -71,7 +82,7 @@ async def job_events(job_id: str) -> StreamingResponse:
keepalive_at = 0
await asyncio.sleep(0.2)
finally:
_sse_active -= 1
release_sse_slot()
return StreamingResponse(
stream(),
+50 -24
View File
@@ -14,16 +14,24 @@ from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import JSONResponse, Response
from pydantic import BaseModel, Field, field_validator
from app.core.config import JOB_ID_RE, JOBS_DIR, MAX_PENDING_JOBS, STEM_NAMES, ffprobe_executable
from app.core.models import Job
from app.core.config import (
JOB_ID_RE,
JOBS_DIR,
MAX_PENDING_UPLOAD_JOBS,
MAX_PENDING_URL_JOBS,
STEM_NAMES,
ffprobe_executable,
)
from app.core.models import Job, _set
from app.core.registry import all_jobs as registry_all_jobs
from app.core.registry import get as registry_get
from app.core.registry import get_proc as registry_get_proc
from app.core.registry import pending_count as registry_pending_count
from app.core.registry import persist as registry_persist
from app.core.registry import register_if_capacity as registry_register_if_capacity
from app.core.registry import remove as registry_remove
from app.core.settings import get_max_duration_sec
from app.pipeline import run_local_pipeline, run_pipeline
from app.pipeline import jobqueue
from app.pipeline.download import InvalidYouTubeURL, validate_youtube_url
router = APIRouter(tags=["jobs"])
@@ -33,6 +41,15 @@ _ALLOWED_EXTS = frozenset((".mp3", ".wav", ".flac", ".mp4", ".m4a", ".ogg", ".op
_MAX_UPLOAD_BYTES = 400 * 1024 * 1024 # 400 MB
_WS_RE = re.compile(r"\s+")
# Now that imports queue instead of running immediately, a full queue is a
# capacity statement the user can act on, not a transient "try again".
_URL_QUEUE_FULL_DETAIL = (
f"Queue is full ({MAX_PENDING_URL_JOBS} links waiting) - cancel a job or wait"
)
_UPLOAD_QUEUE_FULL_DETAIL = (
f"Upload queue is full ({MAX_PENDING_UPLOAD_JOBS} waiting) - cancel a job or wait"
)
def _sanitize_title(filename: str) -> str:
"""Strip extension, normalize whitespace, cap at 120 chars."""
@@ -90,14 +107,6 @@ def _rmtree_job(job_id: str) -> None:
logger.warning("failed to remove job dir %s", job_dir, exc_info=True)
def _task_error_cb(task: asyncio.Task) -> None:
if task.cancelled():
return
exc = task.exception()
if exc is not None:
logger.error("pipeline task raised unhandled exception", exc_info=exc)
class JobRequest(BaseModel):
url: str
# Subset of stems to include in the post-processing "selected mix"
@@ -138,18 +147,20 @@ async def _create_youtube_job(request: Request) -> dict[str, str]:
selected = list(STEM_NAMES)
job = Job(id=uuid.uuid4().hex[:12], selected_stems=selected, source_url=url)
if not registry_register_if_capacity(job, MAX_PENDING_JOBS):
raise HTTPException(status_code=503, detail="Server busy, please try again later")
task = asyncio.create_task(run_pipeline(job, url, JOBS_DIR))
task.add_done_callback(_task_error_cb)
if not registry_register_if_capacity(job, MAX_PENDING_URL_JOBS):
raise HTTPException(status_code=503, detail=_URL_QUEUE_FULL_DETAIL)
jobqueue.enqueue(job.id)
registry_persist(JOBS_DIR)
return {"job_id": job.id}
async def _create_local_job(request: Request) -> dict[str, str]:
# Fast pre-check: if already at capacity, reject before touching disk.
# The real atomic check happens in register_if_capacity after the upload.
if sum(1 for j in registry_all_jobs().values() if j.status == "queued") >= MAX_PENDING_JOBS:
raise HTTPException(status_code=503, detail="Server busy, please try again later")
# Only other uploads count here: a queue full of links costs no disk and
# must not block a file import.
if registry_pending_count(uploads=True) >= MAX_PENDING_UPLOAD_JOBS:
raise HTTPException(status_code=503, detail=_UPLOAD_QUEUE_FULL_DETAIL)
# Quick pre-check on Content-Length to fail fast for obviously oversized
# uploads without buffering the whole body first.
@@ -228,11 +239,11 @@ async def _create_local_job(request: Request) -> dict[str, str]:
duration_sec=duration,
source_url=local_source_url,
)
if not registry_register_if_capacity(job, MAX_PENDING_JOBS):
if not registry_register_if_capacity(job, MAX_PENDING_UPLOAD_JOBS):
shutil.rmtree(job_dir, ignore_errors=True)
raise HTTPException(status_code=503, detail="Server busy, please try again later")
task = asyncio.create_task(run_local_pipeline(job, source_path, JOBS_DIR))
task.add_done_callback(_task_error_cb)
raise HTTPException(status_code=503, detail=_UPLOAD_QUEUE_FULL_DETAIL)
jobqueue.enqueue(job.id)
registry_persist(JOBS_DIR)
return {"job_id": job.id}
@@ -264,9 +275,24 @@ def cancel_job(job_id: str) -> dict:
if job.status in ("done", "error", "cancelled"):
return job.to_state()
job.cancel_requested = True
proc = registry_get_proc(job_id)
if proc is not None and proc.poll() is None:
proc.terminate()
# Still waiting: the worker will never pick it up, so finalise it here.
# Previously a queued job only honoured cancel once its turn arrived, kept
# occupying a capacity slot until then, and a queued upload held its source
# file (up to 400 MB) for the whole wait.
if jobqueue.discard(job_id):
_set(job, status="cancelled", stage="Cancelled")
jobqueue.cleanup_job_dir(job_id)
registry_persist(JOBS_DIR)
return job.to_state()
# Only the running job owns the shared demucs worker. Terminating on any
# other id would kill someone else's separation if a stale set_proc entry
# ever survived -- cheap insurance now that many job ids are live at once.
if job_id == jobqueue.running_id():
proc = registry_get_proc(job_id)
if proc is not None and proc.poll() is None:
proc.terminate()
return job.to_state()
+159
View File
@@ -0,0 +1,159 @@
"""Playlist import: expand a playlist URL into one queued job per track.
Two steps on purpose. Expansion is a network round trip whose result the user
has to agree to -- "this is 47 tracks, still want it?" -- and a single endpoint
would either queue 47 jobs with no warning or make the client re-post a list of
URLs it was handed. The client never supplies the track list: preview and
create both expand server-side, so nothing outside the SSRF allowlist can be
smuggled into the pipeline between the two calls.
"""
from __future__ import annotations
import asyncio
import logging
import uuid
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel
from app.core.config import JOBS_DIR, MAX_PENDING_URL_JOBS, STEM_NAMES
from app.core.models import Job
from app.core.registry import pending_count as registry_pending_count
from app.core.registry import persist as registry_persist
from app.core.registry import register_if_capacity as registry_register_if_capacity
from app.core.settings import get_max_duration_sec, get_playlist_max_items
from app.pipeline import jobqueue
from app.pipeline.download import InvalidPlaylistURL, expand_playlist
logger = logging.getLogger("stemdeck.api")
router = APIRouter(tags=["playlist"])
class PlaylistRequest(BaseModel):
url: str
stems: list[str] | None = None
def _capacity_left() -> int:
# Playlist tracks are links, so only other waiting links count against
# them -- a backlog of file uploads is bounded separately and holds disk
# rather than a registry record.
return max(0, MAX_PENDING_URL_JOBS - registry_pending_count(uploads=False))
async def _expand(url: str) -> dict[str, Any]:
"""yt-dlp is blocking and a large playlist takes seconds. Off the event loop
it goes, or every SSE stream (including the running job's progress) stalls
for the duration."""
try:
return await asyncio.to_thread(expand_playlist, url, get_playlist_max_items())
except InvalidPlaylistURL as e:
raise HTTPException(status_code=422, detail=str(e)) from e
except Exception as e:
logger.exception("playlist expansion failed")
raise HTTPException(status_code=502, detail="Could not read that playlist") from e
def _partition(items: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]:
"""Split out tracks that are longer than the configured limit. They would be
accepted here and then fail one by one in the pipeline, so the dialog is a
better place to say so."""
max_duration = get_max_duration_sec()
cap = get_playlist_max_items()
keep, too_long = [], 0
for item in items:
duration = item.get("duration")
if isinstance(duration, int | float) and duration > max_duration:
too_long += 1
continue
keep.append(item)
return keep[:cap], too_long
@router.post("/preview")
async def preview_playlist(request: Request) -> dict[str, Any]:
"""Expand a playlist and report what importing it would do. Creates nothing."""
try:
body = await request.json()
except Exception as e:
raise HTTPException(status_code=422, detail=f"Invalid JSON: {e}") from e
try:
payload = PlaylistRequest(**body)
except Exception as e:
raise HTTPException(status_code=422, detail=str(e)) from e
result = await _expand(payload.url)
items, too_long = _partition(result["items"])
capacity = _capacity_left()
return {
"playlist_title": result["playlist_title"],
"total_found": len(result["items"]) + result["unavailable"],
"will_queue": min(len(items), capacity),
"skipped_unavailable": result["unavailable"],
"skipped_too_long": too_long,
"capacity_left": capacity,
"cap": get_playlist_max_items(),
# The playlist has more tracks than the cap allows us to look at, so
# total_found is a floor, not the real total.
"truncated": result["truncated"],
"items": [{"title": i["title"], "duration": i["duration"]} for i in items],
}
@router.post("")
async def create_playlist_jobs(request: Request) -> dict[str, Any]:
"""Queue one job per playlist track, in playlist order.
Fills whatever capacity is available rather than failing the whole import:
the preview already told the user how many would be queued, and a partial
fill with an honest count beats a 503 after they agreed to it.
"""
try:
body = await request.json()
except Exception as e:
raise HTTPException(status_code=422, detail=f"Invalid JSON: {e}") from e
try:
payload = PlaylistRequest(**body)
except Exception as e:
raise HTTPException(status_code=422, detail=str(e)) from e
selected = [s for s in payload.stems if s in STEM_NAMES] if payload.stems else list(STEM_NAMES)
if not selected:
selected = list(STEM_NAMES)
result = await _expand(payload.url)
items, too_long = _partition(result["items"])
if not items:
raise HTTPException(status_code=422, detail="No importable tracks in that playlist")
if _capacity_left() == 0:
raise HTTPException(status_code=503, detail="Queue is full - wait or cancel a job")
created: list[dict[str, Any]] = []
for item in items:
job = Job(
id=uuid.uuid4().hex[:12],
selected_stems=list(selected),
source_url=item["url"],
# Pre-filled from the flat metadata so queue rows carry real names
# immediately, instead of a URL until each download starts.
title=item["title"] or None,
thumbnail=item.get("thumbnail"),
)
if not registry_register_if_capacity(job, MAX_PENDING_URL_JOBS):
break # queue filled up mid-loop; report what did land
jobqueue.enqueue(job.id)
created.append({"job_id": job.id, "title": job.title or "", "source_url": item["url"]})
registry_persist(JOBS_DIR)
return {
"playlist_title": result["playlist_title"],
"jobs": created,
"queued": len(created),
"skipped_unavailable": result["unavailable"],
"skipped_too_long": too_long,
"skipped_no_capacity": len(items) - len(created),
}
+165
View File
@@ -0,0 +1,165 @@
"""Aggregate view of the import queue.
One stream for every waiting job rather than one per job. A Map of per-job
EventSources would be simpler on the client, but browsers cap concurrent
connections per origin at around six on HTTP/1.1, and the studio already spends
most of that budget fetching stem WAVs -- twenty queue streams would starve
audio loading long before hitting the server-side connection cap.
The payload is deliberately compact (to_queue_state, not to_state). The
foreground import keeps its own /api/jobs/{id}/events stream, which is what
carries the full completion state the studio needs.
"""
from __future__ import annotations
import asyncio
import json
from collections.abc import AsyncIterator
from typing import Any
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from app.api.events import _MAX_SSE_SECONDS, claim_sse_slot, release_sse_slot
from app.core.config import JOB_ID_RE, JOBS_DIR, MAX_PENDING_UPLOAD_JOBS, MAX_PENDING_URL_JOBS
from app.core.registry import get as registry_get
from app.core.registry import pending_count as registry_pending_count
from app.core.registry import persist as registry_persist
from app.pipeline import jobqueue
router = APIRouter(tags=["queue"])
def _snapshot() -> dict[str, Any]:
"""Current queue as the client sees it. Position is derived here rather than
stored on the Job: it changes for every waiting job whenever the head is
dequeued, so writing it through _set() would bump N versions and wake N
streams for a value that is just a list index."""
running_id, waiting = jobqueue.snapshot()
running = None
if running_id is not None:
job = registry_get(running_id)
if job is not None:
running = job.to_queue_state()
queued = []
for position, job_id in enumerate(waiting):
job = registry_get(job_id)
if job is None:
continue
rec = job.to_queue_state()
rec["position"] = position
queued.append(rec)
# Capacity is per kind: a waiting upload holds its source file on disk, a
# waiting link holds nothing, so they are bounded separately.
uploads_pending = registry_pending_count(uploads=True)
urls_pending = registry_pending_count(uploads=False)
return {
"running": running,
"queued": queued,
# True when jobs were restored from a previous session and are waiting
# for the user to start them.
"paused": jobqueue.is_paused(),
"max_pending_uploads": MAX_PENDING_UPLOAD_JOBS,
"max_pending_urls": MAX_PENDING_URL_JOBS,
"capacity_left_uploads": max(0, MAX_PENDING_UPLOAD_JOBS - uploads_pending),
"capacity_left_urls": max(0, MAX_PENDING_URL_JOBS - urls_pending),
}
def _fingerprint() -> tuple:
"""Cheap change detector: the ids in order plus each job's version counter,
which _set() already bumps on every field write."""
running_id, waiting = jobqueue.snapshot()
ids = ([running_id] if running_id else []) + waiting
out: list[object] = [jobqueue.is_paused()]
for job_id in ids:
job = registry_get(job_id)
out.append((job_id, job.version if job is not None else -1))
return tuple(out)
@router.post("/start")
def start_queue() -> dict[str, Any]:
"""Begin working through a queue restored from a previous session."""
jobqueue.resume()
return _snapshot()
class ReorderRequest(BaseModel):
job_id: str
# The job it should sit directly after. None means "move to the front",
# which is also what a Move to top control sends.
after: str | None = None
@router.post("/reorder")
def reorder_queue(payload: ReorderRequest) -> dict[str, Any]:
"""Move a waiting job. Returns the resulting queue, so a client whose drag
raced a job finishing re-syncs from the answer instead of guessing."""
if not JOB_ID_RE.match(payload.job_id):
raise HTTPException(status_code=404, detail="job not found")
if payload.after is not None and not JOB_ID_RE.match(payload.after):
raise HTTPException(status_code=404, detail="job not found")
if payload.job_id == payload.after:
raise HTTPException(status_code=422, detail="a job cannot follow itself")
if not jobqueue.reorder(payload.job_id, payload.after):
# Already started or finished. Not an error the user can act on -- the
# snapshot below tells the client what is true now.
raise HTTPException(status_code=409, detail="that job is no longer waiting")
registry_persist(JOBS_DIR)
return _snapshot()
@router.get("")
def get_queue() -> dict[str, Any]:
"""The queue right now. Used for first paint and as the polling fallback
when the stream cannot connect."""
return _snapshot()
@router.get("/events")
async def queue_events() -> StreamingResponse:
"""SSE stream of the whole queue.
Unlike the per-job stream this never self-closes on a terminal status: it
outlives any individual job and is expected to stay open for the session,
so only the 4 h ceiling ends it.
"""
claim_sse_slot()
async def stream() -> AsyncIterator[str]:
try:
last_fp: tuple | None = None
keepalive_at = 0
loop = asyncio.get_running_loop()
deadline = loop.time() + _MAX_SSE_SECONDS
while loop.time() < deadline:
fp = _fingerprint()
if fp != last_fp:
snapshot = _snapshot()
if _fingerprint() != fp:
# _set() landed mid-serialize; the snapshot could mix
# pre- and post-write fields. Re-read next tick rather
# than emit a torn frame (same guard as job_events).
continue
yield f"data: {json.dumps(snapshot)}\n\n"
last_fp = fp
keepalive_at = 0
keepalive_at += 1
if keepalive_at >= 60: # ~15 s at the poll interval below
yield ": keepalive\n\n"
keepalive_at = 0
await asyncio.sleep(0.25)
finally:
release_sse_slot()
return StreamingResponse(
stream(),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
+4
View File
@@ -5,7 +5,9 @@ from fastapi import APIRouter
from app.api.config import router as config_router
from app.api.events import router as events_router
from app.api.jobs import router as jobs_router
from app.api.playlist import router as playlist_router
from app.api.qr import router as qr_router
from app.api.queue import router as queue_router
from app.api.stems import router as stems_router
router = APIRouter()
@@ -14,3 +16,5 @@ router.include_router(jobs_router, prefix="/jobs", tags=["jobs"])
router.include_router(events_router, tags=["events"])
router.include_router(stems_router, tags=["stems"])
router.include_router(qr_router, tags=["qr"])
router.include_router(queue_router, prefix="/queue", tags=["queue"])
router.include_router(playlist_router, prefix="/playlist", tags=["playlist"])
+13 -1
View File
@@ -83,7 +83,19 @@ JOB_TTL_SECONDS = max(300, _env_int("STEMDECK_JOB_TTL_SECONDS", 24 * 3600)) # 2
# Swept unconditionally -- even deployments with a persistent library must not
# accumulate failure evidence forever.
FAILED_TTL_SECONDS = max(3600, _env_int("STEMDECK_FAILED_TTL_SECONDS", 7 * 24 * 3600)) # 7 d
MAX_PENDING_JOBS = max(1, min(50, _env_int("STEMDECK_MAX_PENDING_JOBS", 3)))
# Depth of the import queue: jobs waiting for their turn, not counting the one
# running. Counted separately by kind, because the two cost wildly different
# things. A queued upload holds its source file on disk for the whole wait, so
# 20 of them is already an 8 GB worst case. A queued URL holds nothing at all --
# it downloads when its turn comes -- so the only real cost is a registry
# record, and a 50-track playlist should not have to be imported in batches.
MAX_PENDING_UPLOAD_JOBS = max(1, min(200, _env_int("STEMDECK_MAX_PENDING_JOBS", 20)))
MAX_PENDING_URL_JOBS = max(1, min(500, _env_int("STEMDECK_MAX_PENDING_URL_JOBS", 200)))
# Ceiling on how much of a playlist one import may expand to. Enforced twice:
# as yt-dlp's playlistend so nothing beyond it is ever fetched, and again after
# normalization. Unrelated to MAX_PENDING_JOBS, which bounds the queue itself --
# a playlist larger than the queue has room for fills what it can and says so.
PLAYLIST_MAX_ITEMS = max(1, min(200, _env_int("STEMDECK_PLAYLIST_MAX_ITEMS", 50)))
TIMEOUT_FFMPEG = _env_int("STEMDECK_TIMEOUT_FFMPEG", 300)
TIMEOUT_ANALYZE = _env_int("STEMDECK_TIMEOUT_ANALYZE", 120)
TIMEOUT_DEMUCS_STALL = _env_int("STEMDECK_TIMEOUT_DEMUCS_STALL", 1800)
+24
View File
@@ -83,6 +83,15 @@ class Job:
# tear-detection state for the SSE stream -- not surfaced via to_state()
# or persisted, same as cancel_requested.
version: int = 0
# Place in the waiting queue, rewritten whenever the queue changes. Only
# exists so a reordered queue comes back in the user's order rather than
# submission order after a restart; the position the UI shows is derived
# from the live deque. Old records default to 0, where created_at decides.
queue_position: int = 0
# How many times a restart has put this job back in the queue. Persisted,
# so a job that reliably kills the process is failed rather than retried on
# every start. Old records without the field default to 0 via from_record.
resume_attempts: int = 0
# Wall-clock timestamps for metadata-based sweep -- more predictable
# than directory mtime, which can be touched by unrelated FS events.
created_at: float = field(default_factory=time.time)
@@ -120,6 +129,21 @@ class Job:
"created_at": self.created_at,
}
def to_queue_state(self) -> dict[str, Any]:
"""The compact record the queue view needs. Deliberately not to_state():
the queue stream carries every waiting job several times a second, and
stems/sections/analysis are only meaningful once a job is done."""
return {
"job_id": self.id,
"status": self.status,
"progress": self.progress,
"stage": self.stage_message,
"title": self.title,
"thumbnail": self.thumbnail,
"source_url": self.source_url,
"error": self.error,
}
def to_record(self) -> dict[str, Any]:
return {field: getattr(self, field) for field in _JOB_FIELDS}
+40
View File
@@ -0,0 +1,40 @@
"""Cross-platform process liveness, shared by the parent-death watchdogs.
Lives here rather than in app/main.py so the demucs worker can use it without
importing FastAPI and the whole application: the worker is spawned per device
and its startup time is on the critical path of every separation.
"""
from __future__ import annotations
import os
def process_exists(pid: int) -> bool:
"""Whether a process with this pid is alive.
Errs on the side of "alive": a permission error means the process is there
but owned by someone else, and a watchdog must not shoot on ambiguity.
"""
if pid <= 0:
return False
if os.name != "nt":
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
import ctypes
PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
ERROR_INVALID_PARAMETER = 87
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
if handle:
kernel32.CloseHandle(handle)
return True
return ctypes.get_last_error() != ERROR_INVALID_PARAMETER
+98 -6
View File
@@ -8,7 +8,7 @@ import threading
import uuid
from pathlib import Path
from app.core.config import JOB_ID_RE, STEM_NAMES
from app.core.config import DEMUCS_MODEL, JOB_ID_RE, STEM_NAMES
from app.core.models import Job
logger = logging.getLogger("stemdeck.registry")
@@ -23,6 +23,20 @@ _procs: dict[str, subprocess.Popen] = {}
_lock = threading.Lock()
_REGISTRY_FILE = "registry.json"
_TERMINAL = {"done"}
# Jobs that were queued or mid-flight when the process died. They are persisted
# too (so a queue left running survives a restart) but restored as "queued" and
# handed back to the queue -- see restore(). _TERMINAL stays narrow because
# restore() and _recover_done_job() both mean "finished successfully" by it.
_RESUMABLE = {"queued", "processing", "downloading", "analyzing", "separating"}
_PERSISTED = _TERMINAL | _RESUMABLE
# One resume only. A job that reliably kills the process (a demucs OOM taking
# the backend down with it) would otherwise be re-queued on every start,
# wedging the queue forever.
_MAX_RESUME_ATTEMPTS = 1
# Ids restore() wants re-queued, drained once by the app lifespan.
_pending_resume: list[str] = []
def register(job: Job) -> Job:
@@ -31,11 +45,27 @@ def register(job: Job) -> Job:
return job
def is_upload(job: Job) -> bool:
"""Uploads are the jobs that occupy disk while they wait."""
return (job.source_url or "").startswith("local:")
def pending_count(*, uploads: bool) -> int:
"""How many jobs of one kind are waiting for their turn."""
with _lock:
return sum(1 for j in _jobs.values() if j.status == "queued" and is_upload(j) == uploads)
def register_if_capacity(job: Job, max_pending: int) -> bool:
"""Atomically check pending count and register if under capacity.
Returns True if registered, False if the queue is full."""
Returns True if registered, False if the queue is full.
Capacity is counted per kind: a waiting upload holds its source file on
disk, a waiting URL holds nothing, so a backlog of one must not block the
other."""
uploads = is_upload(job)
with _lock:
pending = sum(1 for j in _jobs.values() if j.status == "queued")
pending = sum(1 for j in _jobs.values() if j.status == "queued" and is_upload(j) == uploads)
if pending >= max_pending:
return False
_jobs[job.id] = job
@@ -94,7 +124,7 @@ def persist(jobs_dir: Path) -> None:
records = [
job.to_record()
for job in sorted(_jobs.values(), key=lambda item: item.created_at)
if job.status in _TERMINAL
if job.status in _PERSISTED
]
payload = json.dumps({"version": REGISTRY_VERSION, "jobs": records}, indent=2) + "\n"
tmp = jobs_dir / f".registry.{uuid.uuid4().hex}.tmp"
@@ -111,22 +141,45 @@ def restore(jobs_dir: Path) -> None:
"""Load persisted jobs and recover completed orphan jobs from disk."""
jobs_dir.mkdir(parents=True, exist_ok=True)
path = registry_path(jobs_dir)
changed = False
if path.is_file():
try:
data = _migrate(json.loads(path.read_text(encoding="utf-8")))
to_add = {}
resume: list[Job] = []
for record in data.get("jobs", []):
job = Job.from_record(record)
if JOB_ID_RE.match(job.id) and job.status in _TERMINAL and job.title:
if not JOB_ID_RE.match(job.id):
continue
if job.status in _TERMINAL and job.title:
to_add[job.id] = job
elif job.status in _RESUMABLE:
recovered = _resume_or_recover(job, jobs_dir / job.id)
if recovered is not None:
to_add[recovered.id] = recovered
# The bumped resume_attempts has to reach disk here. The
# next persist is whenever the job finishes, so without
# this a job that kills the process every time is read
# back with the same count on every start and retried
# forever -- the exact loop the cap exists to stop.
changed = True
if recovered.status in _RESUMABLE:
resume.append(recovered)
with _lock:
_jobs.update(to_add)
# The user's order first, then oldest first. queue_position is
# rewritten whenever the queue changes, so a queue the user
# reordered comes back in that order rather than submission order.
# Records written before that field existed all default to 0, where
# created_at decides exactly as it used to.
_pending_resume.extend(
j.id for j in sorted(resume, key=lambda j: (j.queue_position, j.created_at))
)
except (OSError, json.JSONDecodeError, TypeError, ValueError):
logger.warning("failed to load registry from %s", path, exc_info=True)
with _lock:
known = set(_jobs)
changed = False
for job_dir in jobs_dir.iterdir():
if not job_dir.is_dir() or not JOB_ID_RE.match(job_dir.name) or job_dir.name in known:
continue
@@ -139,6 +192,45 @@ def restore(jobs_dir: Path) -> None:
persist(jobs_dir)
def _resume_or_recover(job: Job, job_dir: Path) -> Job | None:
"""Decide what a job that was in flight when the process died becomes.
Three states of the same directory need telling apart. If the stems are all
there, the crash landed between the last stem being written and the "done"
persist, so the work is finished and re-running it would duplicate the
library entry. Otherwise the job goes back in the queue from the top, after
the partial demucs output is cleared so collect() cannot mistake it for
results. A job that has already burned its resume is failed loudly rather
than retried forever."""
recovered = _recover_done_job(job_dir)
if recovered is not None:
return recovered
job.resume_attempts += 1
if job.resume_attempts > _MAX_RESUME_ATTEMPTS:
job.status = "error"
job.stage_message = "Error: Interrupted"
job.error = "Interrupted by a restart twice. Import it again to retry."
return job
shutil.rmtree(job_dir / DEMUCS_MODEL, ignore_errors=True)
job.status = "queued"
job.stage_message = "Queued"
job.progress = 0.0
job.cancel_requested = False
job.stems = []
job.mix_url = None
return job
def take_pending_resume() -> list[str]:
"""Ids to re-queue after a restart. Pops, so a resume can only fire once."""
with _lock:
ids = list(_pending_resume)
_pending_resume.clear()
return ids
def _recover_done_job(job_dir: Path) -> Job | None:
stems_dir = job_dir / "stems"
if not stems_dir.is_dir():
+22
View File
@@ -5,6 +5,7 @@ at startup), so the Settings UI can change them without a restart:
- `allow_network` — whether StemDeck answers requests from other devices.
- `max_duration_sec` — longest track accepted for processing.
- `playlist_max_items` — how many tracks one playlist import may queue.
- `video_max_height` — max video resolution for MP4 export / YouTube pulls.
- `export_sample_rate` — sample rate for exported mixes/regions (WAV/FLAC/MP3).
- `demucs_device` — compute device for separation: auto | cuda | mps | cpu.
@@ -24,6 +25,7 @@ import threading
from app.core.config import (
DATA_DIR,
MAX_DURATION_SEC,
PLAYLIST_MAX_ITEMS,
VIDEO_MAX_HEIGHT,
available_torch_devices,
detect_torch_device,
@@ -38,6 +40,7 @@ _state: dict | None = None # whole settings dict, loaded lazily
# Clamp bounds. Max track length is capped at 20 min (the product ceiling).
_DURATION_MIN, _DURATION_MAX = 60, 1200 # 1 min .. 20 min
_HEIGHT_MIN, _HEIGHT_MAX = 144, 2160
_PLAYLIST_MIN, _PLAYLIST_MAX = 1, 200
_PORT_MIN, _PORT_MAX = 1024, 65535
DEFAULT_PORT = 8000
@@ -122,6 +125,25 @@ def set_max_duration_sec(value: int) -> int:
return clamped
# ── playlist_max_items ──
# How many tracks one playlist import may queue. A waiting link costs a registry
# record, so the ceiling is generous; the real reason to keep this adjustable is
# that "import 200 tracks" is a decision about the user's evening, not about
# resources.
def get_playlist_max_items() -> int:
with _LOCK:
v = _num(_ensure().get("playlist_max_items"))
return max(_PLAYLIST_MIN, min(_PLAYLIST_MAX, v)) if v is not None else PLAYLIST_MAX_ITEMS
def set_playlist_max_items(value: int) -> int:
with _LOCK:
clamped = max(_PLAYLIST_MIN, min(_PLAYLIST_MAX, int(value)))
_ensure()["playlist_max_items"] = clamped
_save()
return clamped
# ── video_max_height ──
def get_video_max_height() -> int:
with _LOCK:
+38 -22
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
import ctypes
import functools
import io
import logging
@@ -33,8 +32,9 @@ from app.core.config import (
ensure_runtime_dirs,
)
from app.core.logging_setup import configure_logging
from app.core.process import process_exists as _process_exists
from app.core.registry import all_jobs as registry_all_jobs
from app.core.registry import registry_path
from app.core.registry import registry_path, take_pending_resume
from app.core.registry import reset_all as reset_registry
from app.core.registry import restore as restore_registry
from app.core.settings import (
@@ -43,6 +43,7 @@ from app.core.settings import (
get_demucs_device_choice,
get_export_sample_rate,
get_max_duration_sec,
get_playlist_max_items,
get_port,
get_separation_quality,
get_video_max_height,
@@ -50,6 +51,7 @@ from app.core.settings import (
set_demucs_device,
set_export_sample_rate,
set_max_duration_sec,
set_playlist_max_items,
set_port,
set_separation_quality,
set_video_max_height,
@@ -80,26 +82,6 @@ except ImportError:
_log = logging.getLogger("stemdeck")
def _process_exists(pid: int) -> bool:
if os.name != "nt":
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
ERROR_INVALID_PARAMETER = 87
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
if handle:
kernel32.CloseHandle(handle)
return True
return ctypes.get_last_error() != ERROR_INVALID_PARAMETER
def app_version() -> str:
# Version is git-tag-derived via hatch-vcs (#169). Prefer installed package
# metadata (set at install/build from the tag); fall back to the generated
@@ -170,6 +152,29 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
t = asyncio.create_task(_sweep_loop())
_background_tasks.add(t)
t.add_done_callback(_background_tasks.discard)
# The single queue consumer. Started here rather than at import time because
# restore_registry() runs at module scope, where there is no running loop.
from app.pipeline import jobqueue
qt = jobqueue.start_worker()
_background_tasks.add(qt)
qt.add_done_callback(_background_tasks.discard)
# Jobs that were queued or in flight when the process last died. restore()
# already put them back to "queued"; this puts them back in the queue.
#
# Deliberately paused: opening the app must not start separating on its own.
# A restored queue can be dozens of tracks and hours of GPU, and the user
# may well have opened StemDeck to do something else entirely. They press
# Start (or simply import something new, which lifts the pause).
resumed = take_pending_resume()
if resumed:
jobqueue.pause()
for job_id in resumed:
jobqueue.enqueue(job_id, autostart=False)
_log.info(
"restored %d interrupted job(s) from the previous session; queue is paused",
len(resumed),
)
if os.environ.get("STEMDECK_DESKTOP") == "1":
parent_pid = os.environ.get("STEMDECK_PARENT_PID")
if parent_pid:
@@ -183,6 +188,10 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
_background_tasks.add(wt)
wt.add_done_callback(_background_tasks.discard)
yield
# Stop taking new work. Deliberately not cancelling the in-flight job: it is
# blocked in asyncio.to_thread, which cannot be cancelled, so it dies with
# the process exactly as it did before the queue existed.
jobqueue.request_stop()
# Tear down the persistent demucs worker (#309) so a clean shutdown never
# leaves it as an orphaned process -- it has no parent-death watchdog of
# its own, unlike the desktop backend itself.
@@ -264,6 +273,7 @@ def _settings_payload() -> dict[str, object]:
return {
"allow_network": get_allow_network(),
"max_duration_sec": get_max_duration_sec(),
"playlist_max_items": get_playlist_max_items(),
"video_max_height": get_video_max_height(),
"export_sample_rate": get_export_sample_rate(),
"separation_quality": get_separation_quality(),
@@ -302,6 +312,7 @@ async def update_settings(request: Request) -> dict[str, object]:
set_allow_network(bool(body["allow_network"]))
for key, setter in (
("max_duration_sec", set_max_duration_sec),
("playlist_max_items", set_playlist_max_items),
("video_max_height", set_video_max_height),
("port", set_port),
):
@@ -355,6 +366,11 @@ def reset_app_data() -> dict[str, object]:
active = [j for j in registry_all_jobs().values() if j.status in _ACTIVE_JOB_STATUSES]
if active:
raise HTTPException(status_code=409, detail="cannot reset while a job is in progress")
# Clear the queue alongside the registry, or the worker would keep popping
# ids that no longer resolve and the queue view would report ghosts.
from app.pipeline import jobqueue
jobqueue.clear()
reset_registry(JOBS_DIR)
return {"ok": True}
+46
View File
@@ -27,15 +27,22 @@ Protocol:
failure already meant "process is dead, next attempt spawns fresh" --
the reuse win only applies to the happy path.
- EOF on stdin (parent closed the pipe) ends the worker's loop cleanly.
- STEMDECK_PARENT_PID, if set, arms a watchdog that exits the worker when
that process disappears. See _watch_parent for why the pipe alone is not
enough.
"""
from __future__ import annotations
import json
import os
import sys
import threading
import time
from pathlib import Path
from app.core.config import DEMUCS_MODEL
from app.core.process import process_exists
def _run_one_job(model, device: str, req: dict) -> None:
@@ -82,8 +89,47 @@ def _run_one_job(model, device: str, req: dict) -> None:
)
_PARENT_POLL_SECONDS = 1.0
def _watch_parent(parent_pid: int) -> None:
"""Exit as soon as the process that spawned us is gone.
The stdin EOF in the loop below only covers a parent that exits between
jobs. Mid-separation the worker is inside torch and reads nothing, and the
parent may have been killed in a way that ran no cleanup at all (SIGKILL,
Force Quit, Task Manager, a crash). Without this, the worker would keep a
GPU busy with nobody left to collect the result.
os._exit rather than sys.exit: this runs on a daemon thread, and raising
SystemExit there would not interrupt inference running in C code. Nothing
here needs flushing -- a half-written model directory is cleared before the
job is retried.
"""
while True:
if not process_exists(parent_pid):
sys.stderr.write("@@ERROR@@parent process exited\n")
sys.stderr.flush()
os._exit(1)
time.sleep(_PARENT_POLL_SECONDS)
def _arm_parent_watchdog() -> None:
raw = os.environ.get("STEMDECK_PARENT_PID", "").strip()
if not raw:
return
try:
parent_pid = int(raw)
except ValueError:
return
if parent_pid <= 0 or parent_pid == os.getpid():
return
threading.Thread(target=_watch_parent, args=(parent_pid,), daemon=True).start()
def main() -> None:
device = sys.argv[1] if len(sys.argv) > 1 else "cpu"
_arm_parent_watchdog()
from demucs.pretrained import get_model
+125
View File
@@ -108,10 +108,33 @@ _ALLOWED_HOSTS = _YOUTUBE_HOSTS | _SOUNDCLOUD_HOSTS
_ALLOWED_EXTRACTORS = ["youtube", "soundcloud"]
# Expanding a playlist needs the tab/playlist extractors, which the single-video
# allowlist above deliberately excludes. Entries are regexes matched with
# re.fullmatch against IE_NAME.lower(), so "youtube" alone never resolves to
# "youtube:tab" -- hence a second, separate list rather than widening the first.
# "generic" stays out of both: that exclusion is the whole point of #173.
_ALLOWED_PLAYLIST_EXTRACTORS = [
"youtube:tab",
"youtube:playlist",
"youtube",
"soundcloud:set",
"soundcloud",
]
# YouTube list ids. RD-prefixed ones are algorithmic radio: effectively endless
# and different for every viewer, so there is no meaningful set to import.
_PLAYLIST_ID_RE = re.compile(r"^[A-Za-z0-9_-]{2,64}$")
_SOUNDCLOUD_SET_RE = re.compile(r"^/[^/]+/sets/[^/]+/?$")
class InvalidYouTubeURL(ValueError):
"""Raised at the API boundary for URLs we won't hand to yt-dlp."""
class InvalidPlaylistURL(ValueError):
"""Raised for URLs that are not a playlist we are willing to expand."""
def validate_youtube_url(url: str) -> str:
"""Reject anything that isn't an http(s) URL on a known supported host.
YouTube URLs are normalized to single-video form; SoundCloud URLs are
@@ -191,6 +214,108 @@ def normalize_youtube_url(url: str) -> str:
return url
def validate_playlist_url(url: str) -> str:
"""Accept only a playlist we are willing to expand.
The SSRF boundary is unchanged: the host must still be one of the same
allowlisted hosts as a single track. This adds the playlist-shaped checks on
top, so a bare watch URL or a user's profile page is rejected before yt-dlp
ever sees it.
"""
if not isinstance(url, str) or not url.strip():
raise InvalidPlaylistURL("URL is required")
url = url.strip()
try:
parsed = urllib.parse.urlparse(url)
except Exception as e:
raise InvalidPlaylistURL(f"could not parse URL: {e}") from e
if parsed.scheme not in ("http", "https"):
raise InvalidPlaylistURL("URL must use http or https")
host = (parsed.hostname or "").lower()
if host not in _ALLOWED_HOSTS:
raise InvalidPlaylistURL(f"unsupported host: {host or '(empty)'}")
if host in _SOUNDCLOUD_HOSTS:
if not _SOUNDCLOUD_SET_RE.match(parsed.path or ""):
raise InvalidPlaylistURL("not a SoundCloud playlist URL")
return url
list_id = urllib.parse.parse_qs(parsed.query or "").get("list", [""])[0]
if not list_id:
raise InvalidPlaylistURL("URL has no playlist id")
if not _PLAYLIST_ID_RE.match(list_id):
raise InvalidPlaylistURL("invalid playlist id")
if list_id.upper().startswith("RD"):
raise InvalidPlaylistURL("radio playlists cannot be imported")
return f"https://www.youtube.com/playlist?list={list_id}"
def is_playlist_url(url: str) -> bool:
"""Cheap check for the UI: would validate_playlist_url accept this?"""
try:
validate_playlist_url(url)
except InvalidPlaylistURL:
return False
return True
def expand_playlist(url: str, limit: int) -> dict:
"""List a playlist's entries without downloading anything.
Flat extraction, so this is one request rather than one per video. Every
entry URL is put back through validate_youtube_url before it is returned:
entries are attacker-influenced data as far as this process is concerned,
and nothing that failed that check may ever reach the pipeline.
"""
playlist_url = validate_playlist_url(url)
ydl_opts = {
"quiet": True,
"noprogress": True,
"skip_download": True,
"extract_flat": "in_playlist",
"noplaylist": False,
# One past the cap, so a playlist longer than the cap can be reported as
# truncated rather than silently looking like it ends there.
"playlistend": max(1, limit) + 1,
"allowed_extractors": _ALLOWED_PLAYLIST_EXTRACTORS,
"socket_timeout": _SOCKET_TIMEOUT_SEC,
}
with YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(playlist_url, download=False) or {}
entries = [e for e in (info.get("entries") or []) if isinstance(e, dict)]
truncated = len(entries) > limit
entries = entries[:limit]
items: list[dict] = []
unavailable = 0
for entry in entries:
raw = entry.get("url") or entry.get("webpage_url") or ""
try:
normalized = validate_youtube_url(raw)
except InvalidYouTubeURL:
# Deleted, private or region-blocked entries come back as
# placeholders with no usable URL. Count them so the dialog can say
# so, and drop them.
unavailable += 1
continue
items.append(
{
"url": normalized,
"title": entry.get("title") or "",
"duration": entry.get("duration"),
"thumbnail": entry.get("thumbnail"),
}
)
return {
"playlist_title": (info.get("title") or "Playlist").strip() or "Playlist",
"playlist_url": playlist_url,
"items": items,
"unavailable": unavailable,
"truncated": truncated,
}
def _download_video_track(job: Job, url: str, job_dir: Path) -> None:
"""Best-effort: download a video-only H.264/MP4 stream to video.mp4 for the
MP4 export (issue #219). The audio source is downloaded separately as
+282
View File
@@ -0,0 +1,282 @@
"""Serial import queue.
Jobs used to be started with a bare `asyncio.create_task(run_pipeline(...))` and
then all blocked on `_pipeline_lock` inside the pipeline. That worked, but the
"queue" was invisible: waiting jobs were coroutines parked on a semaphore, with
no list to show the user, no stable position, and no way to cancel one before it
started. This module makes the queue explicit.
Exactly one job runs at a time, and that is a correctness requirement rather
than a tuning choice: `separate.py` keeps a single module-global demucs worker
process and `registry.set_proc` maps a job id onto it, so two concurrent jobs
would interleave requests on one stdin, misattribute progress parsed from one
stderr, and a cancel on either would terminate the worker out from under the
other. The single consumer here is what guarantees that; `_pipeline_lock` stays
in runner.py as a second line of defence for anything calling the pipeline
directly (the tests do).
"""
from __future__ import annotations
import asyncio
import logging
import shutil
import threading
from collections import deque
from pathlib import Path
from app.core.config import JOBS_DIR
from app.core.models import Job, _set
from app.core.registry import get as registry_get
from app.core.registry import persist as registry_persist
logger = logging.getLogger("stemdeck.queue")
# Waiting job ids, oldest first. Guarded by _lock because sync (threadpool)
# endpoints read and mutate it -- matching registry.py's threading.Lock model.
_queue: deque[str] = deque()
_running_id: str | None = None
_lock = threading.Lock()
_wake: asyncio.Event | None = None
_loop: asyncio.AbstractEventLoop | None = None
_worker_task: asyncio.Task | None = None
_stopping = False
# Set at startup when jobs are restored from a previous session, so opening the
# app never puts the machine to work on its own. Only picking up NEW work is
# paused; a job already running is unaffected.
_paused = False
def _notify() -> None:
"""Wake the worker. Safe to call from a worker thread: sync FastAPI
endpoints run in Starlette's threadpool, so touching the asyncio.Event
directly from there would be a cross-thread mutation."""
if _loop is None or _wake is None:
return
try:
_loop.call_soon_threadsafe(_wake.set)
except RuntimeError:
# Loop already closed (shutdown races a late enqueue). Nothing to wake.
pass
def enqueue(job_id: str, *, autostart: bool = True) -> None:
"""Add a job to the back of the queue.
autostart=False is for jobs restored from a previous session: they are put
back in the queue but must not start on their own. Everything else is a
thing the user just asked for, so it also lifts a pause -- pressing Process
and having nothing happen would be its own bug.
"""
global _paused
with _lock:
if job_id not in _queue and job_id != _running_id:
_queue.append(job_id)
_renumber_locked()
if autostart:
_paused = False
_notify()
def _renumber_locked() -> None:
"""Stamp each waiting job with its place in line, for persistence only.
Written directly rather than through _set(): the position the UI shows is
derived from the live deque, so bumping every job's version here would wake
every SSE stream to report something they already know.
Caller holds _lock.
"""
for index, job_id in enumerate(_queue):
job = registry_get(job_id)
if job is not None:
job.queue_position = index
def reorder(job_id: str, after_id: str | None) -> bool:
"""Move a waiting job to sit directly after `after_id`, or to the front when
that is None.
Expressed as "after this job" rather than "at index N" on purpose: the queue
moves under the user as jobs finish, so an index captured when the drag
started can easily mean somewhere else by the time it lands.
Returns False if the job is not waiting -- it finished or started while the
user was dragging it.
"""
with _lock:
if job_id not in _queue:
return False
_queue.remove(job_id)
if after_id is None:
_queue.appendleft(job_id)
else:
try:
_queue.insert(_queue.index(after_id) + 1, job_id)
except ValueError:
# The anchor left the queue mid-drag. Falling back to the end is
# the honest answer; the response carries the resulting order so
# the client re-syncs rather than guessing.
_queue.append(job_id)
_renumber_locked()
return True
def discard(job_id: str) -> bool:
"""Remove a job that has not started yet. Returns True if it was still
waiting, which tells the caller it can finalise the job itself -- the
worker will never touch it."""
with _lock:
if job_id in _queue:
_queue.remove(job_id)
return True
return False
def snapshot() -> tuple[str | None, list[str]]:
"""(running_id, waiting_ids). One lock acquisition so the pair is coherent."""
with _lock:
return _running_id, list(_queue)
def running_id() -> str | None:
with _lock:
return _running_id
def depth() -> int:
with _lock:
return len(_queue)
def clear() -> None:
"""Drop every waiting job. Used by /api/reset, which has already refused to
run if anything is in flight."""
with _lock:
_queue.clear()
def _pop_next() -> str | None:
with _lock:
return _queue.popleft() if _queue else None
def _set_running(job_id: str | None) -> None:
global _running_id
with _lock:
_running_id = job_id
def _find_local_source(job_dir: Path) -> Path | None:
"""The uploaded file for a `local:` job. The extension varies, and a resumed
job only knows its directory."""
for candidate in sorted(job_dir.glob("source.*")):
if candidate.is_file():
return candidate
return None
async def _dispatch(job: Job) -> None:
"""Run one job. The mode is derived from source_url rather than passed in,
so a fresh submit and a post-restart resume go through the same path."""
from app.pipeline.runner import run_local_pipeline, run_pipeline
source_url = job.source_url or ""
if source_url.startswith("local:"):
source = _find_local_source(JOBS_DIR / job.id)
if source is None:
logger.warning("[%s] local source missing; cannot run", job.id)
_set(
job,
status="error",
stage="Error: Source file missing",
error="The uploaded file is no longer available. Re-import it to finish.",
)
registry_persist(JOBS_DIR)
return
await run_local_pipeline(job, source, JOBS_DIR)
return
await run_pipeline(job, source_url, JOBS_DIR)
async def _worker_loop() -> None:
assert _wake is not None
while not _stopping:
if _paused:
# Idle without draining. A job already in flight is not touched --
# pausing only stops the queue picking up the next one.
_wake.clear()
await _wake.wait()
continue
job_id = _pop_next()
if job_id is None:
_wake.clear()
await _wake.wait()
continue
job = registry_get(job_id)
if job is None:
continue
if job.cancel_requested or job.status in ("done", "error", "cancelled"):
# Cancelled or finished while it waited; drop it silently.
continue
# Claim it. No await between the pop and this status write, so a job is
# never counted as both waiting and running -- which is what lets
# register_if_capacity keep counting only "queued" as the queue depth.
_set_running(job_id)
_set(job, status="processing", stage="Starting...", progress=0.0)
try:
await _dispatch(job)
except Exception:
# A crash in one job must not take the queue down with it.
logger.exception("[%s] queue worker: job raised", job_id)
finally:
_set_running(None)
def start_worker() -> asyncio.Task:
"""Start the single consumer. Called from the app lifespan, where there is a
running loop -- registry.restore() runs at import time and must not touch
asyncio."""
global _wake, _loop, _worker_task, _stopping, _paused
_stopping = False
_paused = False
_loop = asyncio.get_running_loop()
_wake = asyncio.Event()
_wake.set() # do one pass immediately, in case resume already enqueued work
_worker_task = asyncio.create_task(_worker_loop())
return _worker_task
def pause() -> None:
"""Stop picking up new work until someone asks for it."""
global _paused
_paused = True
def resume() -> None:
global _paused
_paused = False
_notify()
def is_paused() -> bool:
return _paused
def request_stop() -> None:
"""Stop picking up new work. Deliberately does not cancel the in-flight job:
it is blocked in asyncio.to_thread, which cannot be cancelled, so it dies
with the process exactly as it did before the queue existed."""
global _stopping
_stopping = True
_notify()
def cleanup_job_dir(job_id: str) -> None:
"""Remove a cancelled-before-start job's directory. A queued local upload
holds its source file (up to 400 MB) for the whole wait."""
shutil.rmtree(JOBS_DIR / job_id, ignore_errors=True)
+4
View File
@@ -60,6 +60,10 @@ def _get_worker(device: str) -> subprocess.Popen:
_kill_worker()
env = os.environ.copy()
# Our pid, not whatever the backend inherited: the worker watches this and
# exits when we are gone, so it cannot be left holding a GPU after a kill
# that ran no cleanup (SIGKILL, Force Quit, Task Manager, a crash).
env["STEMDECK_PARENT_PID"] = str(os.getpid())
try:
import certifi
+5 -1
View File
@@ -109,4 +109,8 @@ RUN chmod +x /entrypoint.sh
EXPOSE 8000
ENTRYPOINT ["/entrypoint.sh"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
# --timeout-graceful-shutdown bounds the wait for open connections. The queue
# SSE stream stays open while a browser tab is on the app, and without this
# `docker stop` would sit through its full 10s timeout and then SIGKILL --
# which would skip the lifespan teardown that reaps the demucs worker.
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--timeout-graceful-shutdown", "5"]
+9
View File
@@ -632,6 +632,15 @@ fn start_backend(
bind_host,
"--port",
&port.to_string(),
// Bound how long uvicorn waits for open connections on shutdown.
// The import queue's SSE stream stays open for as long as the app
// window is on screen, and uvicorn drains connections before it
// runs the lifespan teardown -- so without this the backend never
// finishes draining, we escalate to SIGKILL below, and the teardown
// that reaps the demucs worker never runs. Kept under the 3 s
// SIGKILL deadline so the clean path wins.
"--timeout-graceful-shutdown",
"2",
]);
#[cfg(windows)]
if let Some(ref pythonhome) = pythonhome {
+16 -5
View File
@@ -33,7 +33,12 @@ start() {
exit 1
fi
echo "starting on http://$HOST:$PORT"
local args=(app.main:app --host "$HOST" --port "$PORT")
# A long-lived SSE stream (the import queue view) keeps a connection open
# for as long as a browser tab is open, and uvicorn waits for open
# connections before it exits. Without a bound, Ctrl-C appears to hang.
# Kept below stop()'s own 5 s deadline so the clean path finishes first and
# the lifespan teardown (which reaps the demucs worker) actually runs.
local args=(app.main:app --host "$HOST" --port "$PORT" --timeout-graceful-shutdown 2)
if [[ "$RELOAD" == "1" ]]; then
args+=(--reload)
fi
@@ -59,8 +64,11 @@ start() {
stop() {
if ! is_running; then
echo "not running"
# also sweep any stray uvicorn for this app
pkill -f "uvicorn app.main:app" 2>/dev/null || true
# Sweep a stray server for THIS port only. A bare
# "uvicorn app.main:app" pattern also matches the backend that
# StemDeck.app spawns, so it would kill the desktop app out from
# under the user (#352).
pkill -f "uvicorn app.main:app.*--port $PORT" 2>/dev/null || true
rm -f "$PID_FILE"
return 0
fi
@@ -76,8 +84,11 @@ stop() {
echo "force-killing pid $pid"
kill -9 "$pid" 2>/dev/null || true
fi
# kill any in-flight demucs children spawned by the app
pkill -f "python -m demucs" 2>/dev/null || true
# No demucs sweep here. The separation worker is a child of the backend
# and exits on its own when the backend dies -- its stdin and stderr
# pipes close with the parent, which ends its read loop. The pattern
# that used to be here ("python -m demucs") never matched it anyway:
# the worker runs as "python -m app.pipeline.demucs_worker".
rm -f "$PID_FILE"
echo "stopped"
}
+111
View File
@@ -430,6 +430,20 @@ input, textarea { font-family: inherit; }
.rail-btn.active { color: var(--accent); background: var(--panel); }
.rail-btn svg { flex-shrink: 0; }
.rail-btn span { white-space: nowrap; }
/* Queue rail button. Only present while something is importing, so the rail
stays as it is today for anyone not using the queue. */
.rail-queue { position: relative; }
.rail-badge {
position: absolute; top: 2px; right: 2px;
min-width: 15px; height: 15px; padding: 0 4px;
border-radius: 8px;
background: var(--accent); color: #1a1a1a;
font-size: 9px; font-weight: 700; line-height: 15px;
text-align: center;
font-variant-numeric: tabular-nums;
pointer-events: none;
}
/* "We Recommend" is wider than the other one-word rail labels, so it stacks onto
two centered lines; let the button grow so the second line + icon aren't clipped. */
.rail-btn.rail-recommend { height: auto; min-height: 40px; padding: 3px 0; }
@@ -641,6 +655,91 @@ input, textarea { font-family: inherit; }
.cat-item.unavailable { cursor: not-allowed; }
.cat-item.unavailable .cat-meta { opacity: 0.45; }
@keyframes cat-pulse { 0%,100%{opacity:1;} 50%{opacity:0.3;} }
/* Import queue: a waiting row and a running row must be told apart at a
glance, so they differ in three ways -- wording, the presence of a progress
bar, and a hollow versus filled status dot. */
.cat-queue-label { color: var(--accent); font-variant-numeric: tabular-nums; }
.cat-item.queue-waiting .cat-queue-label { color: var(--muted); }
/* Waiting rows are not loadable yet, so they read as pending rather than
inviting a click. */
.cat-item.queue-waiting { cursor: default; }
.cat-item.queue-waiting .cat-thumb { opacity: 0.55; }
.cat-item.queue-waiting .cat-title { color: var(--muted); }
.cat-item.queue-waiting .cat-status {
background: transparent;
box-shadow: inset 0 0 0 1.5px var(--muted);
animation: none;
}
.cat-progress {
height: 3px; border-radius: 2px; margin-top: 4px;
background: rgba(255, 255, 255, 0.09); overflow: hidden;
}
.cat-progress-fill {
height: 100%; width: 0; border-radius: 2px;
background: var(--accent);
transition: width 260ms linear;
}
/* Queue view rows. Unlike a library row these are not loadable -- there is
nothing to load yet -- so they get no hover affordance, just a cancel. */
.queue-row { cursor: default; }
.queue-row:hover { background: var(--panel); }
.queue-row .cat-meta { gap: 3px; }
.queue-cancel {
width: 22px; height: 22px; border: 0; border-radius: 5px;
background: transparent; color: var(--muted); cursor: pointer;
display: flex; align-items: center; justify-content: center;
padding: 0; flex-shrink: 0; opacity: 0.65;
transition: opacity var(--t-fast), background var(--t-fast), color var(--t-fast);
}
.queue-row:hover .queue-cancel { opacity: 1; }
.queue-cancel:hover { background: rgba(217, 83, 79, 0.16); color: #e06c68; }
.queue-cancel:disabled { opacity: 0.3; cursor: default; }
.queue-section .lib-section-head { margin-bottom: 6px; }
/* Restored-from-last-session banner. Opening the app never starts separating
on its own, so this is the only way that queue moves. */
.queue-paused-banner {
display: flex; align-items: center; gap: 8px;
margin: 0 0 8px; padding: 8px 10px;
border: 1px solid rgba(232, 168, 56, 0.28);
border-radius: 8px;
background: rgba(232, 168, 56, 0.08);
}
.queue-paused-text { flex: 1; font-size: 11px; color: var(--fg-2); line-height: 1.4; }
.queue-start-btn {
flex-shrink: 0; min-height: 26px; padding: 0 12px;
border-radius: 6px; cursor: pointer;
border: 1px solid rgba(232, 168, 56, 0.45);
background: rgba(232, 168, 56, 0.18);
color: var(--accent);
font-family: var(--font-mono); font-size: 11px; font-weight: 600;
}
.queue-start-btn:hover { background: rgba(232, 168, 56, 0.28); }
.queue-start-btn:disabled { opacity: 0.5; cursor: default; }
/* Reordering. Position in a serial queue is the difference between "ready now"
and "ready in half an hour", so waiting rows drag. */
.queue-row.queue-waiting { cursor: grab; }
.queue-row.dragging { opacity: 0.4; cursor: grabbing; }
.queue-row.drop-below { box-shadow: 0 2px 0 0 var(--accent); }
.queue-grip {
display: flex; align-items: center; flex-shrink: 0;
margin-right: -2px; color: var(--muted); opacity: 0;
transition: opacity var(--t-fast);
}
.queue-row:hover .queue-grip { opacity: 0.7; }
.queue-top {
width: 22px; height: 22px; border: 0; border-radius: 5px;
background: transparent; color: var(--muted); cursor: pointer;
display: flex; align-items: center; justify-content: center;
padding: 0; flex-shrink: 0; opacity: 0;
transition: opacity var(--t-fast), background var(--t-fast), color var(--t-fast);
}
.queue-row:hover .queue-top { opacity: 0.65; }
.queue-top:hover { background: rgba(232, 168, 56, 0.16); color: var(--accent); opacity: 1; }
.queue-top:disabled { opacity: 0.3; cursor: default; }
.cat-del {
width: 20px; height: 20px; border: 0; border-radius: 5px;
background: transparent; color: var(--muted); cursor: pointer;
@@ -887,6 +986,18 @@ input, textarea { font-family: inherit; }
.reset-confirm-cancel { border: 1px solid var(--border); background: rgba(21,31,39,0.52); color: var(--muted); }
.reset-confirm-go { border: 1px solid rgba(214,90,74,0.4); background: rgba(214,90,74,0.18); color: var(--danger); font-weight: 600; }
.reset-confirm-go:disabled { opacity: 0.4; cursor: not-allowed; }
/* Playlist import reuses the confirm shell, but it is a normal action rather
than a destructive one, so it drops the danger colouring. */
.playlist-confirm-title { color: var(--fg); }
.playlist-confirm .playlist-name { color: var(--accent); }
.playlist-confirm-go {
min-height: 30px; border-radius: 7px; padding: 0 12px; cursor: pointer;
font-family: var(--font-mono); font-size: 11px; font-weight: 600;
border: 1px solid rgba(232, 168, 56, 0.45);
background: rgba(232, 168, 56, 0.16);
color: var(--accent);
}
.playlist-confirm-go:disabled { opacity: 0.4; cursor: not-allowed; }
.library-editor-foot { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 11px; }
.library-editor-status { font-size: 10.5px; color: var(--muted); }
+9 -1
View File
@@ -57,7 +57,7 @@
<path d="M12 3v12 M7 8l5-5 5 5 M5 21h14"/>
</svg>
</button>
<input id="fileInput" type="file" accept=".mp3,.wav,.flac,.mp4,.m4a,.ogg,.opus,audio/mpeg,audio/wav,audio/flac,video/mp4,audio/mp4,audio/ogg,audio/opus" style="display:none" aria-hidden="true" />
<input id="fileInput" type="file" multiple accept=".mp3,.wav,.flac,.mp4,.m4a,.ogg,.opus,audio/mpeg,audio/wav,audio/flac,video/mp4,audio/mp4,audio/ogg,audio/opus" style="display:none" aria-hidden="true" />
</div>
<!-- Divider -->
@@ -167,6 +167,14 @@
</svg>
<span>Trash</span>
</button>
<button class="rail-btn rail-queue hidden" type="button" title="Import queue" aria-label="Import queue" aria-pressed="false">
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.6" aria-hidden="true">
<path d="M4 6h10 M4 12h10 M4 18h6"/>
<path d="M16 15l3 3 3-3 M19 18V9"/>
</svg>
<span>Queue</span>
<span class="rail-badge hidden" id="queueBadge" aria-hidden="true"></span>
</button>
<div style="flex:1"></div>
<button class="rail-btn" id="settingsBtn" type="button" aria-label="Settings" aria-haspopup="dialog">
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.6" aria-hidden="true">
+452 -9
View File
@@ -2,8 +2,13 @@
import { STEM_NAMES } from "./constants.js";
import { wireUpAudio, updateFooterTrack } from "./player.js";
import { initSections } from "./sections.js";
import { bpmChip, keyChip, saveSelectedStems, selectedStems, titleEl } from "./state.js";
import { showError, importFromUrl } from "./job.js";
import { bpmChip, foregroundJobId, keyChip, saveSelectedStems, selectedStems, titleEl } from "./state.js";
import { showError, importFromUrl, detachForegroundJob } from "./job.js";
import {
cancelQueuedJob, getQueueSnapshot, isPaused, onJobSettled, onQueueChange, ordinal,
queueCount, queueRowStates, reorderQueuedJob, runningLabel, startQueue,
startQueueStream,
} from "./queue.js";
import { fmtTime, storeGet, storeSet } from "./utils.js";
// Escape user-supplied strings before inserting into innerHTML.
@@ -494,9 +499,9 @@ function moveTrackToTrash(trackId) {
}
function setCatalogView(view) {
catalogView = ["trash", "favorites"].includes(view) ? view : "library";
catalogView = ["trash", "favorites", "queue"].includes(view) ? view : "library";
const app = document.querySelector(".app");
if (catalogView === "trash" || catalogView === "favorites") {
if (catalogView !== "library") {
app?.classList.remove("cat-collapsed");
localStorage.setItem("stemdeck.catalog.collapsed", "0");
}
@@ -522,6 +527,11 @@ async function loadTrackIntoStudio(trackId) {
showError("This track's audio is no longer available. Re-upload to restore it.");
return;
}
// The user has chosen to look at something else, so the running import gives
// up the studio. It keeps running and keeps updating its own row; it just
// stops repainting this view (and, at completion, replacing the audio that
// is about to load here).
if (trackId !== foregroundJobId) detachForegroundJob();
const hadStoredAudio = Boolean(track.audioStems?.length);
const token = ++_loadTrackToken;
@@ -592,6 +602,47 @@ function createFolder() {
openFolderEditor(folder.id);
}
/** Put a whole playlist import in a folder of its own.
*
* Placement happens before addTrackToLibrary, which only assigns a folder to a
* track that is not in one yet -- so claiming the ids first is what keeps these
* tracks out of Unsorted. Reuses an existing folder of the same name so
* re-importing a playlist tops it up instead of creating a duplicate.
*/
export function addPlaylistToLibrary(playlistTitle, jobs) {
const name = String(playlistTitle || "Playlist").trim().slice(0, 80) || "Playlist";
let folder = folders.find((f) => f.id !== TRASH_ID && !f.parentId && f.name === name);
if (!folder) {
folder = makeFolder({ name });
folders.unshift(folder);
}
for (const job of jobs) {
if (!folder.items.includes(job.job_id)) folder.items.push(job.job_id);
addTrackToLibrary({
id: job.job_id,
title: job.title || job.source_url || "Queued track",
channel: "Processing",
thumb: "",
stems: [...selectedStems],
selectedStems: [...selectedStems],
audioStems: [],
status: "queued",
bpm: null,
key: null,
scale: null,
keyConfidence: null,
lufs: null,
peakDb: null,
sourceUrl: job.source_url,
});
}
folder.collapsed = false;
saveState();
render();
return folder.id;
}
function deleteFolder(folderId) {
if (folderId === TRASH_ID || folderId === UNSORTED_ID) return;
// Cascade: delete children first.
@@ -969,7 +1020,7 @@ function renderRecentItem(trackId) {
el.innerHTML = `
<div class="cat-thumb">${thumbHtml(track)}</div>
<div class="cat-meta">
<div class="cat-title">${esc(track.title ?? "Unknown track")}</div>
<div class="cat-title">${esc(displayTitle(track.title))}</div>
<div class="cat-sub"><span>${esc(sub)}</span></div>
</div>
<div class="cat-status${PROCESSING_STATUSES.has(track.status) ? " processing" : isUnavailable ? " unavailable" : ""}"></div>
@@ -980,6 +1031,26 @@ function renderRecentItem(trackId) {
// ─── Rendering ───
// A queued URL import has no title yet -- nothing has been downloaded, so the
// only thing to show is the URL the user pasted, which renders as a truncated
// unreadable string. Name the source instead until the real title arrives.
const _SOURCE_LABELS = [
[/(^|\.)youtube\.com$|(^|\.)youtu\.be$|(^|\.)youtube-nocookie\.com$/, "YouTube"],
[/(^|\.)soundcloud\.com$/, "SoundCloud"],
];
export function displayTitle(title) {
const text = String(title ?? "").trim();
if (!/^https?:\/\//i.test(text)) return text || "Unknown track";
try {
const host = new URL(text).hostname.replace(/^www\./, "");
const match = _SOURCE_LABELS.find(([re]) => re.test(host));
return `${match ? match[1] : host} link`;
} catch {
return text;
}
}
function thumbHtml(track) {
if (track.thumb) return `<img src="${esc(track.thumb)}" alt="" loading="lazy" />`;
return `<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M9 18V5l12-2v13"></path><circle cx="6" cy="18" r="3"></circle><circle cx="18" cy="16" r="3"></circle></svg>`;
@@ -1016,7 +1087,7 @@ function renderTrackItem(trackId, { inTrash = false } = {}) {
el.innerHTML = `
<div class="cat-thumb">${thumbHtml(track)}</div>
<div class="cat-meta">
<div class="cat-title">${esc(track.title ?? "Unknown track")}</div>
<div class="cat-title">${esc(displayTitle(track.title))}</div>
<div class="cat-sub">
<span>${esc(track.channel ?? "")}</span>
<span class="dot">·</span>
@@ -1250,10 +1321,12 @@ function render() {
const trashIds = new Set(trash?.items || []);
const isTrashView = catalogView === "trash";
const isFavoritesView = catalogView === "favorites";
const isLibraryView = !isTrashView && !isFavoritesView;
const isQueueView = catalogView === "queue";
const isLibraryView = !isTrashView && !isFavoritesView && !isQueueView;
catalog?.classList.toggle("trash-view", isTrashView);
catalog?.classList.toggle("favorites-view", isFavoritesView);
catalog?.classList.toggle("queue-view", isQueueView);
document.querySelector(".rail-library")?.classList.toggle("active", isLibraryView);
document.querySelector(".rail-library")?.setAttribute("aria-pressed", String(isLibraryView));
@@ -1261,11 +1334,23 @@ function render() {
document.querySelector(".rail-favorites")?.setAttribute("aria-pressed", String(isFavoritesView));
document.querySelector(".rail-trash")?.classList.toggle("active", isTrashView);
document.querySelector(".rail-trash")?.setAttribute("aria-pressed", String(isTrashView));
document.querySelector(".rail-queue")?.classList.toggle("active", isQueueView);
document.querySelector(".rail-queue")?.setAttribute("aria-pressed", String(isQueueView));
if (searchInput) {
searchInput.placeholder = isTrashView ? "Search trash…" : isFavoritesView ? "Search favorites…" : "Search library…";
}
// ── Queue view ──
// Rendered from the queue snapshot, not the library: it shows what the
// backend is actually working on, in the order it will work on it.
if (isQueueView) {
renderQueueList(list);
renderStrip(strip, folders.filter((f) => f.id !== TRASH_ID && !f.parentId));
updateQueueBadge();
return;
}
const nonTrash = folders.filter((f) => f.id !== TRASH_ID && !f.parentId);
// ── Trash view ──
@@ -1375,6 +1460,343 @@ function render() {
}
renderStrip(strip, nonTrash);
updateQueueBadge();
applyQueueDecorations();
}
// ─── Import queue decoration ───
//
// Rows are patched in place rather than re-rendered. The queue stream delivers
// a frame several times a second, and a full render() rebuilds the whole
// sidebar and re-runs every drag/click wiring -- at that rate it would fight
// the user for the DOM. render() calls this once at the end so a genuine
// rebuild picks the decoration back up.
function progressBarHtml() {
return '<div class="cat-progress"><div class="cat-progress-fill"></div></div>';
}
function decorateRow(el, rowState) {
const sub = el.querySelector(".cat-sub");
if (!sub) return;
if (!rowState) {
// Left the queue (finished, failed or cancelled). render() will have
// rebuilt the row from the library entry, so just drop the decoration.
el.classList.remove("in-queue", "queue-waiting", "queue-running");
el.querySelector(".cat-progress")?.remove();
if (el.dataset.subRestore) {
sub.innerHTML = el.dataset.subRestore;
delete el.dataset.subRestore;
}
return;
}
const waiting = rowState.state === "waiting";
el.classList.add("in-queue");
el.classList.toggle("queue-waiting", waiting);
el.classList.toggle("queue-running", !waiting);
// Keep the original sub line so it can come back if this row is still on
// screen when the job leaves the queue.
if (!el.dataset.subRestore) el.dataset.subRestore = sub.innerHTML;
const label = `<span class="cat-queue-label">${esc(rowState.label)}</span>`;
if (sub.innerHTML !== label) sub.innerHTML = label;
let bar = el.querySelector(".cat-progress");
if (waiting) {
bar?.remove();
return;
}
if (!bar) {
sub.insertAdjacentHTML("afterend", progressBarHtml());
bar = el.querySelector(".cat-progress");
}
const fill = bar?.querySelector(".cat-progress-fill");
if (fill) fill.style.width = `${Math.round(rowState.progress * 100)}%`;
}
/** A background import has finished (or failed, or was cancelled). It has no
* per-job stream, so fetch its final state once and complete its library entry
* -- stems, duration and analysis all land here, which is what makes the track
* playable from the sidebar without a page reload. */
async function completeSettledJob(jobId) {
const existing = tracks[jobId];
if (!existing) return; // not ours (or already deleted)
try {
const res = await fetch(`/api/jobs/${jobId}`, { cache: "no-store" });
if (!res.ok) {
// 404 means the job is gone from the backend entirely.
if (res.status === 404) updateTrackStatus(jobId, "unavailable");
return;
}
const state = await res.json();
if (state.status === "cancelled") {
// Nothing was produced; drop the placeholder row rather than leaving a
// track that can never be loaded.
delete tracks[jobId];
removeTrackFromFolders(jobId);
saveState();
render();
return;
}
const track = stateMetadataToTrack(state, { ...existing, id: jobId });
track.id = jobId;
track.channel = state.status === "done" ? "Extracted" : existing.channel;
addTrackToLibrary(track);
} catch (e) {
console.warn("[catalog] could not finish background job", jobId, e);
}
}
// ─── Queue view ───
function queueEntries(snap) {
const entries = [];
if (snap.running) entries.push({ job: snap.running, running: true });
for (const job of snap.queued ?? []) entries.push({ job, running: false });
return entries;
}
function queueRowHtml({ job, running }, place, { paused = false } = {}) {
const track = tracks[job.job_id];
const label = running ? runningLabel(job) : paused ? "Paused" : `Queued - ${ordinal(place)} in line`;
const thumb = track ? thumbHtml(track) : thumbHtml({ thumb: job.thumbnail });
// The running job cannot be reordered -- it is already running. Only waiting
// rows drag, and only they offer "play next".
const handle = running
? ""
: `<span class="queue-grip" title="Drag to reorder" aria-hidden="true">
<svg viewBox="0 0 24 24" width="10" height="10" fill="currentColor">
<circle cx="9" cy="5" r="1.5"/><circle cx="15" cy="5" r="1.5"/>
<circle cx="9" cy="12" r="1.5"/><circle cx="15" cy="12" r="1.5"/>
<circle cx="9" cy="19" r="1.5"/><circle cx="15" cy="19" r="1.5"/>
</svg>
</span>`;
return `
<div class="cat-item queue-row ${running ? "queue-running" : "queue-waiting"}" data-id="${esc(job.job_id)}"${running ? "" : ' draggable="true"'}>
${handle}
<div class="cat-thumb">${thumb}</div>
<div class="cat-meta">
<div class="cat-title">${esc(displayTitle(job.title || track?.title || job.source_url))}</div>
<div class="cat-sub"><span class="cat-queue-label">${esc(label)}</span></div>
${running ? '<div class="cat-progress"><div class="cat-progress-fill"></div></div>' : ""}
</div>
${running || place <= 2 ? "" : `<button class="queue-top" type="button" title="Extract this one next"
aria-label="Move ${esc(displayTitle(job.title || job.source_url))} to the front of the queue">
<svg viewBox="0 0 24 24" width="12" height="12" fill="none" stroke="currentColor" stroke-width="2.2" aria-hidden="true">
<path d="M12 19V5 M5 12l7-7 7 7"></path>
</svg>
</button>`}
<button class="queue-cancel" type="button" title="Cancel this import"
aria-label="Cancel import of ${esc(displayTitle(job.title || job.source_url))}">
<svg viewBox="0 0 24 24" width="12" height="12" fill="none" stroke="currentColor" stroke-width="2.2" aria-hidden="true">
<path d="M18 6 6 18 M6 6l12 12"></path>
</svg>
</button>
</div>`;
}
// True while the user is dragging a queue row. Incoming queue frames arrive
// several times a second; re-rendering the list mid-drag would pull the row out
// from under the cursor and cancel the drag.
let _queueDragId = null;
export function isQueueDragging() {
return _queueDragId !== null;
}
/** The id the dragged row should sit after, given where it was dropped.
* Null means the front of the queue. */
function dropAnchorId(listEl, draggedId, clientY) {
const rows = [...listEl.querySelectorAll(".queue-row")].filter(
(r) => r.dataset.id !== draggedId,
);
let anchor = null;
for (const row of rows) {
const box = row.getBoundingClientRect();
if (clientY > box.top + box.height / 2) anchor = row.dataset.id;
}
return anchor;
}
function wireQueueRow(el, listEl) {
el.querySelector(".queue-cancel")?.addEventListener("click", async (e) => {
e.stopPropagation();
const btn = e.currentTarget;
btn.disabled = true;
await cancelQueuedJob(el.dataset.id);
});
el.querySelector(".queue-top")?.addEventListener("click", async (e) => {
e.stopPropagation();
e.currentTarget.disabled = true;
await reorderQueuedJob(el.dataset.id, null);
});
if (el.getAttribute("draggable") !== "true") return;
el.addEventListener("dragstart", (e) => {
_queueDragId = el.dataset.id;
el.classList.add("dragging");
e.dataTransfer.effectAllowed = "move";
// Firefox refuses to start a drag without payload.
e.dataTransfer.setData("text/plain", el.dataset.id);
});
el.addEventListener("dragend", () => {
_queueDragId = null;
el.classList.remove("dragging");
for (const r of listEl.querySelectorAll(".queue-row")) r.classList.remove("drop-below");
});
}
function wireQueueListDrop(listEl) {
listEl.addEventListener("dragover", (e) => {
if (!_queueDragId) return;
e.preventDefault();
e.dataTransfer.dropEffect = "move";
const anchor = dropAnchorId(listEl, _queueDragId, e.clientY);
for (const r of listEl.querySelectorAll(".queue-row")) {
r.classList.toggle("drop-below", !!anchor && r.dataset.id === anchor);
}
});
listEl.addEventListener("drop", async (e) => {
if (!_queueDragId) return;
e.preventDefault();
const dragged = _queueDragId;
const anchor = dropAnchorId(listEl, dragged, e.clientY);
_queueDragId = null;
for (const r of listEl.querySelectorAll(".queue-row")) r.classList.remove("drop-below");
if (anchor !== dragged) await reorderQueuedJob(dragged, anchor);
});
}
function queuePausedBannerHtml(count) {
const noun = count === 1 ? "track" : "tracks";
return `
<div class="queue-paused-banner">
<div class="queue-paused-text">
Paused - ${count} ${noun} from your last session.
</div>
<button class="queue-start-btn" type="button">Start</button>
</div>`;
}
function renderQueueList(listEl, snap = getQueueSnapshot()) {
const entries = queueEntries(snap);
listEl.innerHTML = "";
const section = document.createElement("div");
section.className = "lib-section queue-section";
section.innerHTML = `<div class="lib-section-head"><span>IMPORT QUEUE</span></div>`;
if (isPaused(snap)) {
section.insertAdjacentHTML("beforeend", queuePausedBannerHtml(entries.length));
section.querySelector(".queue-start-btn")?.addEventListener("click", async (e) => {
e.currentTarget.disabled = true;
e.currentTarget.textContent = "Starting…";
await startQueue();
});
}
if (!entries.length) {
section.insertAdjacentHTML(
"beforeend",
'<span class="folder-empty trash-empty">Nothing importing. Queued tracks appear here.</span>',
);
listEl.appendChild(section);
return;
}
const paused = isPaused(snap);
section.insertAdjacentHTML(
"beforeend",
entries.map((entry, i) => queueRowHtml(entry, i + 1, { paused })).join(""),
);
listEl.appendChild(section);
for (const el of section.querySelectorAll(".queue-row")) wireQueueRow(el, listEl);
wireQueueListDrop(listEl);
updateQueueRows(snap);
}
/** Patch the open queue view in place. Only a change to which jobs are present
* (or their order) costs a rebuild; progress and stage text are written
* straight to the existing nodes, because this runs several times a second. */
function updateQueueRows(snap = getQueueSnapshot()) {
const listEl = document.getElementById("catalogList");
if (!listEl || catalogView !== "queue") return;
// A frame landing mid-drag would rebuild the list and yank the row out
// from under the cursor.
if (isQueueDragging()) return;
const entries = queueEntries(snap);
const shown = [...listEl.querySelectorAll(".queue-row")].map((el) => el.dataset.id);
const wanted = entries.map((e) => e.job.job_id);
const bannerShown = !!listEl.querySelector(".queue-paused-banner");
if (
shown.length !== wanted.length ||
shown.some((id, i) => id !== wanted[i]) ||
bannerShown !== isPaused(snap)
) {
renderQueueList(listEl, snap);
return;
}
entries.forEach((entry, i) => {
const el = listEl.querySelector(`.queue-row[data-id="${entry.job.job_id}"]`);
if (!el) return;
const label = entry.running
? runningLabel(entry.job)
: isPaused(snap)
? "Paused"
: `Queued - ${ordinal(i + 1)} in line`;
const labelEl = el.querySelector(".cat-queue-label");
if (labelEl && labelEl.textContent !== label) labelEl.textContent = label;
const fill = el.querySelector(".cat-progress-fill");
if (fill) fill.style.width = `${Math.round((entry.job.progress || 0) * 100)}%`;
});
}
/** The rail button only exists while there is something to look at, and carries
* the count so the queue is legible without opening it. */
function updateQueueBadge(snap = getQueueSnapshot()) {
const btn = document.querySelector(".rail-queue");
const badge = document.getElementById("queueBadge");
if (!btn) return;
const count = queueCount(snap);
btn.classList.toggle("hidden", count === 0 && catalogView !== "queue");
if (badge) {
badge.textContent = count > 99 ? "99+" : String(count);
badge.classList.toggle("hidden", count === 0);
}
}
function onQueueFrame(snap) {
updateQueueBadge(snap);
if (catalogView !== "queue") {
applyQueueDecorations(snap);
return;
}
if (queueCount(snap) === 0) {
// Nothing left to manage. Fall back to the library rather than leaving the
// user in a view that can only ever be empty from here. Deliberately not
// done inside updateQueueBadge, which render() calls -- that would recurse.
setCatalogView("library");
return;
}
updateQueueRows(snap);
}
function applyQueueDecorations(snap = getQueueSnapshot()) {
const states = queueRowStates(snap);
for (const el of document.querySelectorAll(".cat-item[data-id]")) {
const state = states.get(el.dataset.id);
// Only touch rows that are, or just were, in the queue.
if (!state && !el.classList.contains("in-queue")) continue;
decorateRow(el, state);
}
}
// ─── Catalog panel collapse ───
@@ -1418,6 +1840,7 @@ function wireCatalogRailViews() {
document.querySelector(".rail-library")?.addEventListener("click", () => setCatalogView("library"));
document.querySelector(".rail-favorites")?.addEventListener("click", () => setCatalogView("favorites"));
document.querySelector(".rail-trash")?.addEventListener("click", () => setCatalogView("trash"));
document.querySelector(".rail-queue")?.addEventListener("click", () => setCatalogView("queue"));
document.getElementById("clearBinBtn")?.addEventListener("click", () => {
const trash = getTrashFolder();
const toDelete = [...(trash?.items || [])];
@@ -2070,17 +2493,19 @@ function networkSettingsHtml() {
`;
}
// General settings: max track length (minutes) + MP4 video quality. Read live
// General settings: max track length (minutes), playlist import limit, and
// MP4 video quality. Read live
// and POSTed on change to /api/settings (same runtime store as the toggle).
async function wireGeneralSettings(overlay) {
const durInput = overlay.querySelector(".set-max-duration");
const playlistInput = overlay.querySelector(".set-playlist-max");
const heightSel = overlay.querySelector(".set-video-height");
const sampleRateSel = overlay.querySelector(".set-export-samplerate");
const portInput = overlay.querySelector(".set-port");
const deviceSel = overlay.querySelector(".set-demucs-device");
const deviceResolved = overlay.querySelector(".set-demucs-resolved");
const qualitySel = overlay.querySelector(".set-separation-quality");
if (!durInput && !heightSel && !sampleRateSel && !portInput && !deviceSel && !qualitySel) return;
if (!durInput && !playlistInput && !heightSel && !sampleRateSel && !portInput && !deviceSel && !qualitySel) return;
// Last server-confirmed device choice, to revert the select when the server
// rejects a forced device (e.g. CUDA not available on this machine).
@@ -2088,6 +2513,7 @@ async function wireGeneralSettings(overlay) {
const apply = (d) => {
if (durInput && d.max_duration_sec) durInput.value = String(Math.round(d.max_duration_sec / 60));
if (playlistInput && d.playlist_max_items) playlistInput.value = String(d.playlist_max_items);
if (heightSel && d.video_max_height) heightSel.value = String(d.video_max_height);
if (sampleRateSel && d.export_sample_rate) sampleRateSel.value = String(d.export_sample_rate);
if (portInput && d.port) portInput.value = String(d.port);
@@ -2120,6 +2546,7 @@ async function wireGeneralSettings(overlay) {
if (cleaned !== input.value) input.value = cleaned;
});
digitsOnly(durInput);
digitsOnly(playlistInput);
digitsOnly(portInput);
try {
@@ -2142,6 +2569,10 @@ async function wireGeneralSettings(overlay) {
const mins = Math.max(1, Math.min(20, parseInt(durInput.value, 10) || 20));
post({ max_duration_sec: mins * 60 });
});
playlistInput?.addEventListener("change", () => {
const items = Math.max(1, Math.min(200, parseInt(playlistInput.value, 10) || 50));
post({ playlist_max_items: items });
});
heightSel?.addEventListener("change", () => {
post({ video_max_height: parseInt(heightSel.value, 10) });
});
@@ -2482,6 +2913,13 @@ function openLibraryEditor() {
</div>
<input type="text" class="settings-num-input set-max-duration" inputmode="numeric" maxlength="2" aria-label="Max track length in minutes" />
</div>
<div class="settings-row">
<div class="settings-row-text">
<div class="settings-row-title">Playlist import limit</div>
<div class="settings-row-desc">Most tracks one playlist import will queue (max 200).</div>
</div>
<input type="text" class="settings-num-input set-playlist-max" inputmode="numeric" maxlength="3" aria-label="Playlist import limit" />
</div>
</div>
<div class="settings-section">
<div class="settings-row">
@@ -2806,6 +3244,11 @@ export async function initCatalog() {
setDisplayedVersion(currentVersion);
render();
// Patch rows in place on every queue frame. A full render() here would
// rebuild the sidebar several times a second.
onQueueChange(onQueueFrame);
onJobSettled(completeSettledJob);
startQueueStream();
loadCurrentVersion().finally(checkForUpdate);
syncWithServer();
+264 -55
View File
@@ -1,13 +1,16 @@
import {
form, urlInput, submitBtn, errorEl, jobBox, jobTitleEl, jobStageEl,
jobDetailEl, jobCancelBtn, progressEl, titleEl, bpmChip, keyChip,
eventSource, setEventSource, setCurrentJobId, currentJobId,
eventSource, setEventSource, setCurrentJobId,
foregroundJobId, setForegroundJobId,
audioEngine, multitrack,
selectedStems,
} from "./state.js";
import { destroyPlayer, wireUpAudio, setWaveformLoading, updateFooterTrack } from "./player.js";
import { stagePhrases } from "./phrases.js";
import { addTrackToLibrary, setCurrentTrack, updateTrackStatus, applyStemPresenceCards } from "./catalog.js";
import { initSections } from "./sections.js";
import { importPlaylist, looksLikePlaylist } from "./playlist.js";
// Playful stage label rotation (Claude-Code-style flair). The backend
// emits truthful stage strings; we surface them in the small #job-detail
@@ -21,6 +24,20 @@ const jobSources = new Map();
const TERMINAL_STATUSES = new Set(["done", "error", "cancelled"]);
// Last library-visible values written per job, so a 4 Hz progress stream does
// not re-run addTrackToLibrary (a full localStorage write plus a whole-sidebar
// render) for frames that change nothing the sidebar shows. Every frame carries
// the complete job state, so skipping redundant ones loses nothing: the next
// frame that does change status carries the analysis fields too.
const libraryRowKeys = new Map();
function libraryRowKey(state) {
return [state.status, state.title || "", state.thumbnail || ""].join("\u0000");
}
// `processing` here means "a submit is in flight", not "a job is running".
// With a queue the form has to come back the instant the job is accepted, so
// the user can queue the next one.
function setSubmitProcessing(processing) {
submitBtn.disabled = processing;
submitBtn.classList.toggle("loading", processing);
@@ -29,6 +46,13 @@ function setSubmitProcessing(processing) {
if (label) label.textContent = processing ? "Processing" : "Process";
}
/** True when audio is loaded in the studio. Either engine counts: the Web Audio
* path sets audioEngine, the streaming path sets multitrack, and destroyPlayer
* clears both. Read at call time so the live bindings are current. */
function studioHasTrack() {
return !!(audioEngine || multitrack);
}
function pickPhrase(status) {
const pool = stagePhrases[status] || stagePhrases.default;
return pool[Math.floor(Math.random() * pool.length)];
@@ -98,7 +122,15 @@ export function showError(message, detail, { retry = true } = {}) {
errorEl.classList.remove("hidden");
}
export function reset() {
function clearImportError() {
errorEl.classList.add("hidden");
errorEl.textContent = "";
}
// Clear the import chrome (progress box, error, phrase rotation, foreground
// SSE) without touching the studio. Split out of reset() so a submit that goes
// to the back of the queue does not tear down audio the user is playing.
function resetImportUi() {
if (eventSource) {
eventSource.close();
setEventSource(null);
@@ -106,9 +138,7 @@ export function reset() {
stopJobPolling();
stopPhraseRotation();
lastStatus = null;
destroyPlayer();
errorEl.classList.add("hidden");
errorEl.textContent = "";
clearImportError();
jobBox.classList.add("hidden");
jobCancelBtn.classList.add("hidden");
jobTitleEl.textContent = "";
@@ -116,48 +146,33 @@ export function reset() {
jobDetailEl.textContent = "";
progressEl.value = 0;
setSubmitProcessing(false);
setForegroundJobId(null);
}
export function reset() {
resetImportUi();
destroyPlayer();
setCurrentJobId(null);
}
function applyState(state) {
if (state.job_id) {
addTrackToLibrary({
id: state.job_id,
title: state.title || urlInput.value || "Processing track",
channel: state.status === "done" ? "Extracted" : "Processing",
thumb: state.thumbnail,
stems: state.selected_stems || state.stems?.map((stem) => stem.name) || [...selectedStems],
selectedStems: state.selected_stems || [...selectedStems],
audioStems: state.stems || [],
status: state.status,
duration: state.duration,
bpm: state.bpm,
key: state.key,
scale: state.scale,
keyConfidence: state.key_confidence,
lufs: state.lufs,
peakDb: state.peak_db,
stemPresence: state.stem_presence,
sourceUrl: jobSources.get(state.job_id) || urlInput.value,
createdAt: state.created_at,
});
setCurrentTrack(state.job_id);
}
if (state.title) {
jobTitleEl.textContent = state.title;
titleEl.textContent = state.title;
}
if (state.bpm) bpmChip.textContent = `${state.bpm} BPM`;
if (state.key) keyChip.textContent = state.key;
if (state.title || state.bpm || state.key || state.thumbnail) {
updateFooterTrack({
title: state.title,
thumbnail: state.thumbnail,
key: state.key,
bpm: state.bpm,
stemCount: state.stems ? state.stems.filter((s) => s.name !== "original").length : null,
});
}
// The running import no longer owns the studio: the user opened another track.
// The job keeps running and its SSE stays connected -- it still updates the
// library row -- it just stops repainting a view that is now showing something
// else. Cancel moves to the queue view, which is why the button goes away.
export function detachForegroundJob() {
if (!foregroundJobId) return;
setForegroundJobId(null);
stopPhraseRotation();
setWaveformLoading(false);
jobBox.classList.add("hidden");
jobCancelBtn.classList.add("hidden");
}
// The analysis cards under the waveform. Split out of applyState so the
// studio-owned DOM lives behind one call: only the job the user is actually
// looking at may write here, and that is far easier to see when it is one
// named function than fifty inline element lookups.
function applyStudioSummary(state) {
const summaryKey = document.getElementById("summary-key");
const summaryBpm = document.getElementById("summary-bpm");
const summaryScale = document.getElementById("summary-scale");
@@ -208,6 +223,63 @@ function applyState(state) {
if (state.stem_presence != null) {
applyStemPresenceCards(state.stem_presence);
}
}
function applyState(state) {
// Everything below the library update writes to DOM the studio owns, and
// only the import the user is actually watching may touch it. A background
// job still gets its library row updated -- that is the point of the queue.
const isForeground = !!state.job_id && state.job_id === foregroundJobId;
if (state.job_id && libraryRowKeys.get(state.job_id) !== libraryRowKey(state)) {
libraryRowKeys.set(state.job_id, libraryRowKey(state));
addTrackToLibrary({
id: state.job_id,
// urlInput only speaks for the foreground job. While a background import
// runs the user may already be typing the next URL in there, and it must
// not end up as some other track's title or source.
title: state.title || (isForeground ? urlInput.value : "") || "Processing track",
channel: state.status === "done" ? "Extracted" : "Processing",
thumb: state.thumbnail,
stems: state.selected_stems || state.stems?.map((stem) => stem.name) || [...selectedStems],
selectedStems: state.selected_stems || [...selectedStems],
audioStems: state.stems || [],
status: state.status,
duration: state.duration,
bpm: state.bpm,
key: state.key,
scale: state.scale,
keyConfidence: state.key_confidence,
lufs: state.lufs,
peakDb: state.peak_db,
stemPresence: state.stem_presence,
sourceUrl: jobSources.get(state.job_id) || (isForeground ? urlInput.value : ""),
createdAt: state.created_at,
});
}
if (state.job_id && TERMINAL_STATUSES.has(state.status)) libraryRowKeys.delete(state.job_id);
// Everything from here down is studio DOM.
if (!isForeground) return;
setCurrentTrack(state.job_id);
if (state.title) {
jobTitleEl.textContent = state.title;
titleEl.textContent = state.title;
}
if (state.bpm) bpmChip.textContent = `${state.bpm} BPM`;
if (state.key) keyChip.textContent = state.key;
if (state.title || state.bpm || state.key || state.thumbnail) {
updateFooterTrack({
title: state.title,
thumbnail: state.thumbnail,
key: state.key,
bpm: state.bpm,
stemCount: state.stems ? state.stems.filter((s) => s.name !== "original").length : null,
});
}
applyStudioSummary(state);
// Stage label is owned by the phrase-rotation timer below; we don't
// overwrite it from each SSE tick. The truthful backend stage goes
// to the small detail line instead.
@@ -224,22 +296,26 @@ function applyState(state) {
lastStatus = state.status;
}
// The three terminal branches no longer clear the submit button: it is
// released as soon as the POST returns, so the next import can be queued
// while this one is still running.
if (state.status === "error") {
stopJobPolling();
updateTrackStatus(state.job_id, "error");
setWaveformLoading(false);
showError(state.error || "Unknown error", state.error_detail);
setSubmitProcessing(false);
setForegroundJobId(null);
} else if (state.status === "cancelled") {
stopJobPolling();
updateTrackStatus(state.job_id, "cancelled");
setWaveformLoading(false);
jobBox.classList.add("hidden");
setSubmitProcessing(false);
setForegroundJobId(null);
} else if (state.status === "done") {
stopJobPolling();
updateTrackStatus(state.job_id, "done");
jobBox.classList.add("hidden");
setForegroundJobId(null);
if (!renderedJobs.has(state.job_id)) {
renderedJobs.add(state.job_id);
wireUpAudio(
@@ -254,7 +330,6 @@ function applyState(state) {
);
initSections(state.job_id, state.sections, state.duration || 0);
}
setSubmitProcessing(false);
}
}
@@ -350,7 +425,9 @@ function connectEvents(jobId) {
}
async function cancelCurrentJob() {
const id = currentJobId;
// The import, not the track in the studio. Reading currentJobId here meant
// that opening another track mid-import pointed Cancel at the wrong job.
const id = foregroundJobId;
if (!id) return;
jobCancelBtn.disabled = true;
jobCancelBtn.textContent = "Cancelling…";
@@ -366,6 +443,41 @@ async function cancelCurrentJob() {
}
}
async function postFileJob(file) {
const fd = new FormData();
fd.append("file", file);
fd.append("stems", JSON.stringify([...selectedStems]));
const res = await fetch("/api/jobs", { method: "POST", body: fd });
const data = await res.json();
if (!res.ok) throw new Error(data.detail || res.statusText);
return data.job_id;
}
/** Give an accepted upload its library row straight away, so a batch appears as
* rows the moment each file lands rather than only once every upload is done. */
function registerUploadRow(jobId, file) {
const title = sanitizeFilename(file.name);
const sourceUrl = `local:${title}`;
jobSources.set(jobId, sourceUrl);
addTrackToLibrary({
id: jobId,
title,
channel: "Processing",
thumb: "",
stems: [...selectedStems],
selectedStems: [...selectedStems],
audioStems: [],
status: "queued",
bpm: null,
key: null,
scale: null,
keyConfidence: null,
lufs: null,
peakDb: null,
sourceUrl,
});
}
function sanitizeFilename(name) {
// Strip extension, collapse whitespace, cap at 120 chars — mirrors the
// backend _sanitize_title() so title and sourceUrl match on both sides.
@@ -403,7 +515,9 @@ export async function importFromUrl(url, { title, stems } = {}) {
return null;
}
setSubmitProcessing(false);
setCurrentJobId(jobId);
setForegroundJobId(jobId);
jobSources.set(jobId, url);
// Merges into the existing library entry by sourceUrl (replaceTrackId),
// preserving its folder placement; status updates as SSE frames arrive.
@@ -439,13 +553,87 @@ export function wireJobForm() {
form.addEventListener("submit", async (e) => {
e.preventDefault();
reset();
// An import must never take a loaded studio away from the user. With a
// track playing, the new job goes straight to the background: no player
// teardown, no loading overlay, no takeover when it finishes. It reports
// progress on its library row instead.
//
// An import already holding the foreground keeps it, too. Otherwise
// queueing a second track would point the studio overlay at a job that has
// not started, and the first import -- the one about to finish -- would no
// longer be the one that loads.
const background = studioHasTrack() || !!foregroundJobId;
if (background) {
// Deliberately NOT resetImportUi(): that closes the running import's
// event stream and drops its foreground claim, which would leave the job
// about to finish with nothing listening for its completion.
clearImportError();
} else {
reset();
}
setSubmitProcessing(true);
const fileInput = document.getElementById("fileInput");
// Prefer _file cache: browsers (WKWebView, Chromium) silently clear
// fileInput.files after a fetch() submission, breaking re-submits.
const file = fileInput?._file ?? fileInput?.files?.[0] ?? null;
// A playlist is its own flow: expand, confirm the count, then queue every
// track into a folder named after it. Nothing takes the studio.
if (!file && looksLikePlaylist(urlInput.value)) {
const url = urlInput.value;
const queued = await importPlaylist(url, [...selectedStems]);
setSubmitProcessing(false);
if (queued) urlInput.value = "";
return;
}
// Several files dropped at once: upload them one after another. Parallel
// uploads of several 400 MB bodies would thrash memory on both ends, and
// the endpoint takes exactly one file per request anyway. The first one
// takes the studio if it is free, exactly as a single import would; the
// rest queue behind it.
const batch = fileInput?._files ?? null;
if (batch && batch.length > 1) {
if (!background) setWaveformLoading(true, "Uploading…");
let queued = 0;
let failure = null;
for (const item of batch) {
try {
const id = await postFileJob(item);
registerUploadRow(id, item);
queued += 1;
if (queued === 1 && !background) {
setCurrentJobId(id);
setForegroundJobId(id);
setCurrentTrack(id);
jobBox.classList.add("hidden");
jobCancelBtn.classList.add("hidden");
startPhraseRotation("queued");
lastStatus = "queued";
connectEvents(id);
}
} catch (err) {
failure = err;
break; // a full queue will reject the rest too; stop asking
}
}
setSubmitProcessing(false);
fileInput._clear?.();
if (failure) {
if (!queued && !background) {
setWaveformLoading(false);
setForegroundJobId(null);
}
showError(
`Queued ${queued} of ${batch.length} files: ${failure.message}`,
null,
{ retry: false },
);
}
return;
}
const sanitized = file ? sanitizeFilename(file.name) : null;
const sourceUrl = file ? `local:${sanitized}` : urlInput.value;
const displayTitle = sanitized ?? (urlInput.value || "Processing track");
@@ -455,9 +643,13 @@ export function wireJobForm() {
// Show overlay immediately for both paths. File uploads show "Uploading…"
// in the overlay phrase until the fetch completes and SSE takes over.
setWaveformLoading(true, file ? "Uploading…" : "");
if (file) {
lastStatus = "queued";
// Skipped entirely for a background import -- the overlay covers the
// studio, which is exactly what must not happen here.
if (!background) {
setWaveformLoading(true, file ? "Uploading…" : "");
if (file) {
lastStatus = "queued";
}
}
let fetchInit;
@@ -492,7 +684,12 @@ export function wireJobForm() {
return;
}
setCurrentJobId(jobId);
// Released here, not when the job finishes: the queue is what the button
// hands off to now, so the form is free again the moment the job exists.
setSubmitProcessing(false);
// The server has the upload; disarm the picker so the next click cannot
// silently import the same file a second time.
if (file) fileInput._clear?.();
jobSources.set(jobId, sourceUrl);
addTrackToLibrary({
id: jobId,
@@ -511,10 +708,22 @@ export function wireJobForm() {
peakDb: null,
sourceUrl,
});
if (background) {
// No per-job stream: opening one per queued import would burn through
// the browser's ~6 connections per origin and starve stem loading. The
// shared queue stream drives the row, and catalog.js completes the
// library entry when the job leaves the queue.
if (postUrlText) postUrlText.textContent = "";
return;
}
setCurrentJobId(jobId);
setForegroundJobId(jobId);
setCurrentTrack(jobId);
// Both paths: keep job box hidden, overlay drives the UI.
// Start phrase rotation now that the job exists on the server.
// Keep job box hidden, overlay drives the UI. Start phrase rotation now
// that the job exists on the server.
jobBox.classList.add("hidden");
jobCancelBtn.classList.add("hidden");
startPhraseRotation("queued");
+48 -18
View File
@@ -337,43 +337,74 @@ function wireFileDrop() {
const MAX_UPLOAD_BYTES = 400 * 1024 * 1024; // must match server _MAX_UPLOAD_BYTES
function applyFile(file) {
if (!file) return;
const lower = file.name.toLowerCase();
if (!lower.endsWith(".mp3") && !lower.endsWith(".wav") && !lower.endsWith(".flac") &&
!lower.endsWith(".mp4") && !lower.endsWith(".m4a") &&
!lower.endsWith(".ogg") && !lower.endsWith(".opus")) {
const AUDIO_EXTS = [".mp3", ".wav", ".flac", ".mp4", ".m4a", ".ogg", ".opus"];
const isAudioFile = (file) => AUDIO_EXTS.some((ext) => file.name.toLowerCase().endsWith(ext));
function applyFiles(fileList) {
const all = [...(fileList || [])];
if (!all.length) return;
// Filter here rather than letting the server reject each one: dropping a
// folder, or a folder of mixed content, would otherwise mean one 422 per
// stray file. Only complain if nothing usable came through.
const audio = all.filter(isAudioFile);
if (!audio.length) {
showError("Only MP3, WAV, FLAC, MP4, M4A, OGG, and Opus files are supported.");
return;
}
if (file.size > MAX_UPLOAD_BYTES) {
showError(`File is too large (${formatBytes(file.size)}). Maximum is 400 MB.`);
const files = audio.filter((f) => f.size <= MAX_UPLOAD_BYTES);
const oversized = audio.length - files.length;
if (!files.length) {
showError(`File is too large (${formatBytes(audio[0].size)}). Maximum is 400 MB.`);
return;
}
if (fileName) fileName.textContent = file.name;
if (fileSize) fileSize.textContent = formatBytes(file.size);
const skipped = all.length - files.length;
if (fileName) {
fileName.textContent =
files.length === 1 ? files[0].name : `${files.length} files`;
}
if (fileSize) {
const bytes = files.reduce((sum, f) => sum + f.size, 0);
fileSize.textContent = formatBytes(bytes);
}
filePill.classList.remove("hidden");
urlWrap.classList.add("has-file");
// Cache the File object directly on the element so job.js can always
// retrieve it even after the browser clears fileInput.files following
// a fetch() submission (known WKWebView / Chromium behaviour).
fileInput._file = file;
// Cache the File objects directly on the element so job.js can always
// retrieve them even after the browser clears fileInput.files following
// a fetch() submission (known WKWebView / Chromium behaviour). _file stays
// as the first one so any older single-file reader keeps working.
fileInput._files = files;
fileInput._file = files[0];
const dt = new DataTransfer();
dt.items.add(file);
for (const f of files) dt.items.add(f);
fileInput.files = dt.files;
urlInput.value = "";
urlInput.removeAttribute("required");
if (skipped > 0) {
const reason = oversized > 0 ? "too large or not audio" : "not audio";
showError(`Skipped ${skipped} file${skipped === 1 ? "" : "s"} (${reason}).`, null, {
retry: false,
});
}
}
function clearFile() {
filePill.classList.add("hidden");
urlWrap.classList.remove("has-file");
fileInput._file = null;
fileInput._files = null;
fileInput.value = "";
urlInput.setAttribute("required", "");
}
fileClear?.addEventListener("click", clearFile);
// Exposed on the element, same convention as _file above, so job.js can drop
// the selection once the upload has been handed to the server. Without it the
// chip stays armed and the (now immediately re-enabled) Process button will
// happily import the same file twice.
fileInput._clear = clearFile;
urlWrap.addEventListener("dragover", (e) => {
if (!e.dataTransfer.types.includes("Files")) return;
@@ -387,12 +418,11 @@ function wireFileDrop() {
urlWrap.addEventListener("drop", (e) => {
e.preventDefault();
urlWrap.classList.remove("drag-over");
const file = e.dataTransfer.files[0];
if (file) applyFile(file);
applyFiles(e.dataTransfer.files);
});
fileInput.addEventListener("change", () => {
if (fileInput.files[0]) applyFile(fileInput.files[0]);
applyFiles(fileInput.files);
});
}
+141
View File
@@ -0,0 +1,141 @@
// playlist.js — importing a whole playlist as one queued batch.
//
// Two steps by design. Expanding a playlist is a network round trip whose
// result the user has to agree to ("this is 47 tracks, still want it?"), and
// queueing 47 jobs from a single click with no warning is not a thing to do to
// someone's machine. The track list is never sent from here: the server expands
// the URL again when creating, so nothing outside its allowlist can be
// smuggled in between the two calls.
import { addPlaylistToLibrary } from "./catalog.js";
import { showError } from "./job.js";
/** Does this URL look like a playlist we should offer to expand? Deliberately
* permissive -- the server is the authority, this only decides which flow the
* submit button takes. */
export function looksLikePlaylist(url) {
const text = String(url || "").trim();
if (!/^https?:\/\//i.test(text)) return false;
let parsed;
try {
parsed = new URL(text);
} catch {
return false;
}
const host = parsed.hostname.toLowerCase().replace(/^(www|m|music)\./, "");
if (host === "soundcloud.com") return /^\/[^/]+\/sets\/[^/]+\/?$/.test(parsed.pathname);
if (!/^(youtube\.com|youtube-nocookie\.com)$/.test(host)) return false;
const list = parsed.searchParams.get("list");
// RD* is algorithmic radio: endless and per-viewer, so there is no set to
// import. Let it fall through to the single-track path instead.
return !!list && !/^RD/i.test(list);
}
function countLine(preview) {
const parts = [];
if (preview.skipped_unavailable) parts.push(`${preview.skipped_unavailable} unavailable`);
if (preview.skipped_too_long) parts.push(`${preview.skipped_too_long} too long`);
const overflow =
preview.total_found - preview.skipped_unavailable - preview.skipped_too_long - preview.will_queue;
if (overflow > 0) parts.push(`${overflow} that will not fit in the queue right now`);
if (preview.truncated) parts.push(`anything past the first ${preview.cap}`);
if (!parts.length) return "";
const list =
parts.length > 1 ? `${parts.slice(0, -1).join(", ")} and ${parts.at(-1)}` : parts[0];
return `Skipping ${list}.`;
}
let openDialog = null;
function closeDialog() {
openDialog?.remove();
openDialog = null;
}
function confirmImport(preview) {
return new Promise((resolve) => {
closeDialog();
const overlay = document.createElement("div");
overlay.className = "reset-confirm-backdrop playlist-confirm";
const skipped = countLine(preview);
const plural = preview.will_queue === 1 ? "track" : "tracks";
overlay.innerHTML = `
<div class="reset-confirm-card" role="dialog" aria-modal="true" aria-label="Import playlist">
<div class="reset-confirm-title playlist-confirm-title">Import this playlist?</div>
<p class="reset-confirm-body">
<strong class="playlist-name"></strong><br />
Queues <strong>${preview.will_queue}</strong> ${plural}, one at a time, into a folder
of the same name. ${skipped}
</p>
<div class="reset-confirm-actions">
<button class="reset-confirm-cancel" type="button">Cancel</button>
<button class="playlist-confirm-go" type="button">Import ${preview.will_queue} ${plural}</button>
</div>
</div>
`;
// textContent, not innerHTML: the title comes from an external service.
overlay.querySelector(".playlist-name").textContent = preview.playlist_title;
const finish = (value) => { closeDialog(); resolve(value); };
overlay.querySelector(".reset-confirm-cancel").addEventListener("click", () => finish(false));
overlay.querySelector(".playlist-confirm-go").addEventListener("click", () => finish(true));
overlay.addEventListener("mousedown", (e) => { if (e.target === overlay) finish(false); });
document.addEventListener("keydown", function onKey(e) {
if (e.code !== "Escape") return;
document.removeEventListener("keydown", onKey);
finish(false);
});
document.body.appendChild(overlay);
openDialog = overlay;
overlay.querySelector(".playlist-confirm-go").focus();
});
}
/** Expand, confirm, queue, and file the results under a folder named after the
* playlist. Returns the number of tracks queued (0 if cancelled or refused). */
export async function importPlaylist(url, stems) {
let preview;
try {
const res = await fetch("/api/playlist/preview", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ url }),
});
preview = await res.json();
if (!res.ok) throw new Error(preview.detail || res.statusText);
} catch (err) {
showError(`Could not read that playlist: ${err.message}`);
return 0;
}
if (!preview.will_queue) {
showError(
preview.capacity_left === 0
? "The queue is full. Wait for it to drain, or cancel something first."
: "Nothing in that playlist can be imported right now.",
countLine(preview) || null,
{ retry: false },
);
return 0;
}
if (!(await confirmImport(preview))) return 0;
let result;
try {
const res = await fetch("/api/playlist", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ url, stems }),
});
result = await res.json();
if (!res.ok) throw new Error(result.detail || res.statusText);
} catch (err) {
showError(`Could not import that playlist: ${err.message}`);
return 0;
}
addPlaylistToLibrary(result.playlist_title, result.jobs || []);
return result.queued || 0;
}
+236
View File
@@ -0,0 +1,236 @@
// queue.js — the import queue as the client sees it.
//
// One EventSource for the whole queue rather than one per job: browsers cap
// concurrent connections per origin at around six on HTTP/1.1 and the studio
// already spends most of that budget fetching stem WAVs, so twenty per-job
// streams would starve audio loading long before hitting any server limit.
//
// This module owns no DOM. It holds the latest snapshot and tells subscribers
// when it changes; catalog.js decides what that looks like.
const POLL_MS = 2000;
const MAX_SSE_ATTEMPTS = 6;
let snapshot = { running: null, queued: [], max_pending: 0, capacity_left: 0 };
let source = null;
let pollTimerId = null;
let attempt = 0;
const subscribers = new Set();
const settledSubscribers = new Set();
// Ids seen in the previous frame. A job that disappears has reached a terminal
// state -- done, error or cancelled -- since the snapshot only ever carries the
// running job plus those still waiting.
let lastSeenIds = new Set();
// ─── pure helpers (no DOM, no network -- the testable part) ───────────────────
export function queueCount(snap = snapshot) {
return (snap.running ? 1 : 0) + (snap.queued?.length ?? 0);
}
export function ordinal(n) {
const rem100 = n % 100;
if (rem100 >= 11 && rem100 <= 13) return `${n}th`;
const suffix = { 1: "st", 2: "nd", 3: "rd" }[n % 10] ?? "th";
return `${n}${suffix}`;
}
/** Stage text for the running row. The backend stage already carries a
* percentage during separation ("Separating 42%"), so only append one when it
* does not, rather than rendering "Separating 42% 42%". */
export function runningLabel(job) {
const stage = (job?.stage || "Working...").replace(/\.\.\.$/, "");
if (/\d\s*%/.test(stage)) return stage;
const pct = Math.round((job?.progress || 0) * 100);
return pct > 0 ? `${stage} ${pct}%` : stage;
}
/** Per-job view state, keyed by job id, for whoever is drawing rows.
* Position counts the running job, so the first waiting job is 2nd in line. */
export function isPaused(snap = snapshot) {
return !!snap.paused && queueCount(snap) > 0;
}
export async function startQueue() {
try {
const r = await fetch("/api/queue/start", { method: "POST" });
if (r.ok) publish(await r.json());
} catch (e) {
console.warn("[queue] start failed:", e);
}
}
export function queueRowStates(snap = snapshot) {
const rows = new Map();
if (snap.running) {
rows.set(snap.running.job_id, {
state: "running",
label: runningLabel(snap.running),
progress: snap.running.progress || 0,
position: 1,
});
}
const offset = snap.running ? 2 : 1;
const paused = isPaused(snap);
(snap.queued ?? []).forEach((job, i) => {
const place = i + offset;
rows.set(job.job_id, {
state: "waiting",
// A paused queue is not "2nd in line" for anything -- nothing is moving.
// Say so, or the row looks stuck.
label: paused ? "Paused" : `Queued - ${ordinal(place)} in line`,
progress: 0,
position: place,
});
});
return rows;
}
// ─── state ───────────────────────────────────────────────────────────────────
export function getQueueSnapshot() {
return snapshot;
}
export function onQueueChange(fn) {
subscribers.add(fn);
return () => subscribers.delete(fn);
}
/** Called with a job id once it leaves the queue. A background import has no
* per-job SSE stream of its own -- opening one per queued job would exhaust
* the browser's ~6 connections per origin -- so this is how the library learns
* that a job it is not watching has finished. */
export function onJobSettled(fn) {
settledSubscribers.add(fn);
return () => settledSubscribers.delete(fn);
}
export function currentIds(snap = snapshot) {
const ids = new Set();
if (snap.running) ids.add(snap.running.job_id);
for (const job of snap.queued ?? []) ids.add(job.job_id);
return ids;
}
function publish(next) {
snapshot = next;
const ids = currentIds(next);
const settled = [...lastSeenIds].filter((id) => !ids.has(id));
lastSeenIds = ids;
for (const fn of subscribers) {
try { fn(snapshot); } catch (e) { console.warn("[queue] subscriber failed:", e); }
}
for (const id of settled) {
for (const fn of settledSubscribers) {
try { fn(id); } catch (e) { console.warn("[queue] settled subscriber failed:", e); }
}
}
}
export async function refreshQueue() {
try {
const r = await fetch("/api/queue");
if (!r.ok) return;
publish(await r.json());
} catch (e) {
console.warn("[queue] refresh failed:", e);
}
}
/** Move a waiting job so it runs directly after `afterId`, or first when that
* is null. Returns true if the server accepted it.
*
* "After this job" rather than "at index N": the queue moves while the user
* drags, so an index captured at drag start can mean somewhere else by the
* time it lands. A 409 means the job started or finished mid-drag, which is
* not an error worth showing -- the snapshot that follows corrects the view.
*/
export async function reorderQueuedJob(jobId, afterId) {
try {
const r = await fetch("/api/queue/reorder", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ job_id: jobId, after: afterId ?? null }),
});
if (r.ok) {
publish(await r.json());
return true;
}
await refreshQueue();
return false;
} catch (e) {
console.warn("[queue] reorder failed:", e);
await refreshQueue();
return false;
}
}
export async function cancelQueuedJob(jobId) {
try {
await fetch(`/api/jobs/${jobId}/cancel`, { method: "POST" });
} catch (e) {
console.warn("[queue] cancel failed:", e);
}
// Do not wait for the next frame: the row should go the moment it is clicked.
await refreshQueue();
}
// ─── transport ───────────────────────────────────────────────────────────────
function startPolling() {
if (pollTimerId) return;
pollTimerId = setInterval(refreshQueue, POLL_MS);
refreshQueue();
}
function stopPolling() {
if (pollTimerId) {
clearInterval(pollTimerId);
pollTimerId = null;
}
}
export function startQueueStream() {
if (source) return;
refreshQueue(); // first paint should not wait for the stream to connect
const open = () => {
const es = new EventSource("/api/queue/events");
source = es;
es.onmessage = (ev) => {
attempt = 0;
stopPolling(); // the stream is healthy again
try {
publish(JSON.parse(ev.data));
} catch (e) {
console.warn("[queue] bad frame:", e);
}
};
es.onerror = () => {
es.close();
source = null;
attempt += 1;
if (attempt > MAX_SSE_ATTEMPTS) {
// Give up on SSE and keep the view alive by polling, same fallback
// shape as the per-job stream in job.js.
startPolling();
return;
}
setTimeout(open, 500 * Math.pow(2, attempt - 1)); // 0.5s .. 16s
};
};
open();
}
export function stopQueueStream() {
if (source) {
source.close();
source = null;
}
stopPolling();
}
+7
View File
@@ -82,6 +82,12 @@ export let multitrack = null;
// Web Audio decode-and-mix engine (Safari-safe playback). Null = legacy streaming path.
export let audioEngine = null;
export let currentJobId = null;
// The import whose progress owns the #job box and the studio view. Distinct
// from currentJobId, which is the track loaded in the studio: with a queue the
// two come apart the moment a background import runs while the user browses
// something else. A background job must not repaint the studio, and opening
// another track must not break the running import's Cancel button.
export let foregroundJobId = null;
// `mixerState` is mutated in place (never reassigned). renderMixerRow's
// closures capture each entry by reference, so on a new job we merge
@@ -154,6 +160,7 @@ export function setEventSource(v) { eventSource = v; }
export function setMultitrack(v) { multitrack = v; }
export function setAudioEngine(v) { audioEngine = v; }
export function setCurrentJobId(v) { currentJobId = v; }
export function setForegroundJobId(v) { foregroundJobId = v; }
export function setTrackIndex(v) { trackIndex = v; }
export function setTotalDuration(v) { totalDuration = v; }
export function setLoopEnabled(v) { loopEnabled = v; }
+45
View File
@@ -1,8 +1,53 @@
from __future__ import annotations
import sys
import pytest
from app.core import settings as _settings
from app.pipeline import jobqueue as _jobqueue
@pytest.fixture(autouse=True)
def _isolate_jobs_dir(tmp_path, monkeypatch):
"""Point every JOBS_DIR at a temp dir, for every test, no exceptions.
Several modules do `from app.core.config import JOBS_DIR`, which binds the
value at import time, so patching config alone is not enough. Individual
fixtures used to patch it one module at a time and any test that forgot ran
against the developer's real jobs/ directory. That is not a hypothetical:
the suite writes the registry and starts the TTL sweep through TestClient's
lifespan, and together those wiped a local library.
"""
jobs = tmp_path / "_jobs_root"
jobs.mkdir(parents=True, exist_ok=True)
import app.core.config as cfg
monkeypatch.setattr(cfg, "JOBS_DIR", jobs)
for name, module in list(sys.modules.items()):
if name.startswith("app.") and getattr(module, "JOBS_DIR", None) is not None:
monkeypatch.setattr(module, "JOBS_DIR", jobs, raising=False)
yield
@pytest.fixture(autouse=True)
def _isolate_job_queue():
"""The queue is module-global, so a job left waiting by one test would be
picked up by the next test's worker."""
from app.core import registry as _registry
_jobqueue._queue.clear()
_jobqueue._running_id = None
_jobqueue._paused = False
# restore() populates this at import time from whatever registry is on disk;
# the app lifespan drains it into the queue, which would otherwise leak a
# real job into a test's queue.
_registry._pending_resume.clear()
yield
_jobqueue._queue.clear()
_jobqueue._running_id = None
_jobqueue._paused = False
_registry._pending_resume.clear()
@pytest.fixture(autouse=True)
+90 -18
View File
@@ -7,7 +7,7 @@ from unittest.mock import patch
import pytest
from fastapi.testclient import TestClient
from app.core.config import MAX_PENDING_JOBS
from app.core.config import MAX_PENDING_UPLOAD_JOBS, MAX_PENDING_URL_JOBS
from app.core.models import Job
from app.core.registry import _jobs
@@ -22,10 +22,11 @@ def _isolate_registry():
@pytest.fixture
def client():
async def _noop_pipeline(job, url, jobs_dir):
return None
with patch("app.api.jobs.run_pipeline", _noop_pipeline):
# Stub the enqueue rather than the pipeline: these tests cover the submit
# API, and letting the real worker drain the queue would free capacity
# mid-test and break the 503 cases. Execution is covered by
# tests/test_queue_worker.py.
with patch("app.api.jobs.jobqueue.enqueue", lambda job_id: None):
from app.main import app
with TestClient(app) as c:
@@ -38,15 +39,8 @@ def upload_client(tmp_path, monkeypatch):
monkeypatch.setattr(cfg, "JOBS_DIR", tmp_path)
async def _noop_local(job, source_path, jobs_dir):
return None
async def _noop_youtube(job, url, jobs_dir):
return None
with (
patch("app.api.jobs.run_local_pipeline", _noop_local),
patch("app.api.jobs.run_pipeline", _noop_youtube),
patch("app.api.jobs.jobqueue.enqueue", lambda job_id: None),
patch("app.api.jobs._probe_duration", return_value=60.0),
):
from app.main import app
@@ -107,29 +101,107 @@ def test_cancel_after_done_is_idempotent(client):
assert _jobs[job_id].cancel_requested is False
# ─── Cancelling before a job starts ──────────────────────────────────────────
def test_cancel_while_waiting_finalises_without_running(client, tmp_path, monkeypatch):
"""A queued job used to honour cancel only when its turn arrived: it kept a
capacity slot until then, and a queued upload held its source file the whole
time."""
from app.pipeline import jobqueue
monkeypatch.setattr(jobqueue, "JOBS_DIR", tmp_path)
job = Job(id="aaaaaaaaaaaa")
_jobs[job.id] = job
(tmp_path / job.id).mkdir()
(tmp_path / job.id / "source.mp3").write_bytes(b"ID3")
# Straight into the deque: the client fixture stubs enqueue(), so calling it
# here would be a no-op.
jobqueue._queue.append(job.id)
r = client.post(f"/api/jobs/{job.id}/cancel")
assert r.status_code == 200
assert r.json()["status"] == "cancelled"
assert jobqueue.depth() == 0
assert not (tmp_path / job.id).exists(), "a queued upload must not keep its source"
def test_cancel_while_waiting_frees_a_capacity_slot(client):
from app.pipeline import jobqueue
job = Job(id="aaaaaaaaaaaa")
_jobs[job.id] = job
jobqueue._queue.append(job.id) # enqueue() is stubbed by the client fixture
client.post(f"/api/jobs/{job.id}/cancel")
assert sum(1 for j in _jobs.values() if j.status == "queued") == 0
def test_a_running_job_does_not_consume_a_queue_slot(client):
"""Capacity counts waiting jobs only, so the running one never blocks a
submit. This is what makes the limit mean "queue depth"."""
running = Job(id="aaaaaaaaaaaa")
running.status = "processing"
_jobs[running.id] = running
for _ in range(MAX_PENDING_URL_JOBS):
assert (
client.post("/api/jobs", json={"url": "https://youtu.be/dQw4w9WgXcQ"}).status_code
== 200
)
# ─── Capacity (503) ───────────────────────────────────────────────────────────
def test_youtube_503_when_queue_full(client):
for _ in range(MAX_PENDING_JOBS):
for _ in range(MAX_PENDING_URL_JOBS):
r = client.post("/api/jobs", json={"url": "https://youtu.be/dQw4w9WgXcQ"})
assert r.status_code == 200
r = client.post("/api/jobs", json={"url": "https://youtu.be/dQw4w9WgXcQ"})
assert r.status_code == 503
def test_upload_503_when_queue_full(upload_client):
for _ in range(MAX_PENDING_JOBS):
r = upload_client.post("/api/jobs", json={"url": "https://youtu.be/dQw4w9WgXcQ"})
def test_upload_503_when_upload_queue_full(upload_client):
for i in range(MAX_PENDING_UPLOAD_JOBS):
data = io.BytesIO(b"ID3" + b"\x00" * 128)
r = upload_client.post("/api/jobs", files={"file": (f"track{i}.mp3", data, "audio/mpeg")})
assert r.status_code == 200
data = io.BytesIO(b"ID3" + b"\x00" * 128)
r = upload_client.post(
"/api/jobs",
files={"file": ("track.mp3", data, "audio/mpeg")},
files={"file": ("one-too-many.mp3", data, "audio/mpeg")},
)
assert r.status_code == 503
def test_a_full_link_queue_does_not_block_an_upload(upload_client):
"""The two are bounded separately: a waiting upload holds its source file
on disk, a waiting link holds nothing, so a big playlist must not lock the
user out of importing a file."""
for _ in range(MAX_PENDING_URL_JOBS):
assert (
upload_client.post(
"/api/jobs", json={"url": "https://youtu.be/dQw4w9WgXcQ"}
).status_code
== 200
)
data = io.BytesIO(b"ID3" + b"\x00" * 128)
r = upload_client.post("/api/jobs", files={"file": ("still-fine.mp3", data, "audio/mpeg")})
assert r.status_code == 200
def test_a_full_upload_queue_does_not_block_a_link(upload_client):
for i in range(MAX_PENDING_UPLOAD_JOBS):
data = io.BytesIO(b"ID3" + b"\x00" * 128)
assert (
upload_client.post(
"/api/jobs", files={"file": (f"t{i}.mp3", data, "audio/mpeg")}
).status_code
== 200
)
r = upload_client.post("/api/jobs", json={"url": "https://youtu.be/dQw4w9WgXcQ"})
assert r.status_code == 200
# ─── File upload ─────────────────────────────────────────────────────────────
+302
View File
@@ -0,0 +1,302 @@
from __future__ import annotations
from unittest.mock import patch
import pytest
from fastapi.testclient import TestClient
from app.core.registry import _jobs
from app.pipeline import jobqueue
from app.pipeline.download import (
_ALLOWED_PLAYLIST_EXTRACTORS,
InvalidPlaylistURL,
expand_playlist,
validate_playlist_url,
)
@pytest.fixture(autouse=True)
def _isolate_registry():
_jobs.clear()
yield
_jobs.clear()
class _FakeYDL:
"""Stands in for YoutubeDL so nothing in this file touches the network.
Records the options it was constructed with so the SSRF allowlist can be
asserted on."""
last_opts: dict = {}
info: dict = {}
def __init__(self, opts):
type(self).last_opts = opts
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def extract_info(self, url, download=False):
type(self).last_url = url
return type(self).info
def _entries(n: int, *, duration: int = 200) -> list[dict]:
return [
{
"url": f"https://www.youtube.com/watch?v=vid{i:08d}",
"title": f"Track {i}",
"duration": duration,
}
for i in range(n)
]
@pytest.fixture
def fake_ydl():
_FakeYDL.info = {"title": "My Playlist", "entries": _entries(3)}
with patch("app.pipeline.download.YoutubeDL", _FakeYDL):
yield _FakeYDL
@pytest.fixture
def client(fake_ydl):
# Stub the worker so nothing actually runs; these tests cover the API.
with patch("app.pipeline.jobqueue.enqueue", lambda job_id: None):
from app.main import app
with TestClient(app) as c:
yield c
# ── URL validation ───────────────────────────────────────────────────────────
def test_accepts_a_youtube_playlist_url():
out = validate_playlist_url("https://www.youtube.com/playlist?list=PLabcdef123")
assert out == "https://www.youtube.com/playlist?list=PLabcdef123"
def test_accepts_a_watch_url_carrying_a_list():
out = validate_playlist_url("https://www.youtube.com/watch?v=abc12345678&list=PLxyz987")
assert out == "https://www.youtube.com/playlist?list=PLxyz987"
def test_accepts_a_soundcloud_set():
url = "https://soundcloud.com/someuser/sets/my-set"
assert validate_playlist_url(url) == url
@pytest.mark.parametrize(
"url",
[
"https://evil.example.com/playlist?list=PLabc",
"file:///etc/passwd",
"http://127.0.0.1:8000/playlist?list=PLabc",
"https://www.youtube.com/watch?v=abc12345678", # single video, no list
"https://www.youtube.com/playlist?list=RD1234567", # algorithmic radio
"https://soundcloud.com/someuser", # a profile, not a set
"https://soundcloud.com/someuser/a-track",
"",
],
)
def test_rejects_everything_else(url):
with pytest.raises(InvalidPlaylistURL):
validate_playlist_url(url)
def test_playlist_url_is_still_rejected_by_the_single_track_endpoint(client):
"""Importing one track and importing a playlist stay separate paths."""
r = client.post("/api/jobs", json={"url": "https://www.youtube.com/playlist?list=PLabcdef123"})
assert r.status_code == 422
# ── expansion ────────────────────────────────────────────────────────────────
def test_expansion_never_allows_the_generic_extractor(fake_ydl):
"""The #173 boundary. A playlist needs youtube:tab, which the single-video
allowlist excludes, so it has its own list -- and that list must still keep
"generic" out, or a crafted URL could make yt-dlp fetch anything."""
expand_playlist("https://www.youtube.com/playlist?list=PLabc", 50)
assert fake_ydl.last_opts["allowed_extractors"] == _ALLOWED_PLAYLIST_EXTRACTORS
assert "generic" not in fake_ydl.last_opts["allowed_extractors"]
def test_expansion_asks_for_one_past_the_cap(fake_ydl):
"""One extra entry is what makes a longer playlist distinguishable from one
that happens to end exactly at the cap."""
fake_ydl.info = {"title": "P", "entries": _entries(3)}
out = expand_playlist("https://www.youtube.com/playlist?list=PLabc", 7)
assert fake_ydl.last_opts["playlistend"] == 8
assert out["truncated"] is False
def test_expansion_reports_a_playlist_longer_than_the_cap(fake_ydl):
fake_ydl.info = {"title": "Huge", "entries": _entries(8)}
out = expand_playlist("https://www.youtube.com/playlist?list=PLabc", 7)
assert out["truncated"] is True
assert len(out["items"]) == 7, "the probe entry must not be imported"
def test_entries_are_revalidated(fake_ydl):
"""Entry URLs are data from an external service, not something to trust."""
fake_ydl.info = {
"title": "Mixed",
"entries": [
{"url": "https://www.youtube.com/watch?v=good1234567", "title": "Fine"},
{"url": "https://evil.example.com/pwn", "title": "Hostile"},
{"url": "", "title": "Deleted video"},
],
}
out = expand_playlist("https://www.youtube.com/playlist?list=PLabc", 50)
assert [i["url"] for i in out["items"]] == ["https://www.youtube.com/watch?v=good1234567"]
assert out["unavailable"] == 2
def test_playlist_title_falls_back(fake_ydl):
fake_ydl.info = {"entries": _entries(1)}
assert (
expand_playlist("https://www.youtube.com/playlist?list=PLabc", 50)["playlist_title"]
== "Playlist"
)
# ── the cap is a live setting ────────────────────────────────────────────────
def test_cap_round_trips_through_the_settings_api(client):
r = client.post("/api/settings", json={"playlist_max_items": 120})
assert r.status_code == 200
assert r.json()["playlist_max_items"] == 120
assert client.get("/api/settings").json()["playlist_max_items"] == 120
def test_cap_is_clamped_not_rejected():
from app.core import settings as settings_mod
assert settings_mod.set_playlist_max_items(0) == 1
assert settings_mod.set_playlist_max_items(99999) == 200
def test_changing_the_cap_applies_without_a_restart(client, fake_ydl):
"""Read per request, so an import right after the change honours it."""
fake_ydl.info = {"title": "Big", "entries": _entries(10)}
client.post("/api/settings", json={"playlist_max_items": 4})
body = client.post(
"/api/playlist/preview", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
).json()
assert body["cap"] == 4
assert len(body["items"]) == 4
assert body["truncated"] is True
client.post("/api/settings", json={"playlist_max_items": 50})
body = client.post(
"/api/playlist/preview", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
).json()
assert len(body["items"]) == 10
assert body["truncated"] is False
# ── preview ──────────────────────────────────────────────────────────────────
def test_preview_reports_the_shape_without_creating_jobs(client):
r = client.post(
"/api/playlist/preview", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
)
assert r.status_code == 200
body = r.json()
assert body["playlist_title"] == "My Playlist"
assert body["total_found"] == 3
assert body["will_queue"] == 3
assert _jobs == {}, "preview must not create anything"
def test_preview_counts_tracks_that_are_too_long(client, fake_ydl, monkeypatch):
import app.api.playlist as playlist_mod
monkeypatch.setattr(playlist_mod, "get_max_duration_sec", lambda: 100)
fake_ydl.info = {"title": "Long", "entries": _entries(2, duration=5000)}
body = client.post(
"/api/playlist/preview", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
).json()
assert body["skipped_too_long"] == 2
assert body["will_queue"] == 0
def test_preview_422s_on_a_non_playlist(client):
r = client.post("/api/playlist/preview", json={"url": "https://example.com/x"})
assert r.status_code == 422
# ── create ───────────────────────────────────────────────────────────────────
def test_creates_one_job_per_track_in_order(client):
body = client.post(
"/api/playlist", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
).json()
assert body["queued"] == 3
assert [j["title"] for j in body["jobs"]] == ["Track 0", "Track 1", "Track 2"]
assert len(_jobs) == 3
assert all(j.status == "queued" for j in _jobs.values())
def test_created_jobs_carry_their_title_before_downloading(client):
"""So queue rows read as track names immediately rather than URLs."""
body = client.post(
"/api/playlist", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
).json()
job = _jobs[body["jobs"][0]["job_id"]]
assert job.title == "Track 0"
assert job.source_url.startswith("https://www.youtube.com/watch?v=")
def test_returns_the_playlist_title_for_the_folder(client):
body = client.post(
"/api/playlist", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
).json()
assert body["playlist_title"] == "My Playlist"
def test_partially_fills_when_the_queue_is_nearly_full(client, fake_ydl, monkeypatch):
"""The preview already showed a number; a 503 after the user agreed is worse
than queueing what fits and saying how many did not."""
import app.api.playlist as playlist_mod
monkeypatch.setattr(playlist_mod, "MAX_PENDING_URL_JOBS", 2)
fake_ydl.info = {"title": "Big", "entries": _entries(5)}
body = client.post(
"/api/playlist", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
).json()
assert body["queued"] == 2
assert body["skipped_no_capacity"] == 3
def test_503_only_when_there_is_no_room_at_all(client, monkeypatch):
import app.api.playlist as playlist_mod
monkeypatch.setattr(playlist_mod, "_capacity_left", lambda: 0)
r = client.post("/api/playlist", json={"url": "https://www.youtube.com/playlist?list=PLabc"})
assert r.status_code == 503
def test_422_when_nothing_in_the_playlist_is_importable(client, fake_ydl):
fake_ydl.info = {"title": "Empty", "entries": []}
r = client.post("/api/playlist", json={"url": "https://www.youtube.com/playlist?list=PLabc"})
assert r.status_code == 422
def test_every_created_job_is_enqueued(client, fake_ydl):
"""Playlist jobs go through the same queue as any other import."""
seen: list[str] = []
with patch("app.pipeline.jobqueue.enqueue", seen.append):
body = client.post(
"/api/playlist", json={"url": "https://www.youtube.com/playlist?list=PLabc"}
).json()
assert seen == [j["job_id"] for j in body["jobs"]]
assert jobqueue.running_id() is None
+312
View File
@@ -0,0 +1,312 @@
from __future__ import annotations
import asyncio
import json
import pytest
from fastapi.testclient import TestClient
from app.core.config import MAX_PENDING_UPLOAD_JOBS, MAX_PENDING_URL_JOBS
from app.core.models import Job, _set
from app.core.registry import _jobs
from app.pipeline import jobqueue
@pytest.fixture(autouse=True)
def _isolate_registry():
_jobs.clear()
yield
_jobs.clear()
@pytest.fixture
def client(monkeypatch):
# No worker: these tests assert what the endpoint reports for a given queue
# state, and a live worker would drain it mid-assertion.
monkeypatch.setattr(jobqueue, "start_worker", _NoopTask)
from app.main import app
with TestClient(app) as c:
yield c
class _NoopTask:
"""Stand-in for the worker task; the lifespan only stores and discards it."""
def add_done_callback(self, _cb):
pass
def cancel(self):
pass
def _queued(job_id: str, title: str | None = None) -> Job:
job = Job(id=job_id, title=title, source_url="https://www.youtube.com/watch?v=x")
_jobs[job_id] = job
jobqueue._queue.append(job_id)
return job
# ── GET /api/queue ───────────────────────────────────────────────────────────
def test_empty_queue(client):
body = client.get("/api/queue").json()
assert body["running"] is None
assert body["queued"] == []
assert body["max_pending_urls"] == MAX_PENDING_URL_JOBS
assert body["max_pending_uploads"] == MAX_PENDING_UPLOAD_JOBS
assert body["capacity_left_urls"] == MAX_PENDING_URL_JOBS
assert body["capacity_left_uploads"] == MAX_PENDING_UPLOAD_JOBS
def test_reports_waiting_jobs_in_order_with_positions(client):
for i, jid in enumerate(("aaaaaaaaaaa1", "aaaaaaaaaaa2", "aaaaaaaaaaa3")):
_queued(jid, title=f"Track {i}")
queued = client.get("/api/queue").json()["queued"]
assert [j["job_id"] for j in queued] == ["aaaaaaaaaaa1", "aaaaaaaaaaa2", "aaaaaaaaaaa3"]
assert [j["position"] for j in queued] == [0, 1, 2]
def test_reports_the_running_job_separately(client):
job = _queued("aaaaaaaaaaa1")
jobqueue._queue.remove(job.id)
jobqueue._set_running(job.id)
_set(job, status="processing", stage="Starting...")
body = client.get("/api/queue").json()
assert body["running"]["job_id"] == job.id
assert body["running"]["status"] == "processing"
assert body["queued"] == []
def test_capacity_reflects_waiting_jobs_only(client):
_queued("aaaaaaaaaaa1")
running = Job(id="aaaaaaaaaaa9")
running.status = "processing"
_jobs[running.id] = running
body = client.get("/api/queue").json()
assert body["capacity_left_urls"] == MAX_PENDING_URL_JOBS - 1, (
"the running job must not take a slot"
)
def test_skips_a_queued_id_with_no_registry_entry(client):
"""A reset can empty the registry while ids are still in the deque."""
jobqueue._queue.append("aaaaaaaaaaa1")
assert client.get("/api/queue").json()["queued"] == []
def test_payload_is_compact(client):
"""The queue frame ships for every waiting job several times a second, so it
must not carry stems/sections/analysis."""
job = _queued("aaaaaaaaaaa1", title="T")
job.stems = [{"name": "vocals", "url": "/x"}]
job.sections = [{"id": "a"}]
rec = client.get("/api/queue").json()["queued"][0]
assert set(rec) == {
"job_id",
"status",
"progress",
"stage",
"title",
"thumbnail",
"source_url",
"error",
"position",
}
# ── GET /api/queue/events ────────────────────────────────────────────────────
# Driven directly rather than through TestClient: this stream never self-closes,
# so a TestClient.stream() context would block on exit waiting for it to end.
# Same approach as tests/test_events_stream.py.
class _Stream:
def __init__(self, it):
self._it = it
async def next_data(self, timeout: float = 2.0) -> dict:
while True:
chunk = await asyncio.wait_for(self._it.__anext__(), timeout=timeout)
if chunk.startswith("data: "):
return json.loads(chunk[len("data: ") : -2])
async def expect_no_frame(self, timeout: float = 0.7) -> None:
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(self._it.__anext__(), timeout=timeout)
async def aclose(self) -> None:
await self._it.aclose()
async def _open() -> _Stream:
from app.api.queue import queue_events
response = await queue_events()
return _Stream(response.body_iterator)
async def test_events_emits_an_initial_frame():
_queued("aaaaaaaaaaa1", title="T")
stream = await _open()
try:
frame = await stream.next_data()
assert [j["job_id"] for j in frame["queued"]] == ["aaaaaaaaaaa1"]
finally:
await stream.aclose()
async def test_events_emits_again_when_a_job_changes():
job = _queued("aaaaaaaaaaa1", title="T")
stream = await _open()
try:
await stream.next_data()
_set(job, progress=0.5, stage="Separating")
frame = await stream.next_data()
assert frame["queued"][0]["progress"] == 0.5
finally:
await stream.aclose()
async def test_events_does_not_repeat_an_unchanged_queue():
_queued("aaaaaaaaaaa1", title="T")
stream = await _open()
try:
await stream.next_data()
await stream.expect_no_frame()
finally:
await stream.aclose()
async def test_events_stays_open_when_the_queue_empties():
"""Unlike the per-job stream, this one outlives any single job: it is the
session-long view, so an empty queue must not end it."""
job = _queued("aaaaaaaaaaa1", title="T")
stream = await _open()
try:
await stream.next_data()
jobqueue._queue.remove(job.id)
frame = await stream.next_data()
assert frame["queued"] == []
await stream.expect_no_frame() # still open, just idle
finally:
await stream.aclose()
async def test_events_503_at_the_shared_connection_cap(monkeypatch):
"""The budget is shared with the per-job stream rather than each having its
own, so twenty queue streams cannot starve job streams."""
from fastapi import HTTPException
import app.api.events as events_mod
from app.api.queue import queue_events
monkeypatch.setattr(events_mod, "_sse_active", events_mod._MAX_SSE_CONNECTIONS)
with pytest.raises(HTTPException) as exc:
await queue_events()
assert exc.value.status_code == 503
async def test_events_releases_its_slot_on_close():
"""The slot is released in the generator's finally, so the generator has to
have started -- pull a frame first, as Starlette's response iteration does."""
import app.api.events as events_mod
before = events_mod._sse_active
stream = await _open()
assert events_mod._sse_active == before + 1
await stream.next_data()
await stream.aclose()
assert events_mod._sse_active == before
# ── pausing ──────────────────────────────────────────────────────────────────
def test_snapshot_reports_the_paused_flag(client):
_queued("aaaaaaaaaaa1")
assert client.get("/api/queue").json()["paused"] is False
jobqueue.pause()
try:
assert client.get("/api/queue").json()["paused"] is True
finally:
jobqueue.resume()
def test_start_endpoint_resumes_the_queue(client):
_queued("aaaaaaaaaaa1")
jobqueue.pause()
try:
body = client.post("/api/queue/start").json()
assert body["paused"] is False
assert jobqueue.is_paused() is False
finally:
jobqueue.resume()
def test_start_is_harmless_when_nothing_is_paused(client):
assert client.post("/api/queue/start").status_code == 200
assert jobqueue.is_paused() is False
# ── reordering ───────────────────────────────────────────────────────────────
def test_reorder_moves_a_job_after_another(client):
for jid in ("aaaaaaaaaaa1", "aaaaaaaaaaa2", "aaaaaaaaaaa3"):
_queued(jid)
body = client.post(
"/api/queue/reorder", json={"job_id": "aaaaaaaaaaa3", "after": "aaaaaaaaaaa1"}
).json()
assert [j["job_id"] for j in body["queued"]] == [
"aaaaaaaaaaa1",
"aaaaaaaaaaa3",
"aaaaaaaaaaa2",
]
def test_reorder_to_the_front(client):
for jid in ("aaaaaaaaaaa1", "aaaaaaaaaaa2", "aaaaaaaaaaa3"):
_queued(jid)
body = client.post("/api/queue/reorder", json={"job_id": "aaaaaaaaaaa3"}).json()
assert [j["job_id"] for j in body["queued"]][0] == "aaaaaaaaaaa3"
def test_reorder_renumbers_positions(client):
for jid in ("aaaaaaaaaaa1", "aaaaaaaaaaa2", "aaaaaaaaaaa3"):
_queued(jid)
body = client.post("/api/queue/reorder", json={"job_id": "aaaaaaaaaaa3"}).json()
assert [j["position"] for j in body["queued"]] == [0, 1, 2]
def test_reorder_409s_for_a_job_that_is_no_longer_waiting(client):
"""The user dragged a row that finished or started mid-drag."""
_queued("aaaaaaaaaaa1")
r = client.post("/api/queue/reorder", json={"job_id": "aaaaaaaaaaa9"})
assert r.status_code == 409
def test_reorder_rejects_a_job_following_itself(client):
_queued("aaaaaaaaaaa1")
r = client.post("/api/queue/reorder", json={"job_id": "aaaaaaaaaaa1", "after": "aaaaaaaaaaa1"})
assert r.status_code == 422
def test_reorder_404s_on_a_malformed_id(client):
r = client.post("/api/queue/reorder", json={"job_id": "../../etc/passwd"})
assert r.status_code == 404
def test_a_lost_anchor_puts_the_job_last_rather_than_failing(client):
"""The anchor finished while the row was being dragged onto it."""
for jid in ("aaaaaaaaaaa1", "aaaaaaaaaaa2"):
_queued(jid)
body = client.post(
"/api/queue/reorder", json={"job_id": "aaaaaaaaaaa1", "after": "aaaaaaaaaaa8"}
).json()
assert [j["job_id"] for j in body["queued"]] == ["aaaaaaaaaaa2", "aaaaaaaaaaa1"]
+320
View File
@@ -0,0 +1,320 @@
from __future__ import annotations
import asyncio
import contextlib
import pytest
from app.core.models import Job
from app.core.registry import _jobs
from app.pipeline import jobqueue
@pytest.fixture(autouse=True)
def _isolate_registry():
_jobs.clear()
yield
_jobs.clear()
@pytest.fixture
async def worker(monkeypatch, tmp_path):
"""A running queue worker, torn down after the test."""
monkeypatch.setattr(jobqueue, "JOBS_DIR", tmp_path)
task = jobqueue.start_worker()
yield task
jobqueue.request_stop()
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
def _job(job_id: str, url: str = "https://www.youtube.com/watch?v=dQw4w9WgXcQ") -> Job:
job = Job(id=job_id, source_url=url)
_jobs[job_id] = job
return job
async def _drain(predicate, timeout: float = 3.0) -> None:
"""Wait for the worker to reach a state instead of sleeping a fixed amount."""
deadline = asyncio.get_running_loop().time() + timeout
while asyncio.get_running_loop().time() < deadline:
if predicate():
return
await asyncio.sleep(0.01)
raise AssertionError("timed out waiting for the queue")
async def test_runs_a_queued_job(worker, monkeypatch):
ran = []
async def _fake(job, url, jobs_dir):
ran.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
_job("aaaaaaaaaaaa")
jobqueue.enqueue("aaaaaaaaaaaa")
await _drain(lambda: ran == ["aaaaaaaaaaaa"])
async def test_runs_in_fifo_order(worker, monkeypatch):
ran = []
async def _fake(job, url, jobs_dir):
ran.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
ids = ["aaaaaaaaaaa1", "aaaaaaaaaaa2", "aaaaaaaaaaa3"]
for jid in ids:
_job(jid)
jobqueue.enqueue(jid)
await _drain(lambda: len(ran) == 3)
assert ran == ids
async def test_never_runs_two_jobs_at_once(worker, monkeypatch):
"""The demucs worker in separate.py is a single shared process, so overlap
would interleave two jobs on one stdin. This is the guarantee that stops it."""
active = 0
max_active = 0
async def _fake(job, url, jobs_dir):
nonlocal active, max_active
active += 1
max_active = max(max_active, active)
await asyncio.sleep(0.05) # a real pipeline yields; so must this
active -= 1
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
for jid in ("aaaaaaaaaaa1", "aaaaaaaaaaa2", "aaaaaaaaaaa3"):
_job(jid)
jobqueue.enqueue(jid)
await _drain(lambda: active == 0 and max_active == 1 and jobqueue.depth() == 0)
assert max_active == 1
async def test_a_raising_job_does_not_kill_the_worker(worker, monkeypatch):
ran = []
async def _fake(job, url, jobs_dir):
ran.append(job.id)
if job.id.endswith("1"):
raise RuntimeError("boom")
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
for jid in ("aaaaaaaaaaa1", "aaaaaaaaaaa2"):
_job(jid)
jobqueue.enqueue(jid)
await _drain(lambda: ran == ["aaaaaaaaaaa1", "aaaaaaaaaaa2"])
async def test_marks_the_job_processing_when_picked_up(worker, monkeypatch):
seen: list[str] = []
async def _fake(job, url, jobs_dir):
seen.append(job.status)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
_job("aaaaaaaaaaaa")
jobqueue.enqueue("aaaaaaaaaaaa")
await _drain(lambda: seen == ["processing"])
async def test_skips_a_job_cancelled_while_waiting(worker, monkeypatch):
ran = []
async def _fake(job, url, jobs_dir):
ran.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
job = _job("aaaaaaaaaaaa")
job.cancel_requested = True
jobqueue.enqueue("aaaaaaaaaaaa")
await _drain(lambda: jobqueue.depth() == 0)
assert ran == []
async def test_skips_a_job_that_vanished_from_the_registry(worker):
jobqueue.enqueue("aaaaaaaaaaaa") # never registered
await _drain(lambda: jobqueue.depth() == 0)
async def test_routes_local_jobs_to_the_local_pipeline(worker, monkeypatch, tmp_path):
ran = []
async def _fake_local(job, source, jobs_dir):
ran.append(("local", job.id, source.name))
async def _fake_yt(job, url, jobs_dir):
ran.append(("youtube", job.id, url))
monkeypatch.setattr("app.pipeline.runner.run_local_pipeline", _fake_local)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake_yt)
(tmp_path / "aaaaaaaaaaaa").mkdir()
(tmp_path / "aaaaaaaaaaaa" / "source.mp3").write_bytes(b"ID3")
_job("aaaaaaaaaaaa", url="local:My Song")
jobqueue.enqueue("aaaaaaaaaaaa")
await _drain(lambda: ran == [("local", "aaaaaaaaaaaa", "source.mp3")])
async def test_a_local_job_with_no_source_errors_rather_than_crashing(
worker, monkeypatch, tmp_path
):
"""The upload can be gone after a restart; that must not take the queue down."""
called = []
async def _fake_local(job, source, jobs_dir):
called.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_local_pipeline", _fake_local)
job = _job("aaaaaaaaaaaa", url="local:Missing")
jobqueue.enqueue("aaaaaaaaaaaa")
await _drain(lambda: job.status == "error")
assert called == []
assert "no longer available" in (job.error or "")
async def test_request_stop_drains_nothing_further(worker, monkeypatch):
ran = []
async def _fake(job, url, jobs_dir):
ran.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
jobqueue.request_stop()
_job("aaaaaaaaaaaa")
jobqueue.enqueue("aaaaaaaaaaaa")
await asyncio.sleep(0.1)
assert ran == []
# ── queue bookkeeping (no worker needed) ──
def test_discard_removes_a_waiting_job():
jobqueue.enqueue("aaaaaaaaaaa1")
# Not inside the assert: discard() mutates the queue, and an assert can
# be stripped (python -O), which would silently skip the thing under test.
removed = jobqueue.discard("aaaaaaaaaaa1")
assert removed is True
assert jobqueue.depth() == 0
def test_discard_reports_false_for_an_unknown_job():
removed = jobqueue.discard("aaaaaaaaaaa9")
assert removed is False
def test_enqueue_is_idempotent():
jobqueue.enqueue("aaaaaaaaaaa1")
jobqueue.enqueue("aaaaaaaaaaa1")
assert jobqueue.depth() == 1
def test_snapshot_reports_running_and_waiting():
jobqueue.enqueue("aaaaaaaaaaa1")
jobqueue.enqueue("aaaaaaaaaaa2")
running, waiting = jobqueue.snapshot()
assert running is None
assert waiting == ["aaaaaaaaaaa1", "aaaaaaaaaaa2"]
def test_clear_empties_the_queue():
jobqueue.enqueue("aaaaaaaaaaa1")
jobqueue.clear()
assert jobqueue.depth() == 0
# ── pausing ──────────────────────────────────────────────────────────────────
async def test_a_paused_queue_does_not_start_anything(worker, monkeypatch):
"""Opening the app must not put the machine to work on its own."""
ran = []
async def _fake(job, url, jobs_dir):
ran.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
jobqueue.pause()
_job("aaaaaaaaaaaa")
jobqueue.enqueue("aaaaaaaaaaaa", autostart=False)
await asyncio.sleep(0.3)
assert ran == []
assert jobqueue.depth() == 1, "the job is still queued, just not started"
assert jobqueue.is_paused() is True
async def test_resume_starts_the_queue(worker, monkeypatch):
ran = []
async def _fake(job, url, jobs_dir):
ran.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
jobqueue.pause()
_job("aaaaaaaaaaaa")
jobqueue.enqueue("aaaaaaaaaaaa", autostart=False)
await asyncio.sleep(0.2)
assert ran == []
jobqueue.resume()
await _drain(lambda: ran == ["aaaaaaaaaaaa"])
assert jobqueue.is_paused() is False
async def test_a_new_import_lifts_the_pause(worker, monkeypatch):
"""Pressing Process and having nothing happen would be its own bug, so an
explicit submit starts the queue -- and drains the restored jobs first."""
ran = []
async def _fake(job, url, jobs_dir):
ran.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
jobqueue.pause()
_job("aaaaaaaaaaaa") # restored from the last session
jobqueue.enqueue("aaaaaaaaaaaa", autostart=False)
await asyncio.sleep(0.2)
assert ran == []
_job("bbbbbbbbbbbb") # the user imports something new
jobqueue.enqueue("bbbbbbbbbbbb")
await _drain(lambda: ran == ["aaaaaaaaaaaa", "bbbbbbbbbbbb"])
assert jobqueue.is_paused() is False
async def test_pausing_does_not_touch_a_running_job(worker, monkeypatch):
started = asyncio.Event()
release = asyncio.Event()
finished = []
async def _fake(job, url, jobs_dir):
started.set()
await release.wait()
finished.append(job.id)
monkeypatch.setattr("app.pipeline.runner.run_pipeline", _fake)
_job("aaaaaaaaaaaa")
jobqueue.enqueue("aaaaaaaaaaaa")
await asyncio.wait_for(started.wait(), timeout=3)
jobqueue.pause()
release.set()
await _drain(lambda: finished == ["aaaaaaaaaaaa"])
async def test_start_worker_begins_unpaused(worker):
assert jobqueue.is_paused() is False
+181
View File
@@ -6,6 +6,7 @@ from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from app.core import registry as _registry
from app.core.models import Job
from app.core.registry import _jobs
from app.core.registry import persist as persist_registry
@@ -15,8 +16,153 @@ from app.core.registry import restore as restore_registry
@pytest.fixture(autouse=True)
def _isolate_registry():
_jobs.clear()
_registry._pending_resume.clear()
yield
_jobs.clear()
_registry._pending_resume.clear()
# ── resuming a queue across a restart ────────────────────────────────────────
def _stems_dir(tmp_path: Path, job_id: str, names=("vocals", "drums")) -> None:
d = tmp_path / job_id / "stems"
d.mkdir(parents=True, exist_ok=True)
for n in names:
(d / f"{n}.wav").write_bytes(b"RIFF")
def test_a_queued_job_survives_a_restart(tmp_path: Path):
"""Only done jobs used to be persisted, so closing the app silently threw
away everything the user had queued."""
job = Job(id="abcdef000001", status="queued", title="Waiting", source_url="local:Waiting")
_jobs[job.id] = job
persist_registry(tmp_path)
_jobs.clear()
restore_registry(tmp_path)
assert _jobs[job.id].status == "queued"
assert _registry.take_pending_resume() == [job.id]
def test_take_pending_resume_only_fires_once(tmp_path: Path):
_jobs["abcdef000001"] = Job(id="abcdef000001", status="queued", title="Waiting")
persist_registry(tmp_path)
_jobs.clear()
restore_registry(tmp_path)
assert _registry.take_pending_resume() == ["abcdef000001"]
assert _registry.take_pending_resume() == []
def test_an_interrupted_job_whose_stems_landed_is_done_not_rerun(tmp_path: Path):
"""The crash window between the last stem being written and the done-persist.
Re-running would duplicate the library entry and redo the whole separation."""
job = Job(id="abcdef000002", status="separating", title="Nearly done")
_jobs[job.id] = job
persist_registry(tmp_path)
_stems_dir(tmp_path, job.id)
_jobs.clear()
restore_registry(tmp_path)
assert _jobs[job.id].status == "done"
assert _registry.take_pending_resume() == []
def test_an_interrupted_job_without_stems_is_requeued(tmp_path: Path):
job = Job(id="abcdef000003", status="separating", title="Half done", progress=0.6)
_jobs[job.id] = job
persist_registry(tmp_path)
_jobs.clear()
restore_registry(tmp_path)
restored = _jobs[job.id]
assert restored.status == "queued"
assert restored.progress == 0.0
assert restored.resume_attempts == 1
assert _registry.take_pending_resume() == [job.id]
def test_partial_demucs_output_is_cleared_before_a_resume(tmp_path: Path):
"""collect() would otherwise mistake a half-written model dir for results."""
from app.core.config import DEMUCS_MODEL
job = Job(id="abcdef000004", status="separating", title="Half done")
_jobs[job.id] = job
persist_registry(tmp_path)
partial = tmp_path / job.id / DEMUCS_MODEL / "track"
partial.mkdir(parents=True)
(partial / "vocals.wav").write_bytes(b"partial")
_jobs.clear()
restore_registry(tmp_path)
assert not (tmp_path / job.id / DEMUCS_MODEL).exists()
def test_a_crash_loop_ends_without_the_job_ever_completing(tmp_path: Path):
"""The counter has to reach disk during restore, not only when the job
finally finishes. A job that takes the process down never gets that far, so
if restore did not persist, every start would read resume_attempts back as 0
and retry it forever."""
job = Job(id="abcdef000006", status="separating", title="Poison")
_jobs[job.id] = job
persist_registry(tmp_path)
# Crash 1: nothing ran, nothing else persisted -- just restart.
_jobs.clear()
_registry._pending_resume.clear()
restore_registry(tmp_path)
assert _jobs[job.id].status == "queued"
# Crash 2: the retry took the process down again, still with no persist of
# its own. The restart must read the bumped count off disk and stop.
_jobs.clear()
_registry._pending_resume.clear()
restore_registry(tmp_path)
assert _jobs[job.id].status == "error"
assert _registry.take_pending_resume() == []
def test_a_job_interrupted_twice_fails_instead_of_looping(tmp_path: Path):
"""A job that reliably takes the process down would otherwise be re-queued
on every start, wedging the queue forever."""
job = Job(id="abcdef000005", status="separating", title="Poison", resume_attempts=1)
_jobs[job.id] = job
persist_registry(tmp_path)
_jobs.clear()
restore_registry(tmp_path)
assert _jobs[job.id].status == "error"
assert "again" in (_jobs[job.id].error or "")
assert _registry.take_pending_resume() == []
def test_resumed_jobs_keep_their_original_order(tmp_path: Path):
for i, jid in enumerate(("abcdef00000a", "abcdef00000b", "abcdef00000c")):
_jobs[jid] = Job(id=jid, status="queued", title=f"T{i}", created_at=100.0 + i)
persist_registry(tmp_path)
_jobs.clear()
restore_registry(tmp_path)
assert _registry.take_pending_resume() == ["abcdef00000a", "abcdef00000b", "abcdef00000c"]
def test_cancelled_and_errored_jobs_are_still_not_persisted(tmp_path: Path):
"""Widening persistence to cover the queue must not resurrect dead jobs."""
_jobs["abcdef000006"] = Job(id="abcdef000006", status="cancelled", title="Nope")
_jobs["abcdef000007"] = Job(id="abcdef000007", status="error", title="Nope")
persist_registry(tmp_path)
_jobs.clear()
restore_registry(tmp_path)
assert _jobs == {}
def test_persist_and_restore_terminal_job(tmp_path: Path):
@@ -174,3 +320,38 @@ def test_delete_updates_persisted_registry(tmp_path: Path, monkeypatch):
assert not job_dir.exists()
data = json.loads((tmp_path / "registry.json").read_text(encoding="utf-8"))
assert data["jobs"] == []
def test_a_reordered_queue_comes_back_in_the_users_order(tmp_path: Path):
"""Order is the main thing a user controls in a serial queue, so it has to
survive a restart -- not fall back to submission order."""
from app.pipeline import jobqueue
ids = ["abcdef00000a", "abcdef00000b", "abcdef00000c"]
for i, jid in enumerate(ids):
job = Job(id=jid, status="queued", title=jid[-1], created_at=1000.0 + i)
_jobs[jid] = job
jobqueue.enqueue(jid)
# The user drags the last one to the front.
assert jobqueue.reorder("abcdef00000c", None) is True
persist_registry(tmp_path)
_jobs.clear()
_registry._pending_resume.clear()
jobqueue._queue.clear()
restore_registry(tmp_path)
assert _registry.take_pending_resume() == ["abcdef00000c", "abcdef00000a", "abcdef00000b"]
def test_records_without_a_position_still_restore_oldest_first(tmp_path: Path):
"""Registries written before reordering existed default to 0, where
created_at decides exactly as it used to."""
for i, jid in enumerate(["abcdef0000e1", "abcdef0000e2"]):
_jobs[jid] = Job(id=jid, status="queued", title=jid, created_at=2000.0 - i)
persist_registry(tmp_path)
_jobs.clear()
_registry._pending_resume.clear()
restore_registry(tmp_path)
assert _registry.take_pending_resume() == ["abcdef0000e2", "abcdef0000e1"]
+131
View File
@@ -0,0 +1,131 @@
"""The separation worker must not outlive whoever spawned it.
Closing StemDeck has to leave nothing behind. The worker holds the GPU and is
the one child that can run for minutes, so it cannot depend on the parent
getting a chance to clean up: Force Quit, Task Manager and a crash all skip
that entirely.
"""
from __future__ import annotations
import os
import subprocess
import sys
import textwrap
import time
from app.core.process import process_exists
def test_process_exists_reports_a_live_process():
assert process_exists(os.getpid()) is True
def test_process_exists_reports_a_dead_process():
proc = subprocess.Popen([sys.executable, "-c", "pass"])
proc.wait()
# A pid can be recycled, but not within the moment after a wait().
assert process_exists(proc.pid) is False
def test_process_exists_rejects_nonsense_pids():
assert process_exists(0) is False
assert process_exists(-1) is False
def test_worker_exits_when_its_parent_disappears(tmp_path):
"""The watchdog in isolation: same code path, without torch in the way.
Spawning the real worker would load demucs (seconds, and a model download on
a clean machine), so this drives _arm_parent_watchdog directly with a stand
-in parent it can kill.
"""
parent = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"])
script = textwrap.dedent(
"""
import sys, time
from app.pipeline.demucs_worker import _arm_parent_watchdog
_arm_parent_watchdog()
# Busy the way a separation is busy: never reading stdin, so only the
# watchdog can end this process.
while True:
time.sleep(0.1)
"""
)
env = {**os.environ, "STEMDECK_PARENT_PID": str(parent.pid)}
worker = subprocess.Popen(
[sys.executable, "-c", script],
env=env,
cwd=os.getcwd(),
stderr=subprocess.DEVNULL,
)
try:
time.sleep(2)
assert worker.poll() is None, "worker should still be running while the parent lives"
parent.kill()
parent.wait()
deadline = time.time() + 20
while time.time() < deadline:
if worker.poll() is not None:
break
time.sleep(0.25)
assert worker.poll() is not None, "worker outlived its parent"
finally:
if worker.poll() is None:
worker.kill()
if parent.poll() is None:
parent.kill()
def test_worker_ignores_an_unset_or_bogus_parent_pid(monkeypatch):
"""A worker run by hand (no STEMDECK_PARENT_PID) must not arm the watchdog
and shoot itself."""
from app.pipeline import demucs_worker
started: list[object] = []
monkeypatch.setattr(
demucs_worker.threading,
"Thread",
lambda *a, **k: started.append((a, k)) or _NoopThread(),
)
for value in ("", " ", "not-a-number", "0", "-5", str(os.getpid())):
monkeypatch.setenv("STEMDECK_PARENT_PID", value)
demucs_worker._arm_parent_watchdog()
assert started == [], "watchdog armed on a pid it should have ignored"
class _NoopThread:
def start(self):
pass
def test_the_worker_is_spawned_with_the_parent_pid(monkeypatch):
"""The watchdog is only armed if separate.py actually passes the pid."""
import app.pipeline.separate as separate
captured: dict = {}
class _FakeProc:
stdin = None
stderr = None
def poll(self):
return None
def fake_popen(cmd, **kwargs):
captured.update(kwargs)
return _FakeProc()
monkeypatch.setattr(separate.subprocess, "Popen", fake_popen)
monkeypatch.setattr(separate, "_kill_worker", lambda: None)
separate._worker.clear()
try:
separate._get_worker("cpu")
finally:
separate._worker.clear()
assert captured["env"]["STEMDECK_PARENT_PID"] == str(os.getpid())