fix(tui): give tokio workers the runtime stack the owner thread already had

`#[tokio::main]` expanded to a multi-thread builder with no thread_stack_size,
so every worker carried tokio's 2 MiB default while only the `codewhale-main`
owner thread received CODEWHALE_MAIN_STACK_BYTES. The engine does not run on
that owner thread: spawn_engine -> spawn_supervised -> tokio::spawn puts
Engine::run on a worker, so the explicit stack never applied where the depth
actually is.

A debug-build `agent` dispatch (turn_loop -> FuturesUnordered ->
execute_full_with_context -> AgentTool::execute -> spawn_subagent_from_input)
measured a stack high-water mark between 2.25 and 2.5 MiB and aborted the
process on the guard page. A Rust stack overflow is not a panic — it raises
SIGABRT — so spawn_supervised's catch_unwind could not see it and the process
died with 134 mid-dispatch, before any child request was issued.

This is the release_runtime_qa blocker: both fleet tests observed zero child
requests and then timed out against a corpse, while the retained PTY frame kept
repainting the last screen the TUI drew before it died.

Evidence: dose-response on RUST_MIN_STACK against one binary — 2 MiB FAIL
(31.2s), 2359296 FAIL, 2621440 ok (2.3s), 3/4/8/16 MiB ok. Bounded threshold,
so not recursion. 26 macOS crash reports, every one faulting on tokio-rt-worker
in a Stack Guard between two 2080K stack regions.

Ruled out and recorded so they are not re-chased: "Overwriting existing tool:
File" is registry build-time last-write-wins, fires ~49ms earlier, appears in
passing runs, and is byte-identical at the base commit; the `agent` tool is
registered and model-visible throughout; the #3095 launch gate cannot starve
either test.

Also stops the harness reporting a dead child as a hang, and adds a regression
test that asserts the invariant the default violated — dispatching `agent` must
not kill the process — instead of a counter that a dead process also fails.

Receipt: cargo test -p codewhale-tui --test release_runtime_qa
  -> 20 passed; 0 failed; 1 ignored (was 17 passed; 2 failed)
This commit is contained in:
Hmbown
2026-08-03 18:01:54 -07:00
parent dc2d99e122
commit b22393c672
2 changed files with 133 additions and 2 deletions
+41 -2
View File
@@ -1439,8 +1439,47 @@ fn main() -> Result<()> {
}
}
#[tokio::main]
async fn run_async_main(
fn run_async_main(
cli: Cli,
command: Option<Commands>,
plugin_discovery: Arc<crate::plugins::PluginDiscoveryContext>,
plugin_registry: Arc<crate::plugins::PluginRegistry>,
) -> Result<()> {
build_runtime()?.block_on(run_async_main_inner(
cli,
command,
plugin_discovery,
plugin_registry,
))
}
/// Build the runtime that owns every async task in this binary.
///
/// `#[tokio::main]` used to expand here, which left every worker thread on
/// tokio's 2 MiB default while only the `codewhale-main` owner thread above
/// received `CODEWHALE_MAIN_STACK_BYTES`. The engine does not run on that owner
/// thread — `core::engine::spawn_engine` hands `Engine::run` to
/// `utils::spawn_supervised`, a bare `tokio::spawn` — so the explicit stack
/// never applied where the depth actually is.
///
/// A debug-build `agent` dispatch (turn_loop -> FuturesUnordered ->
/// execute_full_with_context -> AgentTool::execute -> spawn_subagent_from_input)
/// measured a stack high-water mark between 2.25 and 2.5 MiB and aborted the
/// whole process on the guard page. A Rust stack overflow is not a panic: it
/// raises SIGABRT, so `spawn_supervised`'s `catch_unwind` cannot see it and the
/// process dies with 134 mid-dispatch.
///
/// This is behavior-identical to the old `#[tokio::main]` expansion apart from
/// the stack size, and it makes the knob greppable.
pub(crate) fn build_runtime() -> Result<tokio::runtime::Runtime> {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.thread_stack_size(CODEWHALE_MAIN_STACK_BYTES)
.build()
.context("Failed to build the Codewhale Tokio runtime")
}
async fn run_async_main_inner(
cli: Cli,
command: Option<Commands>,
plugin_discovery: Arc<crate::plugins::PluginDiscoveryContext>,
+92
View File
@@ -260,6 +260,17 @@ fn wait_for_counter(
if counter.load(Ordering::SeqCst) >= expected {
return Ok(());
}
// A dead child renders as a hang: `pump()` only feeds *new* bytes into a
// retained frame, so `debug_dump()` keeps painting the last frame the TUI
// drew before it died. Without this probe a SIGABRT (stack overflow aborts
// as 134) burns the whole timeout and then reports a live-looking screen.
if let Some(code) = harness.wait_for_exit(Duration::from_millis(0)) {
return Err(anyhow!(
"TUI exited with {code} before the counter reached {expected}; observed {}\n{}",
counter.load(Ordering::SeqCst),
harness.debug_dump()
));
}
if Instant::now() >= deadline {
return Err(anyhow!(
"counter did not reach {expected} within {timeout:?}; observed {}\n{}",
@@ -697,6 +708,87 @@ async fn release_six_worker_fanout_keeps_typing_render_and_esc_cancel_live() ->
Ok(())
}
struct SingleDispatchResponder {
child_requests: Arc<AtomicUsize>,
}
impl Respond for SingleDispatchResponder {
fn respond(&self, request: &Request) -> ResponseTemplate {
let body = request.body_json::<Value>().unwrap_or(Value::Null);
let raw = body.to_string();
if raw.contains("stay busy worker") && !raw.contains("dispatch one QA worker") {
self.child_requests.fetch_add(1, Ordering::SeqCst);
return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "child-acknowledged"));
}
if raw.contains("dispatch one QA worker") {
return sse_response(fanout_tool_call_sse_n(1));
}
sse_response(text_sse(DEEPSEEK_TEST_MODEL, "unexpected-request"))
}
}
/// Dispatching `agent` must not kill the process.
///
/// Regression guard for the 0.9.4 release blocker: the Tokio runtime was built
/// by `#[tokio::main]`, so every worker thread carried tokio's 2 MiB default
/// while only the `codewhale-main` owner thread got `CODEWHALE_MAIN_STACK_BYTES`.
/// The engine runs on a worker, and a debug-build `agent` dispatch measured a
/// stack high-water mark between 2.25 and 2.5 MiB — it overflowed the guard page
/// and aborted the process with 134, mid-dispatch, before any child request was
/// ever issued.
///
/// This asserts the invariant that the default violated (the process survives an
/// `agent` dispatch) rather than re-asserting a child counter, which a dead
/// process also fails — but fails slowly and for the wrong stated reason.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn release_agent_dispatch_never_aborts_the_runtime() -> Result<()> {
let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await;
let server = MockServer::start().await;
mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await;
let child_requests = Arc::new(AtomicUsize::new(0));
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(SingleDispatchResponder {
child_requests: Arc::clone(&child_requests),
})
.mount(&server)
.await;
let ws = make_sealed_workspace()?;
std::fs::write(
ws.home().join(".codewhale").join("config.toml"),
"[subagents]\nmax_concurrent = 1\nlaunch_concurrency = 1\nmax_admitted = 1\n",
)?;
let mut tui = common_tui_builder(&ws)
.env("CODEWHALE_PROVIDER", "deepseek")
.env("DEEPSEEK_API_KEY", "deepseek-local-test-key")
.env("DEEPSEEK_BASE_URL", server.uri())
.env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL)
.args(["--yolo", "--max-subagents", "1"])
.spawn()?;
enter_launch_session(&mut tui)?;
type_and_submit(&mut tui, "dispatch one QA worker for the stack guard check")?;
// The abort lands inside the first `agent` dispatch, so the process is
// already reaped by the time the child request would have been issued.
// Probe liveness first: it names the mechanism in the failure text instead
// of leaving a 15s timeout over a retained frame of a dead TUI.
assert!(
tui.wait_for_exit(Duration::from_millis(250)).is_none(),
"codewhale-tui exited during `agent` dispatch (a stack overflow aborts as 134); \
the Tokio runtime must carry CODEWHALE_MAIN_STACK_BYTES — see main.rs build_runtime()"
);
wait_for_counter(&mut tui, &child_requests, 1, INTERACTION_TIMEOUT)?;
let _ = tui.shutdown();
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn release_four_read_only_fleet_roles_launch_with_canonical_prompts() -> Result<()> {
let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await;