fix(tui): give tokio workers the runtime stack the owner thread already had
`#[tokio::main]` expanded to a multi-thread builder with no thread_stack_size, so every worker carried tokio's 2 MiB default while only the `codewhale-main` owner thread received CODEWHALE_MAIN_STACK_BYTES. The engine does not run on that owner thread: spawn_engine -> spawn_supervised -> tokio::spawn puts Engine::run on a worker, so the explicit stack never applied where the depth actually is. A debug-build `agent` dispatch (turn_loop -> FuturesUnordered -> execute_full_with_context -> AgentTool::execute -> spawn_subagent_from_input) measured a stack high-water mark between 2.25 and 2.5 MiB and aborted the process on the guard page. A Rust stack overflow is not a panic — it raises SIGABRT — so spawn_supervised's catch_unwind could not see it and the process died with 134 mid-dispatch, before any child request was issued. This is the release_runtime_qa blocker: both fleet tests observed zero child requests and then timed out against a corpse, while the retained PTY frame kept repainting the last screen the TUI drew before it died. Evidence: dose-response on RUST_MIN_STACK against one binary — 2 MiB FAIL (31.2s), 2359296 FAIL, 2621440 ok (2.3s), 3/4/8/16 MiB ok. Bounded threshold, so not recursion. 26 macOS crash reports, every one faulting on tokio-rt-worker in a Stack Guard between two 2080K stack regions. Ruled out and recorded so they are not re-chased: "Overwriting existing tool: File" is registry build-time last-write-wins, fires ~49ms earlier, appears in passing runs, and is byte-identical at the base commit; the `agent` tool is registered and model-visible throughout; the #3095 launch gate cannot starve either test. Also stops the harness reporting a dead child as a hang, and adds a regression test that asserts the invariant the default violated — dispatching `agent` must not kill the process — instead of a counter that a dead process also fails. Receipt: cargo test -p codewhale-tui --test release_runtime_qa -> 20 passed; 0 failed; 1 ignored (was 17 passed; 2 failed)
This commit is contained in:
+41
-2
@@ -1439,8 +1439,47 @@ fn main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn run_async_main(
|
||||
fn run_async_main(
|
||||
cli: Cli,
|
||||
command: Option<Commands>,
|
||||
plugin_discovery: Arc<crate::plugins::PluginDiscoveryContext>,
|
||||
plugin_registry: Arc<crate::plugins::PluginRegistry>,
|
||||
) -> Result<()> {
|
||||
build_runtime()?.block_on(run_async_main_inner(
|
||||
cli,
|
||||
command,
|
||||
plugin_discovery,
|
||||
plugin_registry,
|
||||
))
|
||||
}
|
||||
|
||||
/// Build the runtime that owns every async task in this binary.
|
||||
///
|
||||
/// `#[tokio::main]` used to expand here, which left every worker thread on
|
||||
/// tokio's 2 MiB default while only the `codewhale-main` owner thread above
|
||||
/// received `CODEWHALE_MAIN_STACK_BYTES`. The engine does not run on that owner
|
||||
/// thread — `core::engine::spawn_engine` hands `Engine::run` to
|
||||
/// `utils::spawn_supervised`, a bare `tokio::spawn` — so the explicit stack
|
||||
/// never applied where the depth actually is.
|
||||
///
|
||||
/// A debug-build `agent` dispatch (turn_loop -> FuturesUnordered ->
|
||||
/// execute_full_with_context -> AgentTool::execute -> spawn_subagent_from_input)
|
||||
/// measured a stack high-water mark between 2.25 and 2.5 MiB and aborted the
|
||||
/// whole process on the guard page. A Rust stack overflow is not a panic: it
|
||||
/// raises SIGABRT, so `spawn_supervised`'s `catch_unwind` cannot see it and the
|
||||
/// process dies with 134 mid-dispatch.
|
||||
///
|
||||
/// This is behavior-identical to the old `#[tokio::main]` expansion apart from
|
||||
/// the stack size, and it makes the knob greppable.
|
||||
pub(crate) fn build_runtime() -> Result<tokio::runtime::Runtime> {
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.thread_stack_size(CODEWHALE_MAIN_STACK_BYTES)
|
||||
.build()
|
||||
.context("Failed to build the Codewhale Tokio runtime")
|
||||
}
|
||||
|
||||
async fn run_async_main_inner(
|
||||
cli: Cli,
|
||||
command: Option<Commands>,
|
||||
plugin_discovery: Arc<crate::plugins::PluginDiscoveryContext>,
|
||||
|
||||
@@ -260,6 +260,17 @@ fn wait_for_counter(
|
||||
if counter.load(Ordering::SeqCst) >= expected {
|
||||
return Ok(());
|
||||
}
|
||||
// A dead child renders as a hang: `pump()` only feeds *new* bytes into a
|
||||
// retained frame, so `debug_dump()` keeps painting the last frame the TUI
|
||||
// drew before it died. Without this probe a SIGABRT (stack overflow aborts
|
||||
// as 134) burns the whole timeout and then reports a live-looking screen.
|
||||
if let Some(code) = harness.wait_for_exit(Duration::from_millis(0)) {
|
||||
return Err(anyhow!(
|
||||
"TUI exited with {code} before the counter reached {expected}; observed {}\n{}",
|
||||
counter.load(Ordering::SeqCst),
|
||||
harness.debug_dump()
|
||||
));
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(anyhow!(
|
||||
"counter did not reach {expected} within {timeout:?}; observed {}\n{}",
|
||||
@@ -697,6 +708,87 @@ async fn release_six_worker_fanout_keeps_typing_render_and_esc_cancel_live() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SingleDispatchResponder {
|
||||
child_requests: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Respond for SingleDispatchResponder {
|
||||
fn respond(&self, request: &Request) -> ResponseTemplate {
|
||||
let body = request.body_json::<Value>().unwrap_or(Value::Null);
|
||||
let raw = body.to_string();
|
||||
|
||||
if raw.contains("stay busy worker") && !raw.contains("dispatch one QA worker") {
|
||||
self.child_requests.fetch_add(1, Ordering::SeqCst);
|
||||
return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "child-acknowledged"));
|
||||
}
|
||||
|
||||
if raw.contains("dispatch one QA worker") {
|
||||
return sse_response(fanout_tool_call_sse_n(1));
|
||||
}
|
||||
|
||||
sse_response(text_sse(DEEPSEEK_TEST_MODEL, "unexpected-request"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatching `agent` must not kill the process.
|
||||
///
|
||||
/// Regression guard for the 0.9.4 release blocker: the Tokio runtime was built
|
||||
/// by `#[tokio::main]`, so every worker thread carried tokio's 2 MiB default
|
||||
/// while only the `codewhale-main` owner thread got `CODEWHALE_MAIN_STACK_BYTES`.
|
||||
/// The engine runs on a worker, and a debug-build `agent` dispatch measured a
|
||||
/// stack high-water mark between 2.25 and 2.5 MiB — it overflowed the guard page
|
||||
/// and aborted the process with 134, mid-dispatch, before any child request was
|
||||
/// ever issued.
|
||||
///
|
||||
/// This asserts the invariant that the default violated (the process survives an
|
||||
/// `agent` dispatch) rather than re-asserting a child counter, which a dead
|
||||
/// process also fails — but fails slowly and for the wrong stated reason.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn release_agent_dispatch_never_aborts_the_runtime() -> Result<()> {
|
||||
let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await;
|
||||
let server = MockServer::start().await;
|
||||
mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await;
|
||||
let child_requests = Arc::new(AtomicUsize::new(0));
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(SingleDispatchResponder {
|
||||
child_requests: Arc::clone(&child_requests),
|
||||
})
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let ws = make_sealed_workspace()?;
|
||||
std::fs::write(
|
||||
ws.home().join(".codewhale").join("config.toml"),
|
||||
"[subagents]\nmax_concurrent = 1\nlaunch_concurrency = 1\nmax_admitted = 1\n",
|
||||
)?;
|
||||
let mut tui = common_tui_builder(&ws)
|
||||
.env("CODEWHALE_PROVIDER", "deepseek")
|
||||
.env("DEEPSEEK_API_KEY", "deepseek-local-test-key")
|
||||
.env("DEEPSEEK_BASE_URL", server.uri())
|
||||
.env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL)
|
||||
.args(["--yolo", "--max-subagents", "1"])
|
||||
.spawn()?;
|
||||
enter_launch_session(&mut tui)?;
|
||||
|
||||
type_and_submit(&mut tui, "dispatch one QA worker for the stack guard check")?;
|
||||
|
||||
// The abort lands inside the first `agent` dispatch, so the process is
|
||||
// already reaped by the time the child request would have been issued.
|
||||
// Probe liveness first: it names the mechanism in the failure text instead
|
||||
// of leaving a 15s timeout over a retained frame of a dead TUI.
|
||||
assert!(
|
||||
tui.wait_for_exit(Duration::from_millis(250)).is_none(),
|
||||
"codewhale-tui exited during `agent` dispatch (a stack overflow aborts as 134); \
|
||||
the Tokio runtime must carry CODEWHALE_MAIN_STACK_BYTES — see main.rs build_runtime()"
|
||||
);
|
||||
|
||||
wait_for_counter(&mut tui, &child_requests, 1, INTERACTION_TIMEOUT)?;
|
||||
|
||||
let _ = tui.shutdown();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn release_four_read_only_fleet_roles_launch_with_canonical_prompts() -> Result<()> {
|
||||
let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await;
|
||||
|
||||
Reference in New Issue
Block a user