feat(tui): add safe deferred read_media tool (#5102)
Add first-class read_media tool for safe multimodal media inspection (PNG, JPEG, GIF, WebP) with bounded decoding, decompression-bomb guards, crop regions, detail scaling, and credential redaction. - Defer loading in default catalog to preserve context budgets - Isolate blocking decode, I/O, and base64 pipeline in tokio::task::spawn_blocking - Enforce cancellation checks before dispatch and after await - Guard workspace escapes and canonical credential paths - Support all three provider wire formats with typed receipts Signed-off-by: Hunter Bown <hmbown@gmail.com>
This commit is contained in:
Hunter Bown
committed by
CodeWhale Bot
CodeWhale Bot
parent
46c18c648b
commit
7aff6074f2
Generated
+31
@@ -1153,6 +1153,12 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color_quant"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
@@ -2163,6 +2169,16 @@ dependencies = [
|
||||
"typed-builder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gif"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
|
||||
dependencies = [
|
||||
"color_quant",
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.4"
|
||||
@@ -2616,10 +2632,25 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder-lite",
|
||||
"color_quant",
|
||||
"gif",
|
||||
"image-webp",
|
||||
"moxcms",
|
||||
"num-traits",
|
||||
"png",
|
||||
"tiff",
|
||||
"zune-core",
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image-webp"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
|
||||
dependencies = [
|
||||
"byteorder-lite",
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -94,7 +94,7 @@ webbrowser = "1.0"
|
||||
shlex = "1.3.0"
|
||||
globset = "0.4"
|
||||
ignore = "0.4"
|
||||
image = { version = "0.25", default-features = false, features = ["png"] }
|
||||
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "gif"] }
|
||||
htmd = "0.5.4"
|
||||
# lru >= 0.18.2 fixes RUSTSEC-2026-0253 (panic-unsafe `pop()`).
|
||||
lru = "0.18"
|
||||
|
||||
@@ -3655,6 +3655,27 @@ mod provider_native_search;
|
||||
mod responses;
|
||||
mod stream_entry;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn anthropic_tool_result_content_for_test(
|
||||
content: &str,
|
||||
content_blocks: Option<&[Value]>,
|
||||
) -> Value {
|
||||
anthropic::anthropic_tool_result_content(content, content_blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn responses_tool_output_for_test(
|
||||
content: &str,
|
||||
content_blocks: Option<&[Value]>,
|
||||
) -> Value {
|
||||
responses::responses_tool_output(content, content_blocks)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn chat_messages_for_test(messages: &[crate::models::Message]) -> Vec<Value> {
|
||||
chat::build_chat_messages(None, messages, "gpt-4o")
|
||||
}
|
||||
|
||||
fn extract_sse_data_value(line: &str) -> Option<&str> {
|
||||
line.strip_prefix("data:")
|
||||
.map(|value| value.strip_prefix(' ').unwrap_or(value))
|
||||
|
||||
@@ -616,7 +616,10 @@ fn anthropic_image_block(url: &str) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
fn anthropic_tool_result_content(content: &str, content_blocks: Option<&[Value]>) -> Value {
|
||||
pub(super) fn anthropic_tool_result_content(
|
||||
content: &str,
|
||||
content_blocks: Option<&[Value]>,
|
||||
) -> Value {
|
||||
let (image, omitted) = crate::image_attach::provider_tool_result_image_refs(content_blocks);
|
||||
let content = crate::image_attach::tool_result_text_with_omission(content, omitted);
|
||||
let Some((mime_type, data)) = image else {
|
||||
|
||||
@@ -659,7 +659,7 @@ impl DeepSeekClient {
|
||||
}
|
||||
}
|
||||
|
||||
fn responses_tool_output(content: &str, content_blocks: Option<&[Value]>) -> Value {
|
||||
pub(super) fn responses_tool_output(content: &str, content_blocks: Option<&[Value]>) -> Value {
|
||||
let (image, omitted) = crate::image_attach::provider_tool_result_image_refs(content_blocks);
|
||||
let content = crate::image_attach::tool_result_text_with_omission(content, omitted);
|
||||
let Some((mime_type, data)) = image else {
|
||||
|
||||
@@ -8230,6 +8230,22 @@ async fn registry_discovery_and_start_handlers_exist_in_agent_and_plan_modes() {
|
||||
)
|
||||
.build(engine.build_tool_context(mode, false));
|
||||
assert!(registry.contains("registry_sync"), "missing in {mode:?}");
|
||||
assert!(registry.contains("read_media"), "missing in {mode:?}");
|
||||
let media_tools = registry
|
||||
.to_api_tools_with_cache(true)
|
||||
.into_iter()
|
||||
.filter(|tool| tool.name == "read_media")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
media_tools.len(),
|
||||
1,
|
||||
"read_media must be registered exactly once in {mode:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
media_tools[0].defer_loading,
|
||||
Some(true),
|
||||
"read_media must remain default-off/deferred in {mode:?}"
|
||||
);
|
||||
assert!(
|
||||
registry.contains("start_registry_mcp_server"),
|
||||
"missing in {mode:?}"
|
||||
|
||||
@@ -101,6 +101,7 @@ impl Engine {
|
||||
.with_git_tools()
|
||||
.with_git_history_tools()
|
||||
.with_diagnostics_tool()
|
||||
.with_read_media_tool()
|
||||
.with_skill_tools()
|
||||
.with_validation_tools()
|
||||
.with_handle_tools()
|
||||
|
||||
@@ -426,7 +426,7 @@ fn is_config_or_backup(candidate: &Path, config_path: &Path) -> bool {
|
||||
/// secret-store directories. Other dotfiles remain readable. Model-bound
|
||||
/// redaction is still required because shell tools can read these files and
|
||||
/// arbitrary commands can print credentials without reading a file at all.
|
||||
fn is_codewhale_credential_path(path: &Path) -> bool {
|
||||
pub(crate) fn is_codewhale_credential_path(path: &Path) -> bool {
|
||||
let candidate = canonical_path_for_credential_guard(path);
|
||||
|
||||
if let Ok(active_config) = codewhale_config::resolve_config_path(None)
|
||||
|
||||
@@ -45,6 +45,7 @@ mod pdf;
|
||||
pub mod plan;
|
||||
pub mod plugin;
|
||||
pub mod project;
|
||||
pub mod read_media;
|
||||
pub mod registry;
|
||||
pub mod remember;
|
||||
mod resource_admission;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -902,6 +902,13 @@ impl ToolRegistryBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
/// Include the `read_media` tool for safe multimodal media inspection.
|
||||
#[must_use]
|
||||
pub fn with_read_media_tool(self) -> Self {
|
||||
use super::read_media::ReadMediaTool;
|
||||
self.with_tool(Arc::new(ReadMediaTool::default()))
|
||||
}
|
||||
|
||||
/// Include the `load_skill` tool (#434) so the model can pull a
|
||||
/// SKILL.md body + companion file list into context with one
|
||||
/// call instead of `read_file` + `list_dir` against the path
|
||||
@@ -1248,6 +1255,7 @@ impl ToolRegistryBuilder {
|
||||
.with_revert_turn_tool()
|
||||
.with_pandoc_tools()
|
||||
.with_image_ocr_tools()
|
||||
.with_read_media_tool()
|
||||
.with_finance_tool();
|
||||
|
||||
match shell_policy {
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# `read_media` Operator Guide
|
||||
|
||||
`read_media` is a safe, first-class image reading and preprocessing tool for Codewhale v0.9.10. It allows vision-capable coding models to inspect visual assets (diagrams, UI mockups, screenshots, rendered graphs) with strict security, memory bounds, and privacy guards.
|
||||
|
||||
---
|
||||
|
||||
## 1. Overview and Scope
|
||||
|
||||
- **Supported Formats:** PNG, JPEG, GIF, WebP.
|
||||
- **Out of Scope:** Video files, audio streams, and background/automatic screenshot watching are deliberately excluded.
|
||||
- **Provider-Neutral Wiring:** Decoded and normalized images are converted into native image parts across all supported providers (OpenAI Chat Completions, Anthropic Messages, and OpenAI Responses API).
|
||||
|
||||
### Show the agent a screenshot
|
||||
|
||||
- Paste a clipboard image into the composer with the normal terminal paste
|
||||
shortcut, or run `/attach <path>` for an existing PNG, JPEG, GIF, or WebP.
|
||||
- A visible attachment row appears above the composer before the turn is sent.
|
||||
Temporary macOS `NSIRD_screencaptureui` paths are copied into Codewhale's
|
||||
stable attachment store when ingested.
|
||||
- Ask the agent to inspect the screenshot. The image is sent only as part of
|
||||
that explicit turn action; merely having a screenshot path or artifact does
|
||||
not trigger background analysis.
|
||||
|
||||
`read_media` is the corresponding agent-side path for inspecting another
|
||||
image later in the task without requiring the operator to attach it again.
|
||||
|
||||
---
|
||||
|
||||
## 2. Activation and Catalog Policy
|
||||
|
||||
- **Default-Off / Deferred Loading:** To preserve model context budgets, `read_media` is registered as a deferred tool (`defer_loading = true`) rather than occupying active slots in the default core catalog.
|
||||
- **Explicit Model Invocation:** The tool is invoked explicitly by name when the model or user requests media inspection.
|
||||
- **Eager Configuration:** Operators who want `read_media` always pre-loaded in the model tool catalog can configure:
|
||||
|
||||
```toml
|
||||
[tools]
|
||||
always_load = ["read_media"]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Tool Parameters and Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "docs/architecture.png",
|
||||
"crop": {
|
||||
"x": 100,
|
||||
"y": 50,
|
||||
"width": 800,
|
||||
"height": 600
|
||||
},
|
||||
"detail": "auto"
|
||||
}
|
||||
```
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `path` | string | **Yes** | Workspace-relative or trusted external path to the image file. |
|
||||
| `crop` | object | No | Optional pixel bounding box `{ "x": u32, "y": u32, "width": u32, "height": u32 }` (0-indexed). |
|
||||
| `detail` | string | No | Resolution target: `"auto"` (max 2048px, default), `"low"` (max 1024px), `"high"` / `"original"` (up to 4096px). |
|
||||
|
||||
---
|
||||
|
||||
## 4. Safety, Privacy, and Guardrails
|
||||
|
||||
### 4.1. Workspace Boundary and Credential Protection
|
||||
- **Workspace Containment:** Paths must resolve within the workspace or user-approved trusted external paths (`/trust`). Symlink escapes outside trusted roots are rejected.
|
||||
- **Credential Protection:** Codewhale configuration (`config.toml`, `.codewhale/`, `.deepseek/`, secrets directory) cannot be read via `read_media` and will fail with `PermissionDenied`.
|
||||
|
||||
### 4.2. Decompression-Bomb and Memory Limits
|
||||
- **Source Byte Limit:** Maximum source file size before decoding is **20 MiB** (`MAX_SOURCE_IMAGE_BYTES`). Oversized files are rejected before allocation.
|
||||
- **Dimension Guards:** Maximum permitted image width and height is **8192 px** (`MAX_IMAGE_DIMENSION`).
|
||||
- **Pixel Budget:** Maximum total pixel count is **33,554,432 pixels** (~33.5 megapixels, `MAX_IMAGE_PIXELS`).
|
||||
- **Memory Ceiling:** Safe memory allocation during decode is capped at **64 MiB** (`MAX_DECODE_ALLOC_BYTES`).
|
||||
- **Wire Payload Limit:** Re-encoded image payload is capped at **5 MiB** (`MAX_WIRE_IMAGE_BYTES`), matching provider constraints.
|
||||
|
||||
### 4.3. Active Route Vision Checks
|
||||
- Before reading an image, `read_media` inspects `context.route_capabilities.image_input`.
|
||||
- If the active model route explicitly lacks vision support (`CapabilityState::Unsupported`), the tool returns an actionable error directing the operator to switch to a vision model:
|
||||
|
||||
```text
|
||||
read_media: the active model route does not support image input. Switch to a route marked vision-capable with /model, or configure the route's image_input capability, then try again.
|
||||
```
|
||||
|
||||
Only a known `Unsupported` capability blocks the explicit tool call. An
|
||||
`Unknown` capability is admitted deliberately, matching normal attachment
|
||||
routing: custom and self-hosted providers often do not publish modality
|
||||
metadata, so their provider response remains authoritative. Operators who
|
||||
need a fail-closed route can set its `image_input` capability explicitly.
|
||||
|
||||
---
|
||||
|
||||
## 5. Typed Receipts and Wire Integration
|
||||
|
||||
Each successful execution yields:
|
||||
1. **Human-Readable Receipt (`content`):** Summarizes original format, source and final dimensions, crop details, and byte sizes.
|
||||
2. **Typed JSON Metadata (`metadata`):** Contains structured dimension, crop, and byte information without exposing credentials or internal tokens.
|
||||
3. **Rich Image Content Block (`content_blocks`):** Attaches a standardized `ToolResultContentBlock::Image` which provider adapters wire into outbound requests:
|
||||
- **OpenAI Chat Completions:** `image_url` block in following user message.
|
||||
- **Anthropic Messages:** `image` base64 source inside `tool_result` block.
|
||||
- **OpenAI Responses:** `input_image` block inside function call output.
|
||||
Reference in New Issue
Block a user