Compare commits

...

3 Commits

Author SHA1 Message Date
copilot-swe-agent[bot] 38c0a89673 feat(camera): advertise sensor-supported resolutions in MCP description 2026-08-02 17:30:15 +00:00
copilot-swe-agent[bot] e92a7e2c2c feat(camera): configurable capture resolution for vision explain 2026-08-02 17:01:16 +00:00
copilot-swe-agent[bot] 86c7683e36 Initial plan 2026-08-02 16:51:16 +00:00
11 changed files with 395 additions and 20 deletions
+6
View File
@@ -475,6 +475,12 @@ Supported LCD families include:
- `RndisBoard` - RNDIS-over-USB networking (ESP32-S3 / ESP32-P4).
- `EspVideo` helpers for ESP-Video on ESP32-S3 / ESP32-P4.
### Camera / vision explain resolution
- `Esp32Camera` and other `Camera` implementations power MCP `self.camera.take_photo`.
- Set the board default capture size in the board `.cc` via `camera_config_t.frame_size`, and/or define a board `config.h` macro such as `CAMERA_FRAME_SIZE_NAME` (`"QVGA"`, `"VGA"`, `"SVGA"`, `"UXGA"`, ...) and apply it with `Camera::SetFrameSize()` after init (see `atoms3r-cam-m12-echo-base`).
- Callers can also pass an optional per-request `resolution` argument to `self.camera.take_photo`. `Esp32Camera` reports sensor-supported modes through `GetSupportedFrameSizeNames()` (from `esp_camera_sensor_get_info()->max_size`), and MCP embeds that list in the tool description at registration time. Higher resolutions improve text/label reading but need more PSRAM and longer uploads. Init at a size the sensor can actually support; `Esp32Camera::SetFrameSize` will reinit when a larger framebuffer is required.
### Input helpers
- `Button` - standard push buttons (click, long-press, multi-click).
+1 -1
View File
@@ -105,7 +105,7 @@ A tool registered this way will not appear in a regular `tools/list` response. T
| `self.audio_speaker.set_volume` | Set speaker volume (`volume`: 0-100). |
| `self.screen.set_brightness` | Set screen brightness when a backlight is available (`brightness`: 0-100). |
| `self.screen.set_theme` | Switch UI theme (`theme`: `"light"` or `"dark"`), when LVGL is enabled. |
| `self.camera.take_photo` | Take a picture with the on-board camera (when the board has one) and answer the given `question` about it. |
| `self.camera.take_photo` | Take a picture with the on-board camera (when the board has one) and answer the given `question` about it. When the driver can enumerate sensor modes, the tool description lists the device-supported `resolution` values (e.g. `QVGA` / `VGA` / `SVGA` / … up to the sensor max) and the current default. Higher values help with text/label reading but use more RAM and upload time. |
Board-specific tools are appended after these by each board's `InitializeTools()`.
+18
View File
@@ -109,6 +109,24 @@ void InitializeTools() {
}
```
### 5. 摄像头拍照解释(可选分辨率)
```json
{
"jsonrpc": "2.0",
"method": "tools/call",
"params": {
"name": "self.camera.take_photo",
"arguments": {
"question": "请读出图片中的文字",
"resolution": "SVGA"
}
},
"id": 5
}
```
`resolution` 为可选参数。设备会在 MCP 工具描述中写入当前摄像头传感器实际支持的分辨率列表(以及默认值);不传则使用板级默认值。更高分辨率有利于识别细小文字,但会占用更多内存并增加上传时间。
## 备注
- 工具名称、参数及返回值请以设备端 `AddTool` 注册为准。
- 推荐所有新项目统一采用 MCP 协议进行物联网控制。
+9
View File
@@ -12,6 +12,15 @@ public:
virtual bool SetHMirror(bool enabled) = 0;
virtual bool SetVFlip(bool enabled) = 0;
virtual bool SetSwapBytes(bool enabled) { return false; } // Optional, default no-op
// Optional capture resolution for vision/explain. Accepted names include:
// QQVGA, QVGA, HVGA, VGA, SVGA, XGA, HD, SXGA, UXGA (case-insensitive).
// Empty string is a no-op (returns true). Non-empty default returns false (unsupported).
virtual bool SetFrameSize(const std::string& frame_size) { return frame_size.empty(); }
// Comma-separated named resolutions supported by this camera (e.g. "QVGA, VGA, SVGA").
// Empty when the driver does not expose selectable resolutions.
virtual std::string GetSupportedFrameSizeNames() const { return {}; }
// Current capture resolution name when known (e.g. "SVGA"); empty otherwise.
virtual std::string GetFrameSizeName() const { return {}; }
virtual std::string Explain(const std::string& question) = 0;
};
+239 -7
View File
@@ -1,24 +1,30 @@
#include "sdkconfig.h"
#include <esp_heap_caps.h>
#include <cstdio>
#include <cstring>
#include <esp_log.h>
#include <esp_timer.h>
#include <img_converters.h>
#include "esp32_camera.h"
#include <algorithm>
#include <cctype>
#include <cstdio>
#include <cstring>
#include "board.h"
#include "display.h"
#include "esp32_camera.h"
#include "jpg/image_to_jpeg.h"
#include "lvgl_display.h"
#include "mcp_server.h"
#include "system_info.h"
#include "jpg/image_to_jpeg.h"
#include "esp_timer.h"
#define TAG "Esp32Camera"
Esp32Camera::Esp32Camera(const camera_config_t& config) {
esp_err_t err = esp_camera_init(&config);
config_ = config;
frame_size_ = config.frame_size;
esp_err_t err = esp_camera_init(&config_);
if (err != ESP_OK) {
ESP_LOGE(TAG, "esp_camera_init failed with error 0x%x", err);
return;
@@ -28,8 +34,10 @@ Esp32Camera::Esp32Camera(const camera_config_t &config) {
if (s) {
if (s->id.PID == GC0308_PID) {
s->set_hmirror(s, 0); // Control camera mirror: 1 for mirror, 0 for normal
hmirror_ = false;
}
ESP_LOGI(TAG, "Camera initialized: format=%d", config.pixel_format);
ESP_LOGI(TAG, "Camera initialized: format=%d, frame_size=%d", config_.pixel_format,
(int)frame_size_);
}
streaming_on_ = true;
@@ -135,6 +143,7 @@ bool Esp32Camera::SetHMirror(bool enabled) {
return false;
}
s->set_hmirror(s, enabled ? 1 : 0);
hmirror_ = enabled;
return true;
}
@@ -144,6 +153,7 @@ bool Esp32Camera::SetVFlip(bool enabled) {
return false;
}
s->set_vflip(s, enabled ? 1 : 0);
vflip_ = enabled;
return true;
}
@@ -152,6 +162,228 @@ bool Esp32Camera::SetSwapBytes(bool enabled) {
return true;
}
// Named sizes exposed to MCP / board config. Keep in sync with ParseFrameSize.
static const struct {
framesize_t size;
const char* name;
} kNamedFrameSizes[] = {
{FRAMESIZE_QQVGA, "QQVGA"}, {FRAMESIZE_QVGA, "QVGA"}, {FRAMESIZE_HVGA, "HVGA"},
{FRAMESIZE_VGA, "VGA"}, {FRAMESIZE_SVGA, "SVGA"}, {FRAMESIZE_XGA, "XGA"},
{FRAMESIZE_HD, "HD"}, {FRAMESIZE_SXGA, "SXGA"}, {FRAMESIZE_UXGA, "UXGA"},
{FRAMESIZE_FHD, "FHD"}, {FRAMESIZE_QXGA, "QXGA"}, {FRAMESIZE_QHD, "QHD"},
{FRAMESIZE_WQXGA, "WQXGA"}, {FRAMESIZE_QSXGA, "QSXGA"},
};
const char* Esp32Camera::FrameSizeToName(framesize_t frame_size) {
for (const auto& entry : kNamedFrameSizes) {
if (entry.size == frame_size) {
return entry.name;
}
}
return nullptr;
}
framesize_t Esp32Camera::GetSensorMaxFrameSize() const {
sensor_t* s = esp_camera_sensor_get();
if (s != nullptr) {
// esp_camera_sensor_get_info takes a non-const sensor_id_t*.
sensor_id_t id = s->id;
camera_sensor_info_t* info = esp_camera_sensor_get_info(&id);
if (info != nullptr && info->max_size < FRAMESIZE_INVALID) {
return info->max_size;
}
}
// Fall back to the largest size we have already configured successfully.
return config_.frame_size > frame_size_ ? config_.frame_size : frame_size_;
}
bool Esp32Camera::ParseFrameSize(const std::string& name, framesize_t* out) {
if (out == nullptr || name.empty()) {
return false;
}
std::string n = name;
n.erase(std::remove_if(n.begin(), n.end(),
[](unsigned char c) { return std::isspace(c) || c == '_' || c == '-'; }),
n.end());
std::transform(n.begin(), n.end(), n.begin(),
[](unsigned char c) { return static_cast<char>(std::toupper(c)); });
// Common aliases used by boards / MCP callers.
if (n == "96X96") {
*out = FRAMESIZE_96X96;
} else if (n == "QQVGA") {
*out = FRAMESIZE_QQVGA;
} else if (n == "QCIF") {
*out = FRAMESIZE_QCIF;
} else if (n == "HQVGA") {
*out = FRAMESIZE_HQVGA;
} else if (n == "240X240") {
*out = FRAMESIZE_240X240;
} else if (n == "QVGA") {
*out = FRAMESIZE_QVGA;
} else if (n == "CIF") {
*out = FRAMESIZE_CIF;
} else if (n == "HVGA") {
*out = FRAMESIZE_HVGA;
} else if (n == "VGA") {
*out = FRAMESIZE_VGA;
} else if (n == "SVGA") {
*out = FRAMESIZE_SVGA;
} else if (n == "XGA") {
*out = FRAMESIZE_XGA;
} else if (n == "HD") {
*out = FRAMESIZE_HD;
} else if (n == "SXGA") {
*out = FRAMESIZE_SXGA;
} else if (n == "UXGA") {
*out = FRAMESIZE_UXGA;
} else if (n == "FHD") {
*out = FRAMESIZE_FHD;
} else if (n == "QXGA") {
*out = FRAMESIZE_QXGA;
} else if (n == "QHD") {
*out = FRAMESIZE_QHD;
} else if (n == "WQXGA") {
*out = FRAMESIZE_WQXGA;
} else if (n == "QSXGA" || n == "5MP") {
*out = FRAMESIZE_QSXGA;
} else {
return false;
}
return true;
}
bool Esp32Camera::ApplyFrameSize(framesize_t frame_size) {
if (!streaming_on_) {
return false;
}
if (frame_size == frame_size_) {
return true;
}
if (frame_size >= FRAMESIZE_INVALID) {
return false;
}
if (encoder_thread_.joinable()) {
encoder_thread_.join();
}
const resolution_info_t& wanted = resolution[frame_size];
const resolution_info_t& allocated = resolution[config_.frame_size];
const size_t wanted_pixels = (size_t)wanted.width * (size_t)wanted.height;
const size_t allocated_pixels = (size_t)allocated.width * (size_t)allocated.height;
// Prefer a lightweight sensor-only change when the existing framebuffer is large enough.
if (wanted_pixels <= allocated_pixels) {
sensor_t* s = esp_camera_sensor_get();
if (s == nullptr) {
return false;
}
if (s->set_framesize(s, frame_size) != 0) {
ESP_LOGE(TAG, "set_framesize failed for %dx%d", wanted.width, wanted.height);
return false;
}
frame_size_ = frame_size;
ESP_LOGI(TAG, "Camera frame size set to %dx%d", wanted.width, wanted.height);
return true;
}
// Larger than the current framebuffer allocation: reinit the driver.
if (current_fb_) {
esp_camera_fb_return(current_fb_);
current_fb_ = nullptr;
}
if (encode_buf_) {
heap_caps_free(encode_buf_);
encode_buf_ = nullptr;
encode_buf_size_ = 0;
}
framesize_t previous_size = frame_size_;
framesize_t previous_alloc = config_.frame_size;
esp_camera_deinit();
streaming_on_ = false;
config_.frame_size = frame_size;
esp_err_t err = esp_camera_init(&config_);
if (err != ESP_OK) {
ESP_LOGE(TAG, "esp_camera_init failed while switching to %dx%d: 0x%x", wanted.width,
wanted.height, err);
// Best-effort restore of the previous configuration.
config_.frame_size = previous_alloc;
err = esp_camera_init(&config_);
if (err != ESP_OK) {
ESP_LOGE(TAG, "Failed to restore previous camera configuration: 0x%x", err);
return false;
}
streaming_on_ = true;
frame_size_ = previous_alloc;
sensor_t* s = esp_camera_sensor_get();
if (s) {
s->set_hmirror(s, hmirror_ ? 1 : 0);
s->set_vflip(s, vflip_ ? 1 : 0);
if (previous_size != previous_alloc) {
s->set_framesize(s, previous_size);
frame_size_ = previous_size;
}
}
return false;
}
streaming_on_ = true;
frame_size_ = frame_size;
sensor_t* s = esp_camera_sensor_get();
if (s) {
s->set_hmirror(s, hmirror_ ? 1 : 0);
s->set_vflip(s, vflip_ ? 1 : 0);
}
ESP_LOGI(TAG, "Camera reinitialized at %dx%d", wanted.width, wanted.height);
return true;
}
bool Esp32Camera::SetFrameSize(const std::string& frame_size) {
if (frame_size.empty()) {
return true;
}
framesize_t fs;
if (!ParseFrameSize(frame_size, &fs)) {
ESP_LOGE(TAG, "Unknown camera frame size '%s'", frame_size.c_str());
return false;
}
framesize_t max_size = GetSensorMaxFrameSize();
if (fs > max_size) {
ESP_LOGE(TAG, "Frame size '%s' exceeds sensor maximum", frame_size.c_str());
return false;
}
return ApplyFrameSize(fs);
}
std::string Esp32Camera::GetSupportedFrameSizeNames() const {
if (!streaming_on_) {
return {};
}
framesize_t max_size = GetSensorMaxFrameSize();
std::string names;
for (const auto& entry : kNamedFrameSizes) {
if (entry.size > max_size) {
continue;
}
if (!names.empty()) {
names += ", ";
}
names += entry.name;
}
return names;
}
std::string Esp32Camera::GetFrameSizeName() const {
const char* name = FrameSizeToName(frame_size_);
return name != nullptr ? std::string(name) : std::string();
}
std::string Esp32Camera::Explain(const std::string &question) {
if (explain_url_.empty()) {
throw std::runtime_error("Image explain URL or token is not set");
+12
View File
@@ -22,7 +22,11 @@ struct JpegChunk
class Esp32Camera : public Camera
{
private:
camera_config_t config_ = {};
framesize_t frame_size_ = FRAMESIZE_INVALID;
bool streaming_on_ = false;
bool hmirror_ = false;
bool vflip_ = false;
bool swap_bytes_enabled_ = true; // Swap pixel byte order for RGB565, enabled by default
std::string explain_url_;
std::string explain_token_;
@@ -31,6 +35,11 @@ private:
uint8_t *encode_buf_ = nullptr; // Buffer for JPEG encoding (with optional byte swap)
size_t encode_buf_size_ = 0;
static bool ParseFrameSize(const std::string &name, framesize_t *out);
static const char *FrameSizeToName(framesize_t frame_size);
framesize_t GetSensorMaxFrameSize() const;
bool ApplyFrameSize(framesize_t frame_size);
public:
Esp32Camera(const camera_config_t &config);
~Esp32Camera();
@@ -40,5 +49,8 @@ public:
virtual bool SetHMirror(bool enabled) override;
virtual bool SetVFlip(bool enabled) override;
virtual bool SetSwapBytes(bool enabled) override;
virtual bool SetFrameSize(const std::string &frame_size) override;
virtual std::string GetSupportedFrameSizeNames() const override;
virtual std::string GetFrameSizeName() const override;
virtual std::string Explain(const std::string &question) override;
};
@@ -35,6 +35,12 @@ idf.py menuconfig
- `Partition Table``Custom partition CSV file` → 删除原有内容,输入 `partitions/v2/8m.csv`
- `Serial flasher config``Flash size` → 选择 `8 MB`
### 摄像头视觉分辨率
- 默认捕获/解释分辨率在 `config.h``CAMERA_FRAME_SIZE_NAME` 中配置(M12 / OV3660 默认为 `"SVGA"`)。
- AtomS3R-CAMGC0308)会在传感器识别后限制到 `VGA`
- MCP 工具 `self.camera.take_photo` 支持可选参数 `resolution`(如 `"VGA"` / `"SVGA"` / `"UXGA"`),可按次覆盖默认值,便于细小文字识别。更高分辨率会占用更多 PSRAM 并增加上传耗时。
`S` 保存,按 `Q` 退出。
**编译**
@@ -146,6 +146,8 @@ private:
config.pin_reset = CAMERA_PIN_RESET;
config.xclk_freq_hz = XCLK_FREQ_HZ;
config.pixel_format = PIXFORMAT_RGB565;
// Start at QVGA so both GC0308 (AtomS3R-CAM) and OV3660 (AtomS3R-M12) can
// complete sensor init, then raise resolution for capable sensors.
config.frame_size = FRAMESIZE_QVGA;
config.jpeg_quality = 12;
config.fb_count = 1;
@@ -160,8 +162,17 @@ private:
sensor_t *sensor = esp_camera_sensor_get();
if (sensor && sensor->id.PID == OV3660_PID) {
camera_->SetHMirror(true);
// Prefer board-configured explain/capture resolution (default SVGA).
if (!camera_->SetFrameSize(CAMERA_FRAME_SIZE_NAME)) {
ESP_LOGW(TAG, "Failed to apply CAMERA_FRAME_SIZE_NAME=%s, keeping QVGA",
CAMERA_FRAME_SIZE_NAME);
}
} else {
camera_->SetHMirror(false);
// GC0308 tops out around VGA; keep a modest default.
if (!camera_->SetFrameSize("VGA")) {
ESP_LOGW(TAG, "Failed to set VGA on non-OV3660 sensor, keeping QVGA");
}
}
}
@@ -45,5 +45,12 @@
#define CAMERA_XCLK_FREQ (20000000)
#define XCLK_FREQ_HZ CAMERA_XCLK_FREQ
// Default capture/explain resolution name for OV3660 (AtomS3R-M12).
// Accepted values match Camera::SetFrameSize (e.g. "QVGA", "VGA", "SVGA", "UXGA").
// SVGA (800x600) balances text legibility against RAM/upload cost. GC0308
// (AtomS3R-CAM) is clamped to VGA after sensor detection in the board file.
#ifndef CAMERA_FRAME_SIZE_NAME
#define CAMERA_FRAME_SIZE_NAME "SVGA"
#endif
#endif // _BOARD_CONFIG_H_
+36 -7
View File
@@ -99,19 +99,48 @@ void McpServer::AddCommonTools() {
auto camera = board.GetCamera();
if (camera) {
AddTool("self.camera.take_photo",
std::string description =
"Always remember you have a camera. If the user asks you to see something, use this tool to take a photo and then explain it.\n"
"Args:\n"
" `question`: The question that you want to ask about the photo.\n"
"Return:\n"
" A JSON object that provides the photo information.",
PropertyList({
" `question`: The question that you want to ask about the photo.\n";
PropertyList properties({
Property("question", kPropertyTypeString)
}),
[camera](const PropertyList& properties) -> ReturnValue {
});
// Advertise only the resolutions this camera sensor actually supports.
std::string supported_resolutions = camera->GetSupportedFrameSizeNames();
const bool resolution_supported = !supported_resolutions.empty();
if (resolution_supported) {
description +=
" `resolution`: Optional capture resolution for this photo. Use a higher value when the task needs fine detail "
"(e.g. reading text/labels). Supported values on this device: " +
supported_resolutions + ".";
std::string current_resolution = camera->GetFrameSizeName();
if (!current_resolution.empty()) {
description += " Default (when omitted): " + current_resolution + ".";
} else {
description += " Leave empty to keep the board default.";
}
description += " Higher resolutions use more RAM and take longer to upload.\n";
properties.AddProperty(Property("resolution", kPropertyTypeString, std::string()));
}
description +=
"Return:\n"
" A JSON object that provides the photo information.";
AddTool("self.camera.take_photo", description, properties,
[camera, resolution_supported](const PropertyList& properties) -> ReturnValue {
// Lower the priority to do the camera capture
TaskPriorityReset priority_reset(1);
if (resolution_supported) {
auto resolution = properties["resolution"].value<std::string>();
if (!resolution.empty() && !camera->SetFrameSize(resolution)) {
throw std::runtime_error("Unsupported or failed camera resolution: " + resolution);
}
}
if (!camera->Capture()) {
throw std::runtime_error("Failed to capture photo");
}
+45
View File
@@ -1092,6 +1092,51 @@ class BoardSourceTests(unittest.TestCase):
self.assertEqual(missing, [])
class CameraResolutionConfigTests(unittest.TestCase):
def test_camera_interface_exposes_set_frame_size(self):
camera_h = (ROOT / "main/boards/common/camera.h").read_text(encoding="utf-8")
self.assertIn("SetFrameSize", camera_h)
self.assertIn("GetSupportedFrameSizeNames", camera_h)
self.assertIn("GetFrameSizeName", camera_h)
esp32_camera_h = (ROOT / "main/boards/common/esp32_camera.h").read_text(
encoding="utf-8"
)
self.assertIn("SetFrameSize", esp32_camera_h)
self.assertIn("ParseFrameSize", esp32_camera_h)
self.assertIn("GetSupportedFrameSizeNames", esp32_camera_h)
self.assertIn("GetSensorMaxFrameSize", esp32_camera_h)
esp32_camera_cc = (ROOT / "main/boards/common/esp32_camera.cc").read_text(
encoding="utf-8"
)
for name in ("QVGA", "VGA", "SVGA", "UXGA", "HD"):
self.assertIn(f'"{name}"', esp32_camera_cc)
self.assertIn("esp_camera_sensor_get_info", esp32_camera_cc)
self.assertIn("kNamedFrameSizes", esp32_camera_cc)
mcp = (ROOT / "main/mcp_server.cc").read_text(encoding="utf-8")
self.assertIn('Property("resolution", kPropertyTypeString', mcp)
self.assertIn("GetSupportedFrameSizeNames", mcp)
self.assertIn("Supported values on this device", mcp)
self.assertIn("SetFrameSize(resolution)", mcp)
def test_atoms3r_board_default_frame_size_name(self):
config_h = (
ROOT
/ "main/boards/m5stack/atoms3r-cam-m12-echo-base/config.h"
).read_text(encoding="utf-8")
self.assertIn("CAMERA_FRAME_SIZE_NAME", config_h)
self.assertIn('"SVGA"', config_h)
board_cc = (
ROOT
/ "main/boards/m5stack/atoms3r-cam-m12-echo-base/atoms3r_cam_m12_echo_base.cc"
).read_text(encoding="utf-8")
self.assertIn("SetFrameSize(CAMERA_FRAME_SIZE_NAME)", board_cc)
self.assertIn('SetFrameSize("VGA")', board_cc)
class ZipTests(unittest.TestCase):
def test_zip_is_always_recreated(self):
previous_cwd = Path.cwd()