github-featured/78--xiaozhi-esp32
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

fix(firmware-builder): upload artifacts through control API

Browse Source
This commit is contained in:
Xiaoxia
2026-08-09 04:49:52 +08:00
parent 48142026b5
commit 18a60b8051
4 changed files with 137 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/firmware-builder/Dockerfile
-1
View File
@@ -20,7 +20,6 @@ RUN chmod 0755 \
docker/firmware-builder/entrypoint.sh \
docker/firmware-builder/firmware_builder.py \
&& . "${IDF_PATH}/export.sh" >/dev/null \
&& python3 -m pip install --no-cache-dir oss2==2.19.1 \
&& idf.py --version \
&& python3 scripts/build.py --list-boards --json >/tmp/xiaozhi-boards.json \
&& python3 scripts/build.py --list-languages --json >/tmp/xiaozhi-languages.json
docker/firmware-builder/README.md
+13 -15
View File
@@ -53,28 +53,26 @@ Each successful job writes:
- `manifest.json`: inputs, tool versions, source revision, sizes, and SHA-256
checksums.
To upload the job output to OSS, also pass:
To upload the job output to an HTTP artifact receiver, also pass:
```text
FIRMWARE_OSS_UPLOAD=true
FIRMWARE_OSS_ENDPOINT=oss-cn-shenzhen.aliyuncs.com
FIRMWARE_OSS_PREFIX=custom_firmwares
OSS_BUCKET_NAME=<bucket>
OSS_ACCESS_KEY_ID=<access-key-id>
OSS_ACCESS_KEY_SECRET=<access-key-secret>
FIRMWARE_UPLOAD_URL=https://example.com/api/firmware-builds
FIRMWARE_UPLOAD_TOKEN=<upload-token>
FIRMWARE_JOB_ID=<unique-safe-job-id>
```
The builder uploads the two firmware images, `build.log`, and `manifest.json`
to `custom_firmwares/<job-id>/`. The manifest is uploaded last so consumers do
not observe a completed job before its other objects are available. Transient
OSS connection, timeout, throttling, and server errors are retried up to four
times with exponential backoff; authentication and other permanent errors fail
immediately.
The builder sends an authenticated HTTP `PUT` for the two firmware images,
`build.log`, and `manifest.json` to
`<upload-url>/<job-id>/artifacts/<filename>`. The manifest is uploaded last so
consumers do not observe a completed job before its other objects are available.
Transient connection, timeout, throttling, and server errors are retried up to
four times with exponential backoff; authentication and other permanent errors
fail immediately. Storage credentials and provider details remain entirely on
the receiving service.
Use a unique empty output directory for each job. In ECI, pass the same inputs
as container environment variables and upload the output directory to OSS after
the process exits.
as container environment variables and let the receiver persist the output
after the process exits.
ESP-IDF uses Ninja, which automatically builds in parallel using the CPUs
visible to the container. Allocate at least 8 vCPUs to an ECI build job when
docker/firmware-builder/firmware_builder.py
+62 -95
View File
@@ -17,6 +17,9 @@ import shutil
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Sequence
@@ -30,8 +33,9 @@ ARTIFACTS = {
"ota": Path("build/xiaozhi.bin"),
"full": Path("build/merged-binary.bin"),
}
OSS_UPLOAD_MAX_ATTEMPTS = 4
OSS_UPLOAD_BASE_DELAY_SECONDS = 1
UPLOAD_MAX_ATTEMPTS = 4
UPLOAD_BASE_DELAY_SECONDS = 1
UPLOAD_TIMEOUT_SECONDS = 120
def utc_now() -> str:
@@ -43,42 +47,19 @@ def env(name: str) -> str | None:
return value.strip() if value and value.strip() else None
def env_enabled(name: str) -> bool:
return (env(name) or "").casefold() in {"1", "true", "yes", "on"}
def oss_config() -> dict[str, str] | None:
if not env_enabled("FIRMWARE_OSS_UPLOAD"):
def upload_config() -> dict[str, str] | None:
upload_url = env("FIRMWARE_UPLOAD_URL")
upload_token = env("FIRMWARE_UPLOAD_TOKEN")
if not upload_url and not upload_token:
return None
values = {
"access_key_id": env("OSS_ACCESS_KEY_ID"),
"access_key_secret": env("OSS_ACCESS_KEY_SECRET"),
"bucket": env("OSS_BUCKET_NAME"),
"endpoint": env("FIRMWARE_OSS_ENDPOINT"),
"prefix": env("FIRMWARE_OSS_PREFIX") or "custom_firmwares",
}
missing = [name for name, value in values.items() if not value]
if missing:
if not upload_url or not upload_token:
raise ValueError(
"OSS upload is enabled but configuration is missing: "
+ ", ".join(missing)
"FIRMWARE_UPLOAD_URL and FIRMWARE_UPLOAD_TOKEN must be configured together"
)
prefix = str(values["prefix"]).strip("/")
if not prefix or ".." in Path(prefix).parts:
raise ValueError(f"Invalid OSS prefix: {prefix!r}")
endpoint = str(values["endpoint"])
if not endpoint.startswith(("http://", "https://")):
endpoint = "https://" + endpoint
return {
"access_key_id": str(values["access_key_id"]),
"access_key_secret": str(values["access_key_secret"]),
"bucket": str(values["bucket"]),
"endpoint": endpoint,
"prefix": prefix,
}
parsed_url = urllib.parse.urlparse(upload_url)
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
raise ValueError("FIRMWARE_UPLOAD_URL must be an absolute HTTP(S) URL")
return {"url": upload_url.rstrip("/"), "token": upload_token}
def parser() -> argparse.ArgumentParser:
@@ -309,51 +290,54 @@ def failure_summary(log_path: Path) -> str:
return meaningful[-1][:500] if meaningful else "Firmware build failed"
def is_retryable_oss_error(error: Exception, oss2: Any) -> bool:
exceptions = getattr(oss2, "exceptions", None)
request_error = getattr(exceptions, "RequestError", None)
if isinstance(request_error, type) and isinstance(error, request_error):
return True
server_error = getattr(exceptions, "ServerError", None)
if isinstance(server_error, type) and isinstance(error, server_error):
status = getattr(error, "status", getattr(error, "status_code", None))
code = str(getattr(error, "code", ""))
return (
status in {408, 429}
or isinstance(status, int) and status >= 500
or code in {
"InternalError",
"RequestTimeout",
"ServiceUnavailable",
"Throttling",
}
def is_retryable_upload_error(error: Exception) -> bool:
if isinstance(error, urllib.error.HTTPError):
return error.code in {408, 429} or error.code >= 500
return isinstance(
error,
(urllib.error.URLError, ConnectionError, TimeoutError, OSError),
)
return isinstance(error, (ConnectionError, TimeoutError, OSError))
def upload_file_with_retry(
bucket: Any,
object_key: str,
upload_url: str,
upload_token: str,
local_path: Path,
oss2: Any,
) -> None:
for attempt in range(1, OSS_UPLOAD_MAX_ATTEMPTS + 1):
payload = local_path.read_bytes()
request = urllib.request.Request(
upload_url,
data=payload,
method="PUT",
headers={
"Authorization": f"Bearer {upload_token}",
"Content-Type": "application/octet-stream",
"Content-Length": str(len(payload)),
"X-Artifact-SHA256": hashlib.sha256(payload).hexdigest(),
},
)
for attempt in range(1, UPLOAD_MAX_ATTEMPTS + 1):
try:
bucket.put_object_from_file(object_key, str(local_path))
with urllib.request.urlopen(
request,
timeout=UPLOAD_TIMEOUT_SECONDS,
) as response:
response.read()
return
except Exception as error:
retryable = is_retryable_upload_error(error)
if isinstance(error, urllib.error.HTTPError):
error.close()
if (
attempt >= OSS_UPLOAD_MAX_ATTEMPTS
or not is_retryable_oss_error(error, oss2)
attempt >= UPLOAD_MAX_ATTEMPTS
or not retryable
):
raise
delay = OSS_UPLOAD_BASE_DELAY_SECONDS * (2 ** (attempt - 1))
delay = UPLOAD_BASE_DELAY_SECONDS * (2 ** (attempt - 1))
print(
"firmware-builder: transient OSS upload failure for "
"firmware-builder: transient artifact upload failure for "
f"{local_path.name}; retry {attempt + 1}/"
f"{OSS_UPLOAD_MAX_ATTEMPTS} in {delay}s: {error}",
f"{UPLOAD_MAX_ATTEMPTS} in {delay}s: {error}",
file=sys.stderr,
)
time.sleep(delay)
@@ -365,49 +349,32 @@ def upload_outputs(
config: dict[str, str],
) -> None:
if not manifest.get("job_id"):
raise ValueError("FIRMWARE_JOB_ID is required when OSS upload is enabled")
try:
import oss2
except ImportError as error:
raise RuntimeError("oss2 is required for OSS upload") from error
raise ValueError("FIRMWARE_JOB_ID is required when artifact upload is enabled")
job_id = str(manifest["job_id"])
if not SAFE_JOB_ID.fullmatch(job_id):
raise ValueError(f"Invalid job ID for OSS upload: {job_id!r}")
raise ValueError(f"Invalid job ID for artifact upload: {job_id!r}")
base_key = f"{config['prefix']}/{job_id}"
file_names = ["build.log"]
file_names.extend(str(item["file"]) for item in manifest["artifacts"])
file_names.append("manifest.json")
object_keys = {name: f"{base_key}/{name}" for name in file_names}
manifest["oss"] = {
"bucket": config["bucket"],
"endpoint": config["endpoint"],
"prefix": base_key,
"objects": object_keys,
}
manifest["delivery_status"] = "uploading"
write_manifest(output_dir, manifest)
auth = oss2.Auth(config["access_key_id"], config["access_key_secret"])
bucket = oss2.Bucket(auth, config["endpoint"], config["bucket"])
for name in file_names[:-1]:
upload_file_with_retry(
bucket,
object_keys[name],
f"{config['url']}/{urllib.parse.quote(job_id)}/artifacts/"
f"{urllib.parse.quote(name)}",
config["token"],
output_dir / name,
oss2,
)
manifest["delivery_status"] = "succeeded"
write_manifest(output_dir, manifest)
upload_file_with_retry(
bucket,
object_keys["manifest.json"],
f"{config['url']}/{urllib.parse.quote(job_id)}/artifacts/manifest.json",
config["token"],
output_dir / "manifest.json",
oss2,
)
@@ -416,7 +383,7 @@ def main(argv: Sequence[str] | None = None) -> int:
started_at = utc_now()
try:
validate(args)
upload_config = oss_config()
artifact_upload_config = upload_config()
except ValueError as error:
print(f"firmware-builder: {error}", file=sys.stderr)
return 2
@@ -475,15 +442,15 @@ def main(argv: Sequence[str] | None = None) -> int:
manifest["error"] = failure_summary(log_path)
write_manifest(args.output_dir, manifest)
if upload_config is not None:
if artifact_upload_config is not None:
try:
print("XIAOZHI_STAGE uploading", flush=True)
upload_outputs(args.output_dir, manifest, upload_config)
upload_outputs(args.output_dir, manifest, artifact_upload_config)
except Exception as error:
print(f"firmware-builder: OSS upload failed: {error}", file=sys.stderr)
print(f"firmware-builder: artifact upload failed: {error}", file=sys.stderr)
manifest["status"] = "failed"
manifest["delivery_status"] = "failed"
manifest["error"] = f"OSS upload failed: {error}"
manifest["error"] = f"Artifact upload failed: {error}"
return_code = 1
manifest["exit_code"] = return_code
write_manifest(args.output_dir, manifest)
docker/firmware-builder/test_firmware_builder.py
+58 -89
View File
@@ -3,7 +3,6 @@ import json
import os
import sys
import tempfile
import types
import unittest
from pathlib import Path
from unittest.mock import patch
@@ -154,31 +153,26 @@ sys.exit(%d)
)
def test_success_uploads_outputs_and_manifest_last(self) -> None:
uploads: list[tuple[str, str]] = []
uploads: list[object] = []
class FakeBucket:
def __init__(self, auth: object, endpoint: str, bucket: str) -> None:
self.auth = auth
self.endpoint = endpoint
self.bucket = bucket
class FakeResponse:
def __enter__(self) -> "FakeResponse":
return self
def put_object_from_file(self, key: str, path: str) -> None:
uploads.append((key, path))
def __exit__(self, *args: object) -> None:
return None
def read(self) -> bytes:
return b""
def fake_urlopen(request: object, timeout: int) -> FakeResponse:
uploads.append(request)
self.assertEqual(timeout, firmware_builder.UPLOAD_TIMEOUT_SECONDS)
return FakeResponse()
fake_oss2 = types.SimpleNamespace(
Auth=lambda access_key_id, access_key_secret: (
access_key_id,
access_key_secret,
),
Bucket=FakeBucket,
)
upload_env = {
"FIRMWARE_OSS_UPLOAD": "true",
"FIRMWARE_OSS_ENDPOINT": "oss-cn-shenzhen.aliyuncs.com",
"FIRMWARE_OSS_PREFIX": "custom_firmwares",
"OSS_BUCKET_NAME": "test-bucket",
"OSS_ACCESS_KEY_ID": "test-id",
"OSS_ACCESS_KEY_SECRET": "test-secret",
"FIRMWARE_UPLOAD_URL": "https://example.com/api/firmware-builds",
"FIRMWARE_UPLOAD_TOKEN": "test-token",
}
with tempfile.TemporaryDirectory() as temporary:
@@ -187,7 +181,7 @@ sys.exit(%d)
output = root / "output"
with (
patch.dict(os.environ, upload_env),
patch.dict(sys.modules, {"oss2": fake_oss2}),
patch.object(firmware_builder.urllib.request, "urlopen", fake_urlopen),
):
exit_code = firmware_builder.main(
[
@@ -210,122 +204,97 @@ sys.exit(%d)
self.assertEqual(exit_code, 0)
self.assertEqual(len(uploads), 4)
self.assertEqual(uploads[-1][0], "custom_firmwares/test-job/manifest.json")
self.assertEqual(
uploads[-1].full_url,
"https://example.com/api/firmware-builds/test-job/artifacts/manifest.json",
)
self.assertEqual(uploads[-1].get_header("Authorization"), "Bearer test-token")
manifest = json.loads((output / "manifest.json").read_text())
self.assertEqual(manifest["delivery_status"], "succeeded")
self.assertNotIn("test-secret", json.dumps(manifest))
self.assertNotIn("test-token", json.dumps(manifest))
def test_transient_oss_upload_is_retried_with_exponential_backoff(self) -> None:
class FakeRequestError(Exception):
pass
class FakeBucket:
def test_transient_upload_is_retried_with_exponential_backoff(self) -> None:
attempts = 0
def put_object_from_file(self, key: str, path: str) -> None:
self.attempts += 1
if self.attempts < 3:
raise FakeRequestError("connection timed out")
fake_oss2 = types.SimpleNamespace(
exceptions=types.SimpleNamespace(
RequestError=FakeRequestError,
ServerError=type("FakeServerError", (Exception,), {}),
)
)
def fake_urlopen(request: object, timeout: int) -> object:
nonlocal attempts
attempts += 1
if attempts < 3:
raise firmware_builder.urllib.error.URLError("connection timed out")
return unittest.mock.MagicMock()
with tempfile.TemporaryDirectory() as temporary:
local_path = Path(temporary) / "build.log"
local_path.write_text("build output", encoding="utf-8")
bucket = FakeBucket()
with patch.object(firmware_builder.time, "sleep") as sleep:
with (
patch.object(firmware_builder.time, "sleep") as sleep,
patch.object(firmware_builder.urllib.request, "urlopen", fake_urlopen),
):
firmware_builder.upload_file_with_retry(
bucket,
"firmware/test/build.log",
"https://example.com/build.log",
"test-token",
local_path,
fake_oss2,
)
self.assertEqual(bucket.attempts, 3)
self.assertEqual(attempts, 3)
self.assertEqual(
[call.args[0] for call in sleep.call_args_list],
[1, 2],
)
def test_transient_oss_upload_fails_after_retry_limit(self) -> None:
class FakeRequestError(Exception):
pass
class FakeBucket:
def test_transient_upload_fails_after_retry_limit(self) -> None:
attempts = 0
def put_object_from_file(self, key: str, path: str) -> None:
self.attempts += 1
raise FakeRequestError("connection timed out")
fake_oss2 = types.SimpleNamespace(
exceptions=types.SimpleNamespace(
RequestError=FakeRequestError,
ServerError=type("FakeServerError", (Exception,), {}),
)
)
def fake_urlopen(request: object, timeout: int) -> object:
nonlocal attempts
attempts += 1
raise firmware_builder.urllib.error.URLError("connection timed out")
with tempfile.TemporaryDirectory() as temporary:
local_path = Path(temporary) / "build.log"
local_path.write_text("build output", encoding="utf-8")
bucket = FakeBucket()
with (
patch.object(firmware_builder.time, "sleep") as sleep,
self.assertRaisesRegex(FakeRequestError, "connection timed out"),
patch.object(firmware_builder.urllib.request, "urlopen", fake_urlopen),
self.assertRaisesRegex(firmware_builder.urllib.error.URLError, "connection timed out"),
):
firmware_builder.upload_file_with_retry(
bucket,
"firmware/test/build.log",
"https://example.com/build.log",
"test-token",
local_path,
fake_oss2,
)
self.assertEqual(bucket.attempts, firmware_builder.OSS_UPLOAD_MAX_ATTEMPTS)
self.assertEqual(attempts, firmware_builder.UPLOAD_MAX_ATTEMPTS)
self.assertEqual(
[call.args[0] for call in sleep.call_args_list],
[1, 2, 4],
)
def test_permanent_oss_upload_error_is_not_retried(self) -> None:
class FakeServerError(Exception):
status = 403
code = "AccessDenied"
class FakeBucket:
def test_permanent_upload_error_is_not_retried(self) -> None:
attempts = 0
def put_object_from_file(self, key: str, path: str) -> None:
self.attempts += 1
raise FakeServerError("access denied")
fake_oss2 = types.SimpleNamespace(
exceptions=types.SimpleNamespace(
RequestError=type("FakeRequestError", (Exception,), {}),
ServerError=FakeServerError,
)
def fake_urlopen(request: object, timeout: int) -> object:
nonlocal attempts
attempts += 1
raise firmware_builder.urllib.error.HTTPError(
request.full_url, 403, "Forbidden", {}, None
)
with tempfile.TemporaryDirectory() as temporary:
local_path = Path(temporary) / "build.log"
local_path.write_text("build output", encoding="utf-8")
bucket = FakeBucket()
with (
patch.object(firmware_builder.time, "sleep") as sleep,
self.assertRaisesRegex(FakeServerError, "access denied"),
patch.object(firmware_builder.urllib.request, "urlopen", fake_urlopen),
self.assertRaisesRegex(firmware_builder.urllib.error.HTTPError, "403"),
):
firmware_builder.upload_file_with_retry(
bucket,
"firmware/test/build.log",
"https://example.com/build.log",
"test-token",
local_path,
fake_oss2,
)
self.assertEqual(bucket.attempts, 1)
self.assertEqual(attempts, 1)
sleep.assert_not_called()
def test_rejects_path_traversal_board(self) -> None: