diff --git a/docker/firmware-builder/Dockerfile b/docker/firmware-builder/Dockerfile index c8bf6fa..ed1c5a4 100644 --- a/docker/firmware-builder/Dockerfile +++ b/docker/firmware-builder/Dockerfile @@ -20,7 +20,6 @@ RUN chmod 0755 \ docker/firmware-builder/entrypoint.sh \ docker/firmware-builder/firmware_builder.py \ && . "${IDF_PATH}/export.sh" >/dev/null \ - && python3 -m pip install --no-cache-dir oss2==2.19.1 \ && idf.py --version \ && python3 scripts/build.py --list-boards --json >/tmp/xiaozhi-boards.json \ && python3 scripts/build.py --list-languages --json >/tmp/xiaozhi-languages.json diff --git a/docker/firmware-builder/README.md b/docker/firmware-builder/README.md index 9b1a86d..08bf892 100644 --- a/docker/firmware-builder/README.md +++ b/docker/firmware-builder/README.md @@ -53,28 +53,26 @@ Each successful job writes: - `manifest.json`: inputs, tool versions, source revision, sizes, and SHA-256 checksums. -To upload the job output to OSS, also pass: +To upload the job output to an HTTP artifact receiver, also pass: ```text -FIRMWARE_OSS_UPLOAD=true -FIRMWARE_OSS_ENDPOINT=oss-cn-shenzhen.aliyuncs.com -FIRMWARE_OSS_PREFIX=custom_firmwares -OSS_BUCKET_NAME= -OSS_ACCESS_KEY_ID= -OSS_ACCESS_KEY_SECRET= +FIRMWARE_UPLOAD_URL=https://example.com/api/firmware-builds +FIRMWARE_UPLOAD_TOKEN= FIRMWARE_JOB_ID= ``` -The builder uploads the two firmware images, `build.log`, and `manifest.json` -to `custom_firmwares//`. The manifest is uploaded last so consumers do -not observe a completed job before its other objects are available. Transient -OSS connection, timeout, throttling, and server errors are retried up to four -times with exponential backoff; authentication and other permanent errors fail -immediately. +The builder sends an authenticated HTTP `PUT` for the two firmware images, +`build.log`, and `manifest.json` to +`//artifacts/`. The manifest is uploaded last so +consumers do not observe a completed job before its other objects are available. +Transient connection, timeout, throttling, and server errors are retried up to +four times with exponential backoff; authentication and other permanent errors +fail immediately. Storage credentials and provider details remain entirely on +the receiving service. Use a unique empty output directory for each job. In ECI, pass the same inputs -as container environment variables and upload the output directory to OSS after -the process exits. +as container environment variables and let the receiver persist the output +after the process exits. ESP-IDF uses Ninja, which automatically builds in parallel using the CPUs visible to the container. Allocate at least 8 vCPUs to an ECI build job when diff --git a/docker/firmware-builder/firmware_builder.py b/docker/firmware-builder/firmware_builder.py index 92bb343..d632aea 100644 --- a/docker/firmware-builder/firmware_builder.py +++ b/docker/firmware-builder/firmware_builder.py @@ -17,6 +17,9 @@ import shutil import subprocess import sys import time +import urllib.error +import urllib.parse +import urllib.request from datetime import datetime, timezone from pathlib import Path from typing import Any, Sequence @@ -30,8 +33,9 @@ ARTIFACTS = { "ota": Path("build/xiaozhi.bin"), "full": Path("build/merged-binary.bin"), } -OSS_UPLOAD_MAX_ATTEMPTS = 4 -OSS_UPLOAD_BASE_DELAY_SECONDS = 1 +UPLOAD_MAX_ATTEMPTS = 4 +UPLOAD_BASE_DELAY_SECONDS = 1 +UPLOAD_TIMEOUT_SECONDS = 120 def utc_now() -> str: @@ -43,42 +47,19 @@ def env(name: str) -> str | None: return value.strip() if value and value.strip() else None -def env_enabled(name: str) -> bool: - return (env(name) or "").casefold() in {"1", "true", "yes", "on"} - - -def oss_config() -> dict[str, str] | None: - if not env_enabled("FIRMWARE_OSS_UPLOAD"): +def upload_config() -> dict[str, str] | None: + upload_url = env("FIRMWARE_UPLOAD_URL") + upload_token = env("FIRMWARE_UPLOAD_TOKEN") + if not upload_url and not upload_token: return None - - values = { - "access_key_id": env("OSS_ACCESS_KEY_ID"), - "access_key_secret": env("OSS_ACCESS_KEY_SECRET"), - "bucket": env("OSS_BUCKET_NAME"), - "endpoint": env("FIRMWARE_OSS_ENDPOINT"), - "prefix": env("FIRMWARE_OSS_PREFIX") or "custom_firmwares", - } - missing = [name for name, value in values.items() if not value] - if missing: + if not upload_url or not upload_token: raise ValueError( - "OSS upload is enabled but configuration is missing: " - + ", ".join(missing) + "FIRMWARE_UPLOAD_URL and FIRMWARE_UPLOAD_TOKEN must be configured together" ) - - prefix = str(values["prefix"]).strip("/") - if not prefix or ".." in Path(prefix).parts: - raise ValueError(f"Invalid OSS prefix: {prefix!r}") - endpoint = str(values["endpoint"]) - if not endpoint.startswith(("http://", "https://")): - endpoint = "https://" + endpoint - - return { - "access_key_id": str(values["access_key_id"]), - "access_key_secret": str(values["access_key_secret"]), - "bucket": str(values["bucket"]), - "endpoint": endpoint, - "prefix": prefix, - } + parsed_url = urllib.parse.urlparse(upload_url) + if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc: + raise ValueError("FIRMWARE_UPLOAD_URL must be an absolute HTTP(S) URL") + return {"url": upload_url.rstrip("/"), "token": upload_token} def parser() -> argparse.ArgumentParser: @@ -309,51 +290,54 @@ def failure_summary(log_path: Path) -> str: return meaningful[-1][:500] if meaningful else "Firmware build failed" -def is_retryable_oss_error(error: Exception, oss2: Any) -> bool: - exceptions = getattr(oss2, "exceptions", None) - request_error = getattr(exceptions, "RequestError", None) - if isinstance(request_error, type) and isinstance(error, request_error): - return True - - server_error = getattr(exceptions, "ServerError", None) - if isinstance(server_error, type) and isinstance(error, server_error): - status = getattr(error, "status", getattr(error, "status_code", None)) - code = str(getattr(error, "code", "")) - return ( - status in {408, 429} - or isinstance(status, int) and status >= 500 - or code in { - "InternalError", - "RequestTimeout", - "ServiceUnavailable", - "Throttling", - } - ) - - return isinstance(error, (ConnectionError, TimeoutError, OSError)) +def is_retryable_upload_error(error: Exception) -> bool: + if isinstance(error, urllib.error.HTTPError): + return error.code in {408, 429} or error.code >= 500 + return isinstance( + error, + (urllib.error.URLError, ConnectionError, TimeoutError, OSError), + ) def upload_file_with_retry( - bucket: Any, - object_key: str, + upload_url: str, + upload_token: str, local_path: Path, - oss2: Any, ) -> None: - for attempt in range(1, OSS_UPLOAD_MAX_ATTEMPTS + 1): + payload = local_path.read_bytes() + request = urllib.request.Request( + upload_url, + data=payload, + method="PUT", + headers={ + "Authorization": f"Bearer {upload_token}", + "Content-Type": "application/octet-stream", + "Content-Length": str(len(payload)), + "X-Artifact-SHA256": hashlib.sha256(payload).hexdigest(), + }, + ) + for attempt in range(1, UPLOAD_MAX_ATTEMPTS + 1): try: - bucket.put_object_from_file(object_key, str(local_path)) + with urllib.request.urlopen( + request, + timeout=UPLOAD_TIMEOUT_SECONDS, + ) as response: + response.read() return except Exception as error: + retryable = is_retryable_upload_error(error) + if isinstance(error, urllib.error.HTTPError): + error.close() if ( - attempt >= OSS_UPLOAD_MAX_ATTEMPTS - or not is_retryable_oss_error(error, oss2) + attempt >= UPLOAD_MAX_ATTEMPTS + or not retryable ): raise - delay = OSS_UPLOAD_BASE_DELAY_SECONDS * (2 ** (attempt - 1)) + delay = UPLOAD_BASE_DELAY_SECONDS * (2 ** (attempt - 1)) print( - "firmware-builder: transient OSS upload failure for " + "firmware-builder: transient artifact upload failure for " f"{local_path.name}; retry {attempt + 1}/" - f"{OSS_UPLOAD_MAX_ATTEMPTS} in {delay}s: {error}", + f"{UPLOAD_MAX_ATTEMPTS} in {delay}s: {error}", file=sys.stderr, ) time.sleep(delay) @@ -365,49 +349,32 @@ def upload_outputs( config: dict[str, str], ) -> None: if not manifest.get("job_id"): - raise ValueError("FIRMWARE_JOB_ID is required when OSS upload is enabled") - - try: - import oss2 - except ImportError as error: - raise RuntimeError("oss2 is required for OSS upload") from error + raise ValueError("FIRMWARE_JOB_ID is required when artifact upload is enabled") job_id = str(manifest["job_id"]) if not SAFE_JOB_ID.fullmatch(job_id): - raise ValueError(f"Invalid job ID for OSS upload: {job_id!r}") + raise ValueError(f"Invalid job ID for artifact upload: {job_id!r}") - base_key = f"{config['prefix']}/{job_id}" file_names = ["build.log"] file_names.extend(str(item["file"]) for item in manifest["artifacts"]) file_names.append("manifest.json") - object_keys = {name: f"{base_key}/{name}" for name in file_names} - - manifest["oss"] = { - "bucket": config["bucket"], - "endpoint": config["endpoint"], - "prefix": base_key, - "objects": object_keys, - } manifest["delivery_status"] = "uploading" write_manifest(output_dir, manifest) - auth = oss2.Auth(config["access_key_id"], config["access_key_secret"]) - bucket = oss2.Bucket(auth, config["endpoint"], config["bucket"]) for name in file_names[:-1]: upload_file_with_retry( - bucket, - object_keys[name], + f"{config['url']}/{urllib.parse.quote(job_id)}/artifacts/" + f"{urllib.parse.quote(name)}", + config["token"], output_dir / name, - oss2, ) manifest["delivery_status"] = "succeeded" write_manifest(output_dir, manifest) upload_file_with_retry( - bucket, - object_keys["manifest.json"], + f"{config['url']}/{urllib.parse.quote(job_id)}/artifacts/manifest.json", + config["token"], output_dir / "manifest.json", - oss2, ) @@ -416,7 +383,7 @@ def main(argv: Sequence[str] | None = None) -> int: started_at = utc_now() try: validate(args) - upload_config = oss_config() + artifact_upload_config = upload_config() except ValueError as error: print(f"firmware-builder: {error}", file=sys.stderr) return 2 @@ -475,15 +442,15 @@ def main(argv: Sequence[str] | None = None) -> int: manifest["error"] = failure_summary(log_path) write_manifest(args.output_dir, manifest) - if upload_config is not None: + if artifact_upload_config is not None: try: print("XIAOZHI_STAGE uploading", flush=True) - upload_outputs(args.output_dir, manifest, upload_config) + upload_outputs(args.output_dir, manifest, artifact_upload_config) except Exception as error: - print(f"firmware-builder: OSS upload failed: {error}", file=sys.stderr) + print(f"firmware-builder: artifact upload failed: {error}", file=sys.stderr) manifest["status"] = "failed" manifest["delivery_status"] = "failed" - manifest["error"] = f"OSS upload failed: {error}" + manifest["error"] = f"Artifact upload failed: {error}" return_code = 1 manifest["exit_code"] = return_code write_manifest(args.output_dir, manifest) diff --git a/docker/firmware-builder/test_firmware_builder.py b/docker/firmware-builder/test_firmware_builder.py index e402d33..99f9e81 100644 --- a/docker/firmware-builder/test_firmware_builder.py +++ b/docker/firmware-builder/test_firmware_builder.py @@ -3,7 +3,6 @@ import json import os import sys import tempfile -import types import unittest from pathlib import Path from unittest.mock import patch @@ -154,31 +153,26 @@ sys.exit(%d) ) def test_success_uploads_outputs_and_manifest_last(self) -> None: - uploads: list[tuple[str, str]] = [] + uploads: list[object] = [] - class FakeBucket: - def __init__(self, auth: object, endpoint: str, bucket: str) -> None: - self.auth = auth - self.endpoint = endpoint - self.bucket = bucket + class FakeResponse: + def __enter__(self) -> "FakeResponse": + return self - def put_object_from_file(self, key: str, path: str) -> None: - uploads.append((key, path)) + def __exit__(self, *args: object) -> None: + return None + + def read(self) -> bytes: + return b"" + + def fake_urlopen(request: object, timeout: int) -> FakeResponse: + uploads.append(request) + self.assertEqual(timeout, firmware_builder.UPLOAD_TIMEOUT_SECONDS) + return FakeResponse() - fake_oss2 = types.SimpleNamespace( - Auth=lambda access_key_id, access_key_secret: ( - access_key_id, - access_key_secret, - ), - Bucket=FakeBucket, - ) upload_env = { - "FIRMWARE_OSS_UPLOAD": "true", - "FIRMWARE_OSS_ENDPOINT": "oss-cn-shenzhen.aliyuncs.com", - "FIRMWARE_OSS_PREFIX": "custom_firmwares", - "OSS_BUCKET_NAME": "test-bucket", - "OSS_ACCESS_KEY_ID": "test-id", - "OSS_ACCESS_KEY_SECRET": "test-secret", + "FIRMWARE_UPLOAD_URL": "https://example.com/api/firmware-builds", + "FIRMWARE_UPLOAD_TOKEN": "test-token", } with tempfile.TemporaryDirectory() as temporary: @@ -187,7 +181,7 @@ sys.exit(%d) output = root / "output" with ( patch.dict(os.environ, upload_env), - patch.dict(sys.modules, {"oss2": fake_oss2}), + patch.object(firmware_builder.urllib.request, "urlopen", fake_urlopen), ): exit_code = firmware_builder.main( [ @@ -210,122 +204,97 @@ sys.exit(%d) self.assertEqual(exit_code, 0) self.assertEqual(len(uploads), 4) - self.assertEqual(uploads[-1][0], "custom_firmwares/test-job/manifest.json") + self.assertEqual( + uploads[-1].full_url, + "https://example.com/api/firmware-builds/test-job/artifacts/manifest.json", + ) + self.assertEqual(uploads[-1].get_header("Authorization"), "Bearer test-token") manifest = json.loads((output / "manifest.json").read_text()) self.assertEqual(manifest["delivery_status"], "succeeded") - self.assertNotIn("test-secret", json.dumps(manifest)) + self.assertNotIn("test-token", json.dumps(manifest)) - def test_transient_oss_upload_is_retried_with_exponential_backoff(self) -> None: - class FakeRequestError(Exception): - pass + def test_transient_upload_is_retried_with_exponential_backoff(self) -> None: + attempts = 0 - class FakeBucket: - attempts = 0 - - def put_object_from_file(self, key: str, path: str) -> None: - self.attempts += 1 - if self.attempts < 3: - raise FakeRequestError("connection timed out") - - fake_oss2 = types.SimpleNamespace( - exceptions=types.SimpleNamespace( - RequestError=FakeRequestError, - ServerError=type("FakeServerError", (Exception,), {}), - ) - ) + def fake_urlopen(request: object, timeout: int) -> object: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise firmware_builder.urllib.error.URLError("connection timed out") + return unittest.mock.MagicMock() with tempfile.TemporaryDirectory() as temporary: local_path = Path(temporary) / "build.log" local_path.write_text("build output", encoding="utf-8") - bucket = FakeBucket() - with patch.object(firmware_builder.time, "sleep") as sleep: + with ( + patch.object(firmware_builder.time, "sleep") as sleep, + patch.object(firmware_builder.urllib.request, "urlopen", fake_urlopen), + ): firmware_builder.upload_file_with_retry( - bucket, - "firmware/test/build.log", + "https://example.com/build.log", + "test-token", local_path, - fake_oss2, ) - self.assertEqual(bucket.attempts, 3) + self.assertEqual(attempts, 3) self.assertEqual( [call.args[0] for call in sleep.call_args_list], [1, 2], ) - def test_transient_oss_upload_fails_after_retry_limit(self) -> None: - class FakeRequestError(Exception): - pass + def test_transient_upload_fails_after_retry_limit(self) -> None: + attempts = 0 - class FakeBucket: - attempts = 0 - - def put_object_from_file(self, key: str, path: str) -> None: - self.attempts += 1 - raise FakeRequestError("connection timed out") - - fake_oss2 = types.SimpleNamespace( - exceptions=types.SimpleNamespace( - RequestError=FakeRequestError, - ServerError=type("FakeServerError", (Exception,), {}), - ) - ) + def fake_urlopen(request: object, timeout: int) -> object: + nonlocal attempts + attempts += 1 + raise firmware_builder.urllib.error.URLError("connection timed out") with tempfile.TemporaryDirectory() as temporary: local_path = Path(temporary) / "build.log" local_path.write_text("build output", encoding="utf-8") - bucket = FakeBucket() with ( patch.object(firmware_builder.time, "sleep") as sleep, - self.assertRaisesRegex(FakeRequestError, "connection timed out"), + patch.object(firmware_builder.urllib.request, "urlopen", fake_urlopen), + self.assertRaisesRegex(firmware_builder.urllib.error.URLError, "connection timed out"), ): firmware_builder.upload_file_with_retry( - bucket, - "firmware/test/build.log", + "https://example.com/build.log", + "test-token", local_path, - fake_oss2, ) - self.assertEqual(bucket.attempts, firmware_builder.OSS_UPLOAD_MAX_ATTEMPTS) + self.assertEqual(attempts, firmware_builder.UPLOAD_MAX_ATTEMPTS) self.assertEqual( [call.args[0] for call in sleep.call_args_list], [1, 2, 4], ) - def test_permanent_oss_upload_error_is_not_retried(self) -> None: - class FakeServerError(Exception): - status = 403 - code = "AccessDenied" + def test_permanent_upload_error_is_not_retried(self) -> None: + attempts = 0 - class FakeBucket: - attempts = 0 - - def put_object_from_file(self, key: str, path: str) -> None: - self.attempts += 1 - raise FakeServerError("access denied") - - fake_oss2 = types.SimpleNamespace( - exceptions=types.SimpleNamespace( - RequestError=type("FakeRequestError", (Exception,), {}), - ServerError=FakeServerError, + def fake_urlopen(request: object, timeout: int) -> object: + nonlocal attempts + attempts += 1 + raise firmware_builder.urllib.error.HTTPError( + request.full_url, 403, "Forbidden", {}, None ) - ) with tempfile.TemporaryDirectory() as temporary: local_path = Path(temporary) / "build.log" local_path.write_text("build output", encoding="utf-8") - bucket = FakeBucket() with ( patch.object(firmware_builder.time, "sleep") as sleep, - self.assertRaisesRegex(FakeServerError, "access denied"), + patch.object(firmware_builder.urllib.request, "urlopen", fake_urlopen), + self.assertRaisesRegex(firmware_builder.urllib.error.HTTPError, "403"), ): firmware_builder.upload_file_with_retry( - bucket, - "firmware/test/build.log", + "https://example.com/build.log", + "test-token", local_path, - fake_oss2, ) - self.assertEqual(bucket.attempts, 1) + self.assertEqual(attempts, 1) sleep.assert_not_called() def test_rejects_path_traversal_board(self) -> None: