From a6a4cc514352ac4c939cd5fe518c3d3e65928fac Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Sun, 10 May 2026 14:25:47 +0900 Subject: [PATCH] feat: improve examples auto-run coverage and artifact handling (#3328) --- .gitignore | 1 + examples/basic/non_strict_output_type.py | 9 +- examples/run_examples.py | 18 +++- .../daytona/usaspending_text2sql/setup_db.py | 24 ++++- .../tutorials/vision_website_clone/main.py | 11 ++- tests/test_run_examples_script.py | 14 +++ tests/test_usaspending_setup_db.py | 87 +++++++++++++++++++ 7 files changed, 154 insertions(+), 10 deletions(-) create mode 100644 tests/test_usaspending_setup_db.py diff --git a/.gitignore b/.gitignore index 621e4cec..4f350585 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,4 @@ tmp/ # execplans plans/ +.vercel diff --git a/examples/basic/non_strict_output_type.py b/examples/basic/non_strict_output_type.py index 49fcc4e2..fcb7e4f3 100644 --- a/examples/basic/non_strict_output_type.py +++ b/examples/basic/non_strict_output_type.py @@ -3,7 +3,7 @@ import json from dataclasses import dataclass from typing import Any -from agents import Agent, AgentOutputSchema, AgentOutputSchemaBase, Runner +from agents import Agent, AgentOutputSchema, AgentOutputSchemaBase, ModelBehaviorError, Runner """This example demonstrates how to use an output type that is not in strict mode. Strict mode allows us to guarantee valid JSON output, but some schemas are not strict-compatible. @@ -68,8 +68,11 @@ async def main(): # In some cases, it will raise an error - the schema isn't strict, so the model may # produce an invalid JSON object. agent.output_type = AgentOutputSchema(OutputType, strict_json_schema=False) - result = await Runner.run(agent, input) - print(result.final_output) + try: + result = await Runner.run(agent, input) + print(result.final_output) + except ModelBehaviorError as e: + print(f"Non-strict output validation failed (expected possibility): {e}") # Finally, let's try a custom output type. agent.output_type = CustomOutputSchema() diff --git a/examples/run_examples.py b/examples/run_examples.py index 417e0378..54038b9f 100644 --- a/examples/run_examples.py +++ b/examples/run_examples.py @@ -35,6 +35,7 @@ EXAMPLES_DIR = ROOT_DIR / "examples" MAIN_PATTERN = re.compile(r"__name__\s*==\s*['\"]__main__['\"]") LOG_DIR_DEFAULT = ROOT_DIR / ".tmp" / "examples-start-logs" +ARTIFACTS_DIR_DEFAULT = ROOT_DIR / ".tmp" / "examples-artifacts" RERUN_FILE_DEFAULT = ROOT_DIR / ".tmp" / "examples-rerun.txt" DEFAULT_MAIN_LOG = LOG_DIR_DEFAULT / f"main_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}.log" REDIS_SESSION_EXAMPLE = "examples/memory/redis_session_example.py" @@ -58,8 +59,6 @@ DISCOVERY_EXCLUDE = { # Examples that are noisy, require extra credentials, or hang in auto runs. DEFAULT_AUTO_SKIP = { "examples/agent_patterns/llm_as_a_judge.py", - "examples/agent_patterns/routing.py", - "examples/customer_service/main.py", "examples/hosted_mcp/connectors.py", "examples/mcp/git_example/main.py", # These are helper daemons or multi-process components exercised by sibling examples. @@ -416,6 +415,11 @@ def parse_args() -> argparse.Namespace: default=str(DEFAULT_MAIN_LOG), help="Path to write the main summary log.", ) + parser.add_argument( + "--artifacts-dir", + default=str(ARTIFACTS_DIR_DEFAULT), + help="Directory for example-generated artifacts.", + ) parser.add_argument( "--rerun-file", help="Only run examples listed in this file (one relative path per line).", @@ -580,6 +584,12 @@ def ensure_dirs(path: Path, is_file: bool | None = None) -> None: target.mkdir(parents=True, exist_ok=True) +def artifact_dir_for_example(relpath: str, artifacts_dir: Path) -> Path: + """Return a deterministic scratch directory for one example run.""" + stem = normalize_relpath(str(Path(relpath).with_suffix(""))) + return artifacts_dir / stem.replace("/", "__") + + def parse_rerun_from_log(log_path: Path) -> list[str]: if not log_path.exists(): raise FileNotFoundError(log_path) @@ -610,6 +620,7 @@ def run_examples(examples: Sequence[ExampleScript], args: argparse.Namespace) -> overrides.add("external") logs_dir = Path(args.logs_dir).resolve() + artifacts_dir = Path(args.artifacts_dir).resolve() main_log_path = Path(args.main_log).resolve() auto_mode = args.auto_mode or os.environ.get("EXAMPLES_INTERACTIVE_MODE", "").lower() == "auto" auto_skip_set = load_auto_skip() @@ -618,6 +629,7 @@ def run_examples(examples: Sequence[ExampleScript], args: argparse.Namespace) -> overrides.add("interactive") ensure_dirs(logs_dir, is_file=False) + ensure_dirs(artifacts_dir, is_file=False) ensure_dirs(main_log_path, is_file=True) rerun_entries: list[str] = [] @@ -659,6 +671,7 @@ def run_examples(examples: Sequence[ExampleScript], args: argparse.Namespace) -> env = os.environ.copy() env["PATH"] = command_path env["PYTHONPATH"] = build_python_path(env.get("PYTHONPATH")) + env["EXAMPLES_ARTIFACTS_DIR"] = str(artifact_dir_for_example(relpath, artifacts_dir)) if auto_mode: env["EXAMPLES_INTERACTIVE_MODE"] = "auto" env["APPLY_PATCH_AUTO_APPROVE"] = "1" @@ -759,6 +772,7 @@ def run_examples(examples: Sequence[ExampleScript], args: argparse.Namespace) -> safe_write_main(f"# include: {sorted(overrides)}") safe_write_main(f"# auto_mode: {auto_mode}") safe_write_main(f"# logs_dir: {logs_dir}") + safe_write_main(f"# artifacts_dir: {artifacts_dir}") safe_write_main(f"# jobs: {jobs}") safe_write_main(f"# buffer_output: {buffer_output}") safe_write_main(f"# path_augmented: {path_augmented}") diff --git a/examples/sandbox/extensions/daytona/usaspending_text2sql/setup_db.py b/examples/sandbox/extensions/daytona/usaspending_text2sql/setup_db.py index cec79428..5be4ab3d 100644 --- a/examples/sandbox/extensions/daytona/usaspending_text2sql/setup_db.py +++ b/examples/sandbox/extensions/daytona/usaspending_text2sql/setup_db.py @@ -17,8 +17,11 @@ from __future__ import annotations import argparse import concurrent.futures import csv +import functools import json +import os import sqlite3 +import ssl import sys import time import urllib.error @@ -27,9 +30,10 @@ import zipfile from pathlib import Path from typing import Any -DB_DIR = Path("data") +ARTIFACT_ROOT = Path(os.environ.get("EXAMPLES_ARTIFACTS_DIR", ".")) +DB_DIR = ARTIFACT_ROOT / "data" DB_PATH = DB_DIR / "usaspending.db" -GLOSSARY_PATH = Path("schema") / "glossary.md" +GLOSSARY_PATH = ARTIFACT_ROOT / "schema" / "glossary.md" USASPENDING_API = "https://api.usaspending.gov" BULK_DOWNLOAD_ENDPOINT = f"{USASPENDING_API}/api/v2/bulk_download/awards/" @@ -118,14 +122,26 @@ CREATE INDEX IF NOT EXISTS idx_spending_awarding_office ON spending(awarding_off # --------------------------------------------------------------------------- +@functools.cache +def _urlopen_ssl_context() -> ssl.SSLContext | None: + """Use certifi's CA bundle when available, otherwise keep stdlib defaults.""" + try: + import certifi + except ImportError: + return None + + return ssl.create_default_context(cafile=certifi.where()) + + def _urlopen_with_retry( req: urllib.request.Request, *, timeout: int = 60, retries: int = 3 ) -> bytes: """urlopen with retries for the flaky USAspending endpoints.""" last_exc: Exception | None = None + ssl_context = _urlopen_ssl_context() for attempt in range(1, retries + 1): try: - with urllib.request.urlopen(req, timeout=timeout) as resp: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context) as resp: return bytes(resp.read()) except (urllib.error.URLError, ConnectionError, OSError) as e: last_exc = e @@ -600,7 +616,7 @@ def main() -> None: elif DB_PATH.exists(): DB_PATH.unlink() - tmp_dir = Path("data/tmp_download") + tmp_dir = DB_DIR / "tmp_download" print("=== NASA USAspending Database Builder ===") print(f"Fiscal years: {args.start_fy} - {args.end_fy}\n") diff --git a/examples/sandbox/tutorials/vision_website_clone/main.py b/examples/sandbox/tutorials/vision_website_clone/main.py index 6b829049..5d13321f 100644 --- a/examples/sandbox/tutorials/vision_website_clone/main.py +++ b/examples/sandbox/tutorials/vision_website_clone/main.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse import asyncio +import os import sys from pathlib import Path from textwrap import dedent @@ -68,6 +69,14 @@ AGENTS_MD = dedent( ) +def default_output_dir() -> Path: + """Return the local directory for copied example artifacts.""" + artifacts_dir = os.environ.get("EXAMPLES_ARTIFACTS_DIR") + if artifacts_dir: + return Path(artifacts_dir) + return DEMO_DIR / "output" + + def build_manifest() -> Manifest: return Manifest( entries={ @@ -236,7 +245,7 @@ if __name__ == "__main__": parser.add_argument( "--output-dir", type=Path, - default=DEMO_DIR / "output", + default=default_output_dir(), help="Directory for copied website files.", ) args = parser.parse_args() diff --git a/tests/test_run_examples_script.py b/tests/test_run_examples_script.py index a669bbf5..09794c45 100644 --- a/tests/test_run_examples_script.py +++ b/tests/test_run_examples_script.py @@ -34,6 +34,11 @@ def test_default_auto_skip_keeps_computer_use_example_enabled() -> None: assert "examples/tools/computer_use.py" not in run_examples.DEFAULT_AUTO_SKIP +def test_default_auto_skip_keeps_one_turn_auto_examples_enabled() -> None: + assert "examples/agent_patterns/routing.py" not in run_examples.DEFAULT_AUTO_SKIP + assert "examples/customer_service/main.py" not in run_examples.DEFAULT_AUTO_SKIP + + def test_example_command_runs_python_unbuffered(monkeypatch) -> None: monkeypatch.delenv("EXAMPLES_UV_EXTRAS", raising=False) example = run_examples.ExampleScript( @@ -63,6 +68,15 @@ def test_example_command_includes_configured_uv_extras(monkeypatch) -> None: ] +def test_artifact_dir_for_example_uses_tmp_safe_stem(tmp_path: Path) -> None: + artifact_dir = run_examples.artifact_dir_for_example( + "examples/sandbox/tutorials/vision_website_clone/main.py", + tmp_path, + ) + + assert artifact_dir == tmp_path / "examples__sandbox__tutorials__vision_website_clone__main" + + def test_prepare_redis_for_example_uses_existing_local_redis(monkeypatch) -> None: env: dict[str, str] = {} monkeypatch.setattr(run_examples, "redis_ping_url", lambda url, timeout=0.5: True) diff --git a/tests/test_usaspending_setup_db.py b/tests/test_usaspending_setup_db.py new file mode 100644 index 00000000..21a7e150 --- /dev/null +++ b/tests/test_usaspending_setup_db.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import importlib +import ssl +import sys +import types +import urllib.request +from pathlib import Path +from typing import Any + +from examples.sandbox.extensions.daytona.usaspending_text2sql import setup_db + + +def test_paths_use_examples_artifacts_dir_when_set(monkeypatch: Any, tmp_path: Path) -> None: + monkeypatch.setenv("EXAMPLES_ARTIFACTS_DIR", str(tmp_path)) + reloaded = importlib.reload(setup_db) + + try: + assert reloaded.DB_PATH == tmp_path / "data" / "usaspending.db" + assert reloaded.GLOSSARY_PATH == tmp_path / "schema" / "glossary.md" + finally: + monkeypatch.delenv("EXAMPLES_ARTIFACTS_DIR", raising=False) + importlib.reload(setup_db) + + +def test_urlopen_ssl_context_uses_certifi_when_available(monkeypatch: Any) -> None: + setup_db._urlopen_ssl_context.cache_clear() + ssl_context = object() + certifi = types.SimpleNamespace(where=lambda: "/tmp/certifi.pem") + monkeypatch.setitem(sys.modules, "certifi", certifi) + + def fake_create_default_context(*, cafile: str) -> object: + assert cafile == "/tmp/certifi.pem" + return ssl_context + + monkeypatch.setattr(ssl, "create_default_context", fake_create_default_context) + + try: + assert setup_db._urlopen_ssl_context() is ssl_context + finally: + setup_db._urlopen_ssl_context.cache_clear() + + +def test_urlopen_ssl_context_falls_back_without_certifi(monkeypatch: Any) -> None: + setup_db._urlopen_ssl_context.cache_clear() + monkeypatch.setitem(sys.modules, "certifi", None) + + def fail_create_default_context(**kwargs: object) -> object: + raise AssertionError("stdlib-only fallback should not create a certifi SSL context") + + monkeypatch.setattr(ssl, "create_default_context", fail_create_default_context) + + try: + assert setup_db._urlopen_ssl_context() is None + finally: + setup_db._urlopen_ssl_context.cache_clear() + + +def test_urlopen_with_retry_passes_optional_ssl_context(monkeypatch: Any) -> None: + ssl_context = object() + captured: dict[str, object] = {} + + class DummyResponse: + def __enter__(self) -> DummyResponse: + return self + + def __exit__(self, *args: object) -> None: + return None + + def read(self) -> bytes: + return b"ok" + + def fake_urlopen( + req: urllib.request.Request, *, timeout: int, context: object | None + ) -> DummyResponse: + captured["req"] = req + captured["timeout"] = timeout + captured["context"] = context + return DummyResponse() + + monkeypatch.setattr(setup_db, "_urlopen_ssl_context", lambda: ssl_context) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + req = urllib.request.Request("https://api.usaspending.gov") + + assert setup_db._urlopen_with_retry(req, timeout=12, retries=1) == b"ok" + assert captured == {"req": req, "timeout": 12, "context": ssl_context}