feat: improve examples auto-run coverage and artifact handling (#3328)

2026-05-10 14:25:47 +09:00
parent 94ba76de0f
commit a6a4cc5143
7 changed files with 154 additions and 10 deletions
@@ -158,3 +158,4 @@ tmp/

 # execplans
 plans/
+.vercel
@@ -3,7 +3,7 @@ import json
 from dataclasses import dataclass
 from typing import Any

-from agents import Agent, AgentOutputSchema, AgentOutputSchemaBase, Runner
+from agents import Agent, AgentOutputSchema, AgentOutputSchemaBase, ModelBehaviorError, Runner

 """This example demonstrates how to use an output type that is not in strict mode. Strict mode
 allows us to guarantee valid JSON output, but some schemas are not strict-compatible.
@@ -68,8 +68,11 @@ async def main():
    # In some cases, it will raise an error - the schema isn't strict, so the model may
    # produce an invalid JSON object.
    agent.output_type = AgentOutputSchema(OutputType, strict_json_schema=False)
-    result = await Runner.run(agent, input)
-    print(result.final_output)
+    try:
+        result = await Runner.run(agent, input)
+        print(result.final_output)
+    except ModelBehaviorError as e:
+        print(f"Non-strict output validation failed (expected possibility): {e}")

    # Finally, let's try a custom output type.
    agent.output_type = CustomOutputSchema()
@@ -35,6 +35,7 @@ EXAMPLES_DIR = ROOT_DIR / "examples"
 MAIN_PATTERN = re.compile(r"__name__\s*==\s*['\"]__main__['\"]")

 LOG_DIR_DEFAULT = ROOT_DIR / ".tmp" / "examples-start-logs"
+ARTIFACTS_DIR_DEFAULT = ROOT_DIR / ".tmp" / "examples-artifacts"
 RERUN_FILE_DEFAULT = ROOT_DIR / ".tmp" / "examples-rerun.txt"
 DEFAULT_MAIN_LOG = LOG_DIR_DEFAULT / f"main_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}.log"
 REDIS_SESSION_EXAMPLE = "examples/memory/redis_session_example.py"
@@ -58,8 +59,6 @@ DISCOVERY_EXCLUDE = {
 # Examples that are noisy, require extra credentials, or hang in auto runs.
 DEFAULT_AUTO_SKIP = {
    "examples/agent_patterns/llm_as_a_judge.py",
-    "examples/agent_patterns/routing.py",
-    "examples/customer_service/main.py",
    "examples/hosted_mcp/connectors.py",
    "examples/mcp/git_example/main.py",
    # These are helper daemons or multi-process components exercised by sibling examples.
@@ -416,6 +415,11 @@ def parse_args() -> argparse.Namespace:
        default=str(DEFAULT_MAIN_LOG),
        help="Path to write the main summary log.",
    )
+    parser.add_argument(
+        "--artifacts-dir",
+        default=str(ARTIFACTS_DIR_DEFAULT),
+        help="Directory for example-generated artifacts.",
+    )
    parser.add_argument(
        "--rerun-file",
        help="Only run examples listed in this file (one relative path per line).",
@@ -580,6 +584,12 @@ def ensure_dirs(path: Path, is_file: bool | None = None) -> None:
    target.mkdir(parents=True, exist_ok=True)


+def artifact_dir_for_example(relpath: str, artifacts_dir: Path) -> Path:
+    """Return a deterministic scratch directory for one example run."""
+    stem = normalize_relpath(str(Path(relpath).with_suffix("")))
+    return artifacts_dir / stem.replace("/", "__")
+
+
 def parse_rerun_from_log(log_path: Path) -> list[str]:
    if not log_path.exists():
        raise FileNotFoundError(log_path)
@@ -610,6 +620,7 @@ def run_examples(examples: Sequence[ExampleScript], args: argparse.Namespace) ->
        overrides.add("external")

    logs_dir = Path(args.logs_dir).resolve()
+    artifacts_dir = Path(args.artifacts_dir).resolve()
    main_log_path = Path(args.main_log).resolve()
    auto_mode = args.auto_mode or os.environ.get("EXAMPLES_INTERACTIVE_MODE", "").lower() == "auto"
    auto_skip_set = load_auto_skip()
@@ -618,6 +629,7 @@ def run_examples(examples: Sequence[ExampleScript], args: argparse.Namespace) ->
        overrides.add("interactive")

    ensure_dirs(logs_dir, is_file=False)
+    ensure_dirs(artifacts_dir, is_file=False)
    ensure_dirs(main_log_path, is_file=True)
    rerun_entries: list[str] = []

@@ -659,6 +671,7 @@ def run_examples(examples: Sequence[ExampleScript], args: argparse.Namespace) ->
        env = os.environ.copy()
        env["PATH"] = command_path
        env["PYTHONPATH"] = build_python_path(env.get("PYTHONPATH"))
+        env["EXAMPLES_ARTIFACTS_DIR"] = str(artifact_dir_for_example(relpath, artifacts_dir))
        if auto_mode:
            env["EXAMPLES_INTERACTIVE_MODE"] = "auto"
            env["APPLY_PATCH_AUTO_APPROVE"] = "1"
@@ -759,6 +772,7 @@ def run_examples(examples: Sequence[ExampleScript], args: argparse.Namespace) ->
        safe_write_main(f"# include: {sorted(overrides)}")
        safe_write_main(f"# auto_mode: {auto_mode}")
        safe_write_main(f"# logs_dir: {logs_dir}")
+        safe_write_main(f"# artifacts_dir: {artifacts_dir}")
        safe_write_main(f"# jobs: {jobs}")
        safe_write_main(f"# buffer_output: {buffer_output}")
        safe_write_main(f"# path_augmented: {path_augmented}")
@@ -17,8 +17,11 @@ from __future__ import annotations
 import argparse
 import concurrent.futures
 import csv
+import functools
 import json
+import os
 import sqlite3
+import ssl
 import sys
 import time
 import urllib.error
@@ -27,9 +30,10 @@ import zipfile
 from pathlib import Path
 from typing import Any

-DB_DIR = Path("data")
+ARTIFACT_ROOT = Path(os.environ.get("EXAMPLES_ARTIFACTS_DIR", "."))
+DB_DIR = ARTIFACT_ROOT / "data"
 DB_PATH = DB_DIR / "usaspending.db"
-GLOSSARY_PATH = Path("schema") / "glossary.md"
+GLOSSARY_PATH = ARTIFACT_ROOT / "schema" / "glossary.md"

 USASPENDING_API = "https://api.usaspending.gov"
 BULK_DOWNLOAD_ENDPOINT = f"{USASPENDING_API}/api/v2/bulk_download/awards/"
@@ -118,14 +122,26 @@ CREATE INDEX IF NOT EXISTS idx_spending_awarding_office ON spending(awarding_off
 # ---------------------------------------------------------------------------


+@functools.cache
+def _urlopen_ssl_context() -> ssl.SSLContext | None:
+    """Use certifi's CA bundle when available, otherwise keep stdlib defaults."""
+    try:
+        import certifi
+    except ImportError:
+        return None
+
+    return ssl.create_default_context(cafile=certifi.where())
+
+
 def _urlopen_with_retry(
    req: urllib.request.Request, *, timeout: int = 60, retries: int = 3
 ) -> bytes:
    """urlopen with retries for the flaky USAspending endpoints."""
    last_exc: Exception | None = None
+    ssl_context = _urlopen_ssl_context()
    for attempt in range(1, retries + 1):
        try:
-            with urllib.request.urlopen(req, timeout=timeout) as resp:
+            with urllib.request.urlopen(req, timeout=timeout, context=ssl_context) as resp:
                return bytes(resp.read())
        except (urllib.error.URLError, ConnectionError, OSError) as e:
            last_exc = e
@@ -600,7 +616,7 @@ def main() -> None:
    elif DB_PATH.exists():
        DB_PATH.unlink()

-    tmp_dir = Path("data/tmp_download")
+    tmp_dir = DB_DIR / "tmp_download"

    print("=== NASA USAspending Database Builder ===")
    print(f"Fiscal years: {args.start_fy} - {args.end_fy}\n")
@@ -6,6 +6,7 @@ from __future__ import annotations

 import argparse
 import asyncio
+import os
 import sys
 from pathlib import Path
 from textwrap import dedent
@@ -68,6 +69,14 @@ AGENTS_MD = dedent(
 )


+def default_output_dir() -> Path:
+    """Return the local directory for copied example artifacts."""
+    artifacts_dir = os.environ.get("EXAMPLES_ARTIFACTS_DIR")
+    if artifacts_dir:
+        return Path(artifacts_dir)
+    return DEMO_DIR / "output"
+
+
 def build_manifest() -> Manifest:
    return Manifest(
        entries={
@@ -236,7 +245,7 @@ if __name__ == "__main__":
    parser.add_argument(
        "--output-dir",
        type=Path,
-        default=DEMO_DIR / "output",
+        default=default_output_dir(),
        help="Directory for copied website files.",
    )
    args = parser.parse_args()
@@ -34,6 +34,11 @@ def test_default_auto_skip_keeps_computer_use_example_enabled() -> None:
    assert "examples/tools/computer_use.py" not in run_examples.DEFAULT_AUTO_SKIP


+def test_default_auto_skip_keeps_one_turn_auto_examples_enabled() -> None:
+    assert "examples/agent_patterns/routing.py" not in run_examples.DEFAULT_AUTO_SKIP
+    assert "examples/customer_service/main.py" not in run_examples.DEFAULT_AUTO_SKIP
+
+
 def test_example_command_runs_python_unbuffered(monkeypatch) -> None:
    monkeypatch.delenv("EXAMPLES_UV_EXTRAS", raising=False)
    example = run_examples.ExampleScript(
@@ -63,6 +68,15 @@ def test_example_command_includes_configured_uv_extras(monkeypatch) -> None:
    ]


+def test_artifact_dir_for_example_uses_tmp_safe_stem(tmp_path: Path) -> None:
+    artifact_dir = run_examples.artifact_dir_for_example(
+        "examples/sandbox/tutorials/vision_website_clone/main.py",
+        tmp_path,
+    )
+
+    assert artifact_dir == tmp_path / "examples__sandbox__tutorials__vision_website_clone__main"
+
+
 def test_prepare_redis_for_example_uses_existing_local_redis(monkeypatch) -> None:
    env: dict[str, str] = {}
    monkeypatch.setattr(run_examples, "redis_ping_url", lambda url, timeout=0.5: True)
@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import importlib
+import ssl
+import sys
+import types
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+from examples.sandbox.extensions.daytona.usaspending_text2sql import setup_db
+
+
+def test_paths_use_examples_artifacts_dir_when_set(monkeypatch: Any, tmp_path: Path) -> None:
+    monkeypatch.setenv("EXAMPLES_ARTIFACTS_DIR", str(tmp_path))
+    reloaded = importlib.reload(setup_db)
+
+    try:
+        assert reloaded.DB_PATH == tmp_path / "data" / "usaspending.db"
+        assert reloaded.GLOSSARY_PATH == tmp_path / "schema" / "glossary.md"
+    finally:
+        monkeypatch.delenv("EXAMPLES_ARTIFACTS_DIR", raising=False)
+        importlib.reload(setup_db)
+
+
+def test_urlopen_ssl_context_uses_certifi_when_available(monkeypatch: Any) -> None:
+    setup_db._urlopen_ssl_context.cache_clear()
+    ssl_context = object()
+    certifi = types.SimpleNamespace(where=lambda: "/tmp/certifi.pem")
+    monkeypatch.setitem(sys.modules, "certifi", certifi)
+
+    def fake_create_default_context(*, cafile: str) -> object:
+        assert cafile == "/tmp/certifi.pem"
+        return ssl_context
+
+    monkeypatch.setattr(ssl, "create_default_context", fake_create_default_context)
+
+    try:
+        assert setup_db._urlopen_ssl_context() is ssl_context
+    finally:
+        setup_db._urlopen_ssl_context.cache_clear()
+
+
+def test_urlopen_ssl_context_falls_back_without_certifi(monkeypatch: Any) -> None:
+    setup_db._urlopen_ssl_context.cache_clear()
+    monkeypatch.setitem(sys.modules, "certifi", None)
+
+    def fail_create_default_context(**kwargs: object) -> object:
+        raise AssertionError("stdlib-only fallback should not create a certifi SSL context")
+
+    monkeypatch.setattr(ssl, "create_default_context", fail_create_default_context)
+
+    try:
+        assert setup_db._urlopen_ssl_context() is None
+    finally:
+        setup_db._urlopen_ssl_context.cache_clear()
+
+
+def test_urlopen_with_retry_passes_optional_ssl_context(monkeypatch: Any) -> None:
+    ssl_context = object()
+    captured: dict[str, object] = {}
+
+    class DummyResponse:
+        def __enter__(self) -> DummyResponse:
+            return self
+
+        def __exit__(self, *args: object) -> None:
+            return None
+
+        def read(self) -> bytes:
+            return b"ok"
+
+    def fake_urlopen(
+        req: urllib.request.Request, *, timeout: int, context: object | None
+    ) -> DummyResponse:
+        captured["req"] = req
+        captured["timeout"] = timeout
+        captured["context"] = context
+        return DummyResponse()
+
+    monkeypatch.setattr(setup_db, "_urlopen_ssl_context", lambda: ssl_context)
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+
+    req = urllib.request.Request("https://api.usaspending.gov")
+
+    assert setup_db._urlopen_with_retry(req, timeout=12, retries=1) == b"ok"
+    assert captured == {"req": req, "timeout": 12, "context": ssl_context}