fix tests

.github/workflows/tests.yml

@@ -57,10 +57,13 @@ jobs:
         uses: pre-commit/action@v3.0.1
       - name: Check Python headers
         run: uv run --locked --no-sync scripts/check_headers.py
+        if: matrix.setup == 'fast'
       - name: Run Black
         run: uv run --locked --no-sync black --check .
+        if: matrix.setup != 'next'
       - name: Run isort
         run: uv run --locked --no-sync isort --check-only .
+        if: matrix.setup != 'next'
       - name: Run pyright (fast)
         run: uv run --locked --no-sync pyright -p pyrightconfig.fast.json
         if: matrix.setup == 'fast'

agentlightning/emitter/annotation.py

@@ -246,7 +246,11 @@ class OperationContext:
             flattened = flatten_attributes({LightningSpanAttributes.OPERATION_OUTPUT.value: result})
             recording_ctx.record_attributes(sanitize_attributes(flattened))
 
-        if asyncio.iscoroutinefunction(fn) or inspect.iscoroutinefunction(fn):
+        if inspect.iscoroutinefunction(fn) or (
+            # For backwards compatibility.
+            hasattr(asyncio, "iscoroutinefunction")
+            and asyncio.iscoroutinefunction(fn)  # type: ignore
+        ):
 
             @functools.wraps(fn)
             async def async_wrapper(*args: Any, **kwargs: Any) -> Any:

agentlightning/emitter/reward.py

@@ -92,7 +92,11 @@ def reward(fn: _FnType) -> _FnType:
         return {"type": "reward", "value": float(result)}
 
     # Check if the function is async
-    is_async = asyncio.iscoroutinefunction(fn) or inspect.iscoroutinefunction(fn)
+    is_async = inspect.iscoroutinefunction(fn) or (
+        # For backwards compatibility.
+        hasattr(asyncio, "iscoroutinefunction")
+        and asyncio.iscoroutinefunction(fn)  # type: ignore
+    )
 
     if is_async:
 

agentlightning/instrumentation/weave.py

@@ -424,6 +424,29 @@ class InMemoryWeaveTraceServer(TraceServerClientInterface):
     def score_delete(self, req: tsi.ScoreDeleteReq) -> tsi.ScoreDeleteRes:
         return tsi.ScoreDeleteRes(num_deleted=0)
 
+    # Experimental unstable APIs
+    # We don't support these APIs yet.
+    def annotation_queue_create(self, *args: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError()
+
+    def annotation_queues_query_stream(self, *args: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError()
+
+    def annotation_queue_read(self, *args: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError()
+
+    def annotation_queue_add_calls(self, *args: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError()
+
+    def annotation_queues_stats(self, *args: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError()
+
+    def annotation_queue_items_query(self, *args: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError()
+
+    def annotator_queue_items_progress_update(self, *args: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError()
+
 
 # Module-level storage for originals
 _original_init_weave_get_server: Callable[..., Any] | None = None

agentlightning/store/collection/mongo.py

@@ -663,6 +663,9 @@ class MongoBasedCollection(Collection[T_model]):
                     return_document=ReturnDocument.AFTER,
                 )
 
+            if result_doc is None:  # pyright: ignore[reportUnnecessaryComparison]
+                raise RuntimeError(f"Upsert resulted in no document for filter: {pk_filter}")
+
             # Because upsert=True, result_doc is guaranteed to be not None
             new_item = self._model_validate_item(result_doc)
             upserted_items.append(new_item)

scripts/cleanup_aoai.py

@@ -0,0 +1,87 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import os
+
+import requests
+from openai import OpenAI
+
+# Most common Azure OpenAI setup:
+#   AZURE_OPENAI_ENDPOINT="https://<resource>.openai.azure.com"
+#   AZURE_OPENAI_API_KEY="..."
+# Optional (only if your endpoint requires it):
+#   AZURE_OPENAI_API_VERSION="2025-xx-xx"
+#
+# This script treats "delete finetune job" as "cancel finetune job"
+# because fine-tune jobs are typically cancellable, not deletable.
+
+
+def _client() -> OpenAI:
+    # This script assumes AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY are set in the environment.
+    endpoint = os.environ["AZURE_OPENAI_ENDPOINT"]
+    api_key = os.environ["AZURE_OPENAI_API_KEY"]
+    return OpenAI(api_key=api_key, base_url=endpoint)
+
+
+def list_data_files():
+    c = _client()
+    return c.files.list(limit=100)
+
+
+def list_finetune_jobs():
+    c = _client()
+    return c.fine_tuning.jobs.list(limit=100)
+
+
+def delete_data_file(file_id: str):
+    c = _client()
+    return c.files.delete(file_id)
+
+
+def cancel_finetune_job(job_id: str):
+    c = _client()
+    return c.fine_tuning.jobs.cancel(job_id)
+
+
+def delete_finetune_job(job_id: str):
+    # This script assumes AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY are set in the environment.
+    endpoint = os.environ["AZURE_OPENAI_ENDPOINT"].rstrip("/")
+    api_key = os.environ["AZURE_OPENAI_API_KEY"]
+    root = endpoint.split("/openai")[0]
+
+    url = f"{root}/openai/fine_tuning/jobs/{job_id}"
+    params = {"api-version": os.environ["AZURE_OPENAI_API_VERSION"]}
+
+    resp = requests.delete(url, headers={"api-key": api_key}, params=params, timeout=60)
+    resp.raise_for_status()
+    return resp.content
+
+
+if __name__ == "__main__":
+    # Quick demo: print IDs you could delete
+    jobs = list_finetune_jobs().data
+    files = list_data_files().data
+
+    print("JOBS:")
+    for j in jobs:
+        print(f"  {j.id}  {getattr(j, 'status', '')}  {getattr(j, 'model', '')}")
+
+    print("\nFILES:")
+    for f in files:
+        print(f"  {f.id}  {getattr(f, 'filename', '')}  {getattr(f, 'status', '')}")
+
+    # Delete them all WITHOUT CONFIRMATION!
+    for j in jobs:
+        print(f"Deleting job {j.id}")
+        try:
+            if j.status == "running":
+                cancel_finetune_job(j.id)
+            delete_finetune_job(j.id)
+        except Exception as exc:
+            print(f"  Error deleting job {j.id}: {exc}")
+
+    for f in files:
+        print(f"Deleting file {f.id}")
+        try:
+            delete_data_file(f.id)
+        except Exception as exc:
+            print(f"  Error deleting file {f.id}: {exc}")

tests/assets/prompt_caches.jsonl

@@ -30,3 +30,18 @@
 {"request": {"messages": [{"content": "Answer history questions.", "role": "system"}, {"role": "assistant", "content": "For context, here is the conversation so far between the user and the previous agent:\n<CONVERSATION HISTORY>\n1. user: Who was the first president of the US?\n2. function_call: {\"arguments\": \"{}\", \"call_id\": \"call_tXXmrsNYcFlKrG9hMo2Yk0E3\", \"name\": \"transfer_to_historyagent\", \"id\": \"__fake_id__\"}\n3. function_call_output: {\"call_id\": \"call_tXXmrsNYcFlKrG9hMo2Yk0E3\", \"output\": \"{\\\"assistant\\\": \\\"HistoryAgent\\\"}\"}\n</CONVERSATION HISTORY>"}, {"role": "assistant", "tool_calls": [{"id": "call_tXXmrsNYcFlKrG9hMo2Yk0E3", "type": "function", "function": {"name": "transfer_to_historyagent", "arguments": "{}"}}]}, {"role": "tool", "tool_call_id": "call_tXXmrsNYcFlKrG9hMo2Yk0E3", "content": "{\"assistant\": \"HistoryAgent\"}"}], "model": "gpt-4.1-mini"}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "The first president of the United States was George Washington. He served as president from 1789 to 1797. If you have more questions about U.S. history or any other historical events, feel free to ask!", "refusal": null, "role": "assistant"}}], "created": 1763565707, "id": "chatcmpl-CdeI78NuMQmjmeuwLhx3I0Qqcem99", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 46, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 179, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 225}}}
 {"request": {"messages": [{"role": "system", "content": "You are a helpful assistant. Use the multiply tool to answer math questions."}, {"role": "user", "content": "what is 42 * 12"}], "model": "gpt-4.1-mini", "tools": [{"type": "function", "function": {"name": "multiply", "description": "A simple calculator tool that multiplies two integers.", "parameters": {"type": "object", "properties": {"a_and_b": {"type": "string"}}, "required": ["a_and_b"]}}}]}, "response": {"id": "chatcmpl-test1", "model": "gpt-4.1-mini", "object": "chat.completion", "created": 1750758616, "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_abc123", "type": "function", "function": {"name": "multiply", "arguments": "{\"a_and_b\": \"42 and 12\"}"}}]}, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}}}
 {"request": {"messages": [{"role": "system", "content": "You are a helpful assistant. Use the multiply tool to answer math questions."}, {"role": "user", "content": "what is 42 * 12"}, {"role": "assistant", "content": null, "tool_calls": [{"id": "call_abc123", "type": "function", "function": {"name": "multiply", "arguments": "{\"a_and_b\": \"42 and 12\"}"}}]}, {"role": "tool", "content": "504", "tool_call_id": "call_abc123"}], "model": "gpt-4.1-mini", "tools": [{"type": "function", "function": {"name": "multiply", "description": "A simple calculator tool that multiplies two integers.", "parameters": {"type": "object", "properties": {"a_and_b": {"type": "string"}}, "required": ["a_and_b"]}}}]}, "response": {"id": "chatcmpl-test2", "model": "gpt-4.1-mini", "object": "chat.completion", "created": 1750758617, "choices": [{"index": 0, "message": {"role": "assistant", "content": "The result of 42 * 12 is 504."}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 150, "completion_tokens": 15, "total_tokens": 165}}}
+{"request": {"messages": [{"content": "You are a helpful AI assistant.", "role": "system"}, {"role": "user", "name": "user", "content": "Write a short poem about the fall season."}], "model": "gpt-4.1-mini", "stream": false}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "Leaves of amber, gold, and rust,  \nWhisper secrets in the gust.  \nCrisp air carries autumn\u2019s song,  \nA cozy time where hearts belong.", "refusal": null, "role": "assistant"}}], "created": 1768794783, "id": "chatcmpl-Czac3Je2mDt0SETQ5HBMEGBtUneBF", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 36, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 28, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 64}}}
+{"request": {"messages": [{"content": "Provide constructive feedback. Respond with 'APPROVE' to when your feedbacks are addressed.", "role": "system"}, {"role": "user", "name": "user", "content": "Write a short poem about the fall season."}, {"role": "user", "name": "primary", "content": "Leaves of amber, gold, and rust,  \nWhisper secrets in the gust.  \nCrisp air carries autumn\u2019s song,  \nA cozy time where hearts belong."}], "model": "gpt-4.1-mini", "stream": false}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "Your poem beautifully captures the essence of fall with vivid imagery and a warm tone. The colors \"amber, gold, and rust\" effectively evoke the visual splendor of autumn leaves, and the personification in \"Whisper secrets in the gust\" adds a gentle, mysterious quality to the scene. The contrast between the crisp air and the cozy feeling creates a nice balance, highlighting both the physical and emotional aspects of the season.\n\nTo enhance the poem further, consider adding a subtle sensory detail, such as the smell of pine or the sound of crunching leaves, to engage more senses. Also, you might experiment with varying the rhythm slightly to create a more dynamic flow.\n\nOverall, it's a lovely and evocative short poem that encapsulates the spirit of fall well.", "refusal": null, "role": "assistant"}}], "created": 1768794784, "id": "chatcmpl-Czac4vKGrCz5PGPHcSMsDaosa0MzH", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 154, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 81, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 235}}}
+{"request": {"messages": [{"content": "You are a helpful AI assistant.", "role": "system"}, {"role": "user", "name": "user", "content": "Write a short poem about the fall season."}, {"role": "assistant", "content": "Leaves of amber, gold, and rust,  \nWhisper secrets in the gust.  \nCrisp air carries autumn\u2019s song,  \nA cozy time where hearts belong."}, {"role": "user", "name": "critic", "content": "Your poem beautifully captures the essence of fall with vivid imagery and a warm tone. The colors \"amber, gold, and rust\" effectively evoke the visual splendor of autumn leaves, and the personification in \"Whisper secrets in the gust\" adds a gentle, mysterious quality to the scene. The contrast between the crisp air and the cozy feeling creates a nice balance, highlighting both the physical and emotional aspects of the season.\n\nTo enhance the poem further, consider adding a subtle sensory detail, such as the smell of pine or the sound of crunching leaves, to engage more senses. Also, you might experiment with varying the rhythm slightly to create a more dynamic flow.\n\nOverall, it's a lovely and evocative short poem that encapsulates the spirit of fall well."}], "model": "gpt-4.1-mini", "stream": false}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "Thank you for the thoughtful feedback! Here's a revised version of the poem incorporating subtle sensory details and a more varied rhythm:\n\nLeaves of amber, gold, and rust,  \nWhisper secrets with a gust.  \nCrunch beneath each careful tread,  \nAutumn\u2019s scent of pine and bread.  \n\nCrisp air hums a mellow song,  \nWhere hearts find warmth and belong.  \n\nI hope this version brings the fall season to life even more vividly!", "refusal": null, "role": "assistant"}}], "created": 1768794787, "id": "chatcmpl-Czac7CtCUcyZP6INVVsAWUIGXT595", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 93, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 226, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 319}}}
+{"request": {"messages": [{"content": "Provide constructive feedback. Respond with 'APPROVE' to when your feedbacks are addressed.", "role": "system"}, {"role": "user", "name": "user", "content": "Write a short poem about the fall season."}, {"role": "user", "name": "primary", "content": "Leaves of amber, gold, and rust,  \nWhisper secrets in the gust.  \nCrisp air carries autumn\u2019s song,  \nA cozy time where hearts belong."}, {"role": "assistant", "content": "Your poem beautifully captures the essence of fall with vivid imagery and a warm tone. The colors \"amber, gold, and rust\" effectively evoke the visual splendor of autumn leaves, and the personification in \"Whisper secrets in the gust\" adds a gentle, mysterious quality to the scene. The contrast between the crisp air and the cozy feeling creates a nice balance, highlighting both the physical and emotional aspects of the season.\n\nTo enhance the poem further, consider adding a subtle sensory detail, such as the smell of pine or the sound of crunching leaves, to engage more senses. Also, you might experiment with varying the rhythm slightly to create a more dynamic flow.\n\nOverall, it's a lovely and evocative short poem that encapsulates the spirit of fall well."}, {"role": "user", "name": "primary", "content": "Thank you for the thoughtful feedback! Here's a revised version of the poem incorporating subtle sensory details and a more varied rhythm:\n\nLeaves of amber, gold, and rust,  \nWhisper secrets with a gust.  \nCrunch beneath each careful tread,  \nAutumn\u2019s scent of pine and bread.  \n\nCrisp air hums a mellow song,  \nWhere hearts find warmth and belong.  \n\nI hope this version brings the fall season to life even more vividly!"}], "model": "gpt-4.1-mini", "stream": false}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "Your revised poem wonderfully enhances the sensory experience of the fall season. The addition of \"Crunch beneath each careful tread\" gives a tactile element that brings the reader into the scene, and \"Autumn\u2019s scent of pine and bread\" beautifully evokes familiar and comforting smells of the season. The rhythm feels more varied and natural, which adds a musical quality to the poem.\n\nThe balance between imagery, sound, and scent creates a rich, multi-sensory portrayal of fall that invites the reader to fully immerse themselves in the season. Well done on capturing both the external environment and the internal warmth of the season.\n\nOverall, your revision successfully deepens the imagery and emotional resonance of the poem while maintaining its concise charm. Great work!", "refusal": null, "role": "assistant"}}], "created": 1768794789, "id": "chatcmpl-Czac9gwa8SJ9D9ICAILeBqkAUmx4s", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 146, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 336, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 482}}}
+{"request": {"messages": [{"content": "You are a helpful AI assistant. Solve tasks using your tools. Reply with TERMINATE when the task has been completed.", "role": "system"}, {"role": "user", "name": "user", "content": "What is 42 * 12?"}], "model": "gpt-4.1-mini", "stream": false, "tool_choice": "auto", "tools": [{"type": "function", "function": {"name": "calculate", "description": "Calculates/evaluates the given expression.", "parameters": {"type": "object", "properties": {"expression": {"title": "Expression", "type": "string"}}, "required": ["expression"], "additionalProperties": false}, "strict": false}}]}, "response": {"choices": [{"content_filter_results": {}, "finish_reason": "tool_calls", "index": 0, "logprobs": null, "message": {"annotations": [], "content": null, "refusal": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"expression\":\"42 * 12\"}", "name": "calculate"}, "id": "call_WC4BHTV1fxcnH4bHY1utAT1x", "type": "function"}]}}], "created": 1768794793, "id": "chatcmpl-CzacDCQNDY4aqKOI6oMIC7zb3l7dg", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 17, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 80, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 97}}}
+{"request": {"messages": [{"content": "Answer questions. If it's about homework, say so.", "role": "system"}, {"role": "user", "content": "The teacher asks to answer whether hummingbirds are mammals."}], "model": "gpt-4.1-mini"}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "Hummingbirds are not mammals. They are birds. Mammals are warm-blooded animals that usually have hair or fur and produce milk to feed their young, while hummingbirds have feathers and lay eggs.", "refusal": null, "role": "assistant"}}], "created": 1768794795, "id": "chatcmpl-CzacFKEoI0J1AzNXT1g1Y7Ttq1Xr7", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 43, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 33, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 76}}}
+{"request": {"messages": [{"content": "Check if the user is asking about homework.", "role": "system"}, {"role": "user", "content": "The teacher asks to answer whether hummingbirds are mammals."}], "model": "gpt-4.1-mini", "response_format": {"type": "json_schema", "json_schema": {"name": "final_output", "strict": true, "schema": {"properties": {"is_homework": {"title": "Is Homework", "type": "boolean"}, "reasoning": {"title": "Reasoning", "type": "string"}}, "required": ["is_homework", "reasoning"], "title": "HomeworkOutput", "type": "object", "additionalProperties": false}}}}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "{\"is_homework\":true,\"reasoning\":\"The user mentions that a teacher has asked them to answer a question, which implies it is a homework or classroom assignment.\"}", "refusal": null, "role": "assistant"}}], "created": 1768794795, "id": "chatcmpl-CzacFzUXuV8n4yNp0jCOGjmNDkfFr", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 36, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 82, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 118}}}
+{"request": {"messages": [{"content": "Use the tools to answer the question.", "role": "system"}, {"role": "user", "content": "What is 43*57?"}], "model": "gpt-4.1-mini", "tools": [{"type": "function", "function": {"name": "calculate", "description": "Calculates/evaluates the given expression.", "parameters": {"properties": {"expression": {"title": "Expression", "type": "string"}}, "required": ["expression"], "title": "calculateArguments", "type": "object"}, "strict": false}}]}, "response": {"choices": [{"content_filter_results": {}, "finish_reason": "tool_calls", "index": 0, "logprobs": null, "message": {"annotations": [], "content": null, "refusal": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"expression\":\"43*57\"}", "name": "calculate"}, "id": "call_S7lqEahilFLWrYq2EFdGu5QL", "type": "function"}]}}], "created": 1768794798, "id": "chatcmpl-CzacIDIe0z8HKWuahOf5TGsNhZ0Tp", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 16, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 66, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 82}}}
+{"request": {"messages": [{"content": "Use the tools to answer the question.", "role": "system"}, {"role": "user", "content": "What is 43*57?"}, {"role": "assistant", "content": null, "tool_calls": [{"id": "call_S7lqEahilFLWrYq2EFdGu5QL", "type": "function", "function": {"name": "calculate", "arguments": "{\"expression\":\"43*57\"}"}}]}, {"role": "tool", "tool_call_id": "call_S7lqEahilFLWrYq2EFdGu5QL", "content": "{\"type\":\"text\",\"text\":\"2451\",\"annotations\":null,\"meta\":null}"}], "model": "gpt-4.1-mini", "tools": [{"type": "function", "function": {"name": "calculate", "description": "Calculates/evaluates the given expression.", "parameters": {"properties": {"expression": {"title": "Expression", "type": "string"}}, "required": ["expression"], "title": "calculateArguments", "type": "object"}, "strict": false}}]}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "43 multiplied by 57 equals 2451.", "refusal": null, "role": "assistant"}}], "created": 1768794799, "id": "chatcmpl-CzacJ1SPy3wtv2cYrCZ4OFV39cj6y", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 12, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 106, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 118}}}
+{"request": {"messages": [{"content": "If the question is about math, handoff to MathAgent. Otherwise, handoff to HistoryAgent.", "role": "system"}, {"role": "user", "content": "What is 3+5?"}], "model": "gpt-4.1-mini", "tools": [{"type": "function", "function": {"name": "transfer_to_mathagent", "description": "Handoff to the MathAgent agent to handle the request. ", "parameters": {"additionalProperties": false, "type": "object", "properties": {}, "required": []}, "strict": true}}, {"type": "function", "function": {"name": "transfer_to_historyagent", "description": "Handoff to the HistoryAgent agent to handle the request. ", "parameters": {"additionalProperties": false, "type": "object", "properties": {}, "required": []}, "strict": true}}]}, "response": {"choices": [{"content_filter_results": {}, "finish_reason": "tool_calls", "index": 0, "logprobs": null, "message": {"annotations": [], "content": null, "refusal": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{}", "name": "transfer_to_mathagent"}, "id": "call_DJBnzm9zWWaSHrFx0xtuRDOG", "type": "function"}]}}], "created": 1768794802, "id": "chatcmpl-CzacMLtgEFyw6Vl256NbMfJy2Z5bM", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 13, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 95, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 108}}}
+{"request": {"messages": [{"content": "Add two numbers.", "role": "system"}, {"role": "assistant", "content": "For context, here is the conversation so far between the user and the previous agent:\n<CONVERSATION HISTORY>\n1. user: What is 3+5?\n2. function_call: {\"arguments\": \"{}\", \"call_id\": \"call_DJBnzm9zWWaSHrFx0xtuRDOG\", \"name\": \"transfer_to_mathagent\", \"id\": \"__fake_id__\"}\n3. function_call_output: {\"call_id\": \"call_DJBnzm9zWWaSHrFx0xtuRDOG\", \"output\": \"{\\\"assistant\\\": \\\"MathAgent\\\"}\"}\n</CONVERSATION HISTORY>"}], "model": "gpt-4.1-mini", "response_format": {"type": "json_schema", "json_schema": {"name": "final_output", "strict": true, "schema": {"properties": {"answer": {"title": "Answer", "type": "integer"}}, "required": ["answer"], "title": "MathOutput", "type": "object", "additionalProperties": false}}}, "tools": [{"type": "function", "function": {"name": "add", "description": "", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}}, "required": ["a", "b"], "title": "add_args", "type": "object", "additionalProperties": false}, "strict": true}}]}, "response": {"choices": [{"content_filter_results": {}, "finish_reason": "tool_calls", "index": 0, "logprobs": null, "message": {"annotations": [], "content": null, "refusal": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"a\":3,\"b\":5}", "name": "add"}, "id": "call_iuVHkklvC5fheqGyoguujRV7", "type": "function"}]}}], "created": 1768794803, "id": "chatcmpl-CzacNWeeK8ScJCt6VqXoW4gnwFRsO", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 18, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 220, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 238}}}
+{"request": {"messages": [{"content": "Add two numbers.", "role": "system"}, {"role": "assistant", "content": "For context, here is the conversation so far between the user and the previous agent:\n<CONVERSATION HISTORY>\n1. user: What is 3+5?\n2. function_call: {\"arguments\": \"{}\", \"call_id\": \"call_DJBnzm9zWWaSHrFx0xtuRDOG\", \"name\": \"transfer_to_mathagent\", \"id\": \"__fake_id__\"}\n3. function_call_output: {\"call_id\": \"call_DJBnzm9zWWaSHrFx0xtuRDOG\", \"output\": \"{\\\"assistant\\\": \\\"MathAgent\\\"}\"}\n</CONVERSATION HISTORY>"}, {"role": "assistant", "content": null, "tool_calls": [{"id": "call_iuVHkklvC5fheqGyoguujRV7", "type": "function", "function": {"name": "add", "arguments": "{\"a\":3,\"b\":5}"}}]}, {"role": "tool", "tool_call_id": "call_iuVHkklvC5fheqGyoguujRV7", "content": "8"}], "model": "gpt-4.1-mini", "response_format": {"type": "json_schema", "json_schema": {"name": "final_output", "strict": true, "schema": {"properties": {"answer": {"title": "Answer", "type": "integer"}}, "required": ["answer"], "title": "MathOutput", "type": "object", "additionalProperties": false}}}, "tools": [{"type": "function", "function": {"name": "add", "description": "", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}}, "required": ["a", "b"], "title": "add_args", "type": "object", "additionalProperties": false}, "strict": true}}]}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "{\"answer\":8}", "refusal": null, "role": "assistant"}}], "created": 1768794805, "id": "chatcmpl-CzacPibsAYWUTrWPS9oHfuYKPb88x", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 11, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 245, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 256}}}
+{"request": {"messages": [{"content": "Return 1.0 if the answer is 8, else 0.0.", "role": "system"}, {"role": "user", "content": "8"}], "model": "gpt-4.1-mini", "response_format": {"type": "json_schema", "json_schema": {"name": "final_output", "strict": true, "schema": {"properties": {"response": {"title": "Response", "type": "number"}}, "required": ["response"], "title": "OutputType", "type": "object", "additionalProperties": false}}}}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "{\"response\":1.0}", "refusal": null, "role": "assistant"}}], "created": 1768794806, "id": "chatcmpl-CzacQt9ezHNd9gmiQ6FD4ACESwfsH", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 8, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 65, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 73}}}
+{"request": {"messages": [{"content": "If the question is about math, handoff to MathAgent. Otherwise, handoff to HistoryAgent.", "role": "system"}, {"role": "user", "content": "Who was the first president of the US?"}], "model": "gpt-4.1-mini", "tools": [{"type": "function", "function": {"name": "transfer_to_mathagent", "description": "Handoff to the MathAgent agent to handle the request. ", "parameters": {"additionalProperties": false, "type": "object", "properties": {}, "required": []}, "strict": true}}, {"type": "function", "function": {"name": "transfer_to_historyagent", "description": "Handoff to the HistoryAgent agent to handle the request. ", "parameters": {"additionalProperties": false, "type": "object", "properties": {}, "required": []}, "strict": true}}]}, "response": {"choices": [{"content_filter_results": {}, "finish_reason": "tool_calls", "index": 0, "logprobs": null, "message": {"annotations": [], "content": null, "refusal": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{}", "name": "transfer_to_historyagent"}, "id": "call_MOjhi5CuL8Tm8Kxd3fbRMNo2", "type": "function"}]}}], "created": 1768794807, "id": "chatcmpl-CzacRL4lFqbo9B1zfzGcxjOmHSEzA", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "jailbreak": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 13, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 97, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 110}}}
+{"request": {"messages": [{"content": "Answer history questions.", "role": "system"}, {"role": "assistant", "content": "For context, here is the conversation so far between the user and the previous agent:\n<CONVERSATION HISTORY>\n1. user: Who was the first president of the US?\n2. function_call: {\"arguments\": \"{}\", \"call_id\": \"call_MOjhi5CuL8Tm8Kxd3fbRMNo2\", \"name\": \"transfer_to_historyagent\", \"id\": \"__fake_id__\"}\n3. function_call_output: {\"call_id\": \"call_MOjhi5CuL8Tm8Kxd3fbRMNo2\", \"output\": \"{\\\"assistant\\\": \\\"HistoryAgent\\\"}\"}\n</CONVERSATION HISTORY>"}], "model": "gpt-4.1-mini"}, "response": {"choices": [{"content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "protected_material_code": {"filtered": false, "detected": false}, "protected_material_text": {"filtered": false, "detected": false}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}, "finish_reason": "stop", "index": 0, "logprobs": null, "message": {"annotations": [], "content": "The first president of the United States was George Washington. He served as president from 1789 to 1797. If you have any more questions about U.S. history or presidents, feel free to ask!", "refusal": null, "role": "assistant"}}], "created": 1768794809, "id": "chatcmpl-CzacTV4NFaqq7kWCqD2FiM6TxWRvT", "model": "gpt-4.1-mini-2025-04-14", "object": "chat.completion", "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {}}], "system_fingerprint": "fp_3dcd5944f5", "usage": {"completion_tokens": 44, "completion_tokens_details": {"accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0}, "prompt_tokens": 148, "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, "total_tokens": 192}}}