From fdebd9d5635abd4e7aa607ac42292252128f0db3 Mon Sep 17 00:00:00 2001 From: Haran Rajkumar Date: Thu, 23 Jul 2026 00:59:38 -0700 Subject: [PATCH] feat: Add basic_criteria ADK eval sample Add contributing/samples/evaluation/basic_criteria/, demonstrating the deterministic built-in metrics tool_trajectory_avg_score and response_match_score against the shared home-automation agent, run via `adk eval`. Co-authored-by: Haran Rajkumar PiperOrigin-RevId: 952586163 --- contributing/samples/evaluation/README.md | 5 +- .../evaluation/basic_criteria/README.md | 78 +++++++++++++++++++ .../basic_criteria/eval_config.json | 9 +++ .../home_automation.evalset.json | 62 +++++++++++++++ 4 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 contributing/samples/evaluation/basic_criteria/README.md create mode 100644 contributing/samples/evaluation/basic_criteria/eval_config.json create mode 100644 contributing/samples/evaluation/basic_criteria/home_automation.evalset.json diff --git a/contributing/samples/evaluation/README.md b/contributing/samples/evaluation/README.md index ae8d2c42..8812f8a0 100644 --- a/contributing/samples/evaluation/README.md +++ b/contributing/samples/evaluation/README.md @@ -37,8 +37,9 @@ answer. ## Samples -| Sample | Concept | Criteria | -| ------ | ------- | -------- | +| Sample | Concept | Criteria | +| ------------------------------------- | -------------------------------------- | --------------------------------------------------- | +| [`basic_criteria`](./basic_criteria/) | Deterministic, reference-based scoring | `tool_trajectory_avg_score`, `response_match_score` | ## Graph diff --git a/contributing/samples/evaluation/basic_criteria/README.md b/contributing/samples/evaluation/basic_criteria/README.md new file mode 100644 index 00000000..78314175 --- /dev/null +++ b/contributing/samples/evaluation/basic_criteria/README.md @@ -0,0 +1,78 @@ +# Basic evaluation criteria + +## Overview + +Evaluates the shared home-automation agent with the two deterministic, +reference-based criteria: + +- `tool_trajectory_avg_score`: does the agent call the right tools with the + right args? Each expected tool call (name + args) is compared against what the + agent actually did. +- `response_match_score`: ROUGE-1 word overlap between the agent's final + response and a reference answer. + +Both criteria are computed locally with no judge model, so this sample needs only +a model credential for the agent's own inference (a Gemini API key or Vertex). + +## Sample Inputs + +The eval set (`home_automation.evalset.json`) contains two single-turn cases: + +- `Turn off device_2.` +- `What is the temperature in the Living Room?` + +## How To + +Run the sample from the workspace root: + +```bash +adk eval contributing/samples/evaluation/home_automation_agent \ + contributing/samples/evaluation/basic_criteria/home_automation.evalset.json \ + --config_file_path contributing/samples/evaluation/basic_criteria/eval_config.json \ + --print_detailed_results +``` + +`adk eval` takes the agent folder and the eval-set file as two separate +arguments, so this folder holds only eval data (`home_automation.evalset.json`), +the criteria config (`eval_config.json`), and this README, with no agent code. + +### `match_type` for tool trajectory + +`tool_trajectory_avg_score` has a `match_type` (set to `EXACT` here in +`eval_config.json`) that controls how the expected and actual tool calls are +compared: + +- `EXACT`: the actual tool calls must match the expected calls one-for-one, in + the same order, with identical args. Use this when the trajectory is fully + deterministic (as in this sample). +- `IN_ORDER`: the expected calls must appear in the given order, but extra + actual calls in between are tolerated. Useful when the agent may take + additional, harmless steps. +- `ANY_ORDER`: the expected calls must all appear, but order does not matter. + Useful when the agent may reorder independent tool calls. + +The `threshold` is `1.0`, so every expected call must match for the case to pass. + +### Why `response_match_score` uses a `0.6` threshold + +`adk eval` runs live inference, so the exact wording of the agent's final +response varies from run to run (for example, "I have turned off device_2." vs +"device_2 has been switched off."). `response_match_score` is a ROUGE-1 score, +which measures word overlap rather than exact-string equality, so it tolerates +this phrasing variation. The `0.6` threshold requires the response to share most +of its wording with the reference while still allowing some rewording. Raise it +toward `1.0` for stricter wording, lower it to tolerate more paraphrasing. + +### Expectations captured from a real run + +The expected `tool_uses` (tool names and args) in `home_automation.evalset.json` +were captured from an actual `adk eval` run of the agent: run with +`--print_detailed_results`, read the printed Actual-vs-Expected, then set the +expected values to match what the agent really produced. The reference +`final_response` for each case is an independently written natural answer (not a +copy of the model output), which is exactly what ROUGE-1 is designed to tolerate. + +## Related Guides + +- Evaluation overview: https://adk.dev/evaluate/ +- Evaluation criteria reference: https://adk.dev/evaluate/criteria/ diff --git a/contributing/samples/evaluation/basic_criteria/eval_config.json b/contributing/samples/evaluation/basic_criteria/eval_config.json new file mode 100644 index 00000000..ca1e88fc --- /dev/null +++ b/contributing/samples/evaluation/basic_criteria/eval_config.json @@ -0,0 +1,9 @@ +{ + "criteria": { + "tool_trajectory_avg_score": { + "threshold": 1.0, + "match_type": "EXACT" + }, + "response_match_score": 0.6 + } +} diff --git a/contributing/samples/evaluation/basic_criteria/home_automation.evalset.json b/contributing/samples/evaluation/basic_criteria/home_automation.evalset.json new file mode 100644 index 00000000..0db1da22 --- /dev/null +++ b/contributing/samples/evaluation/basic_criteria/home_automation.evalset.json @@ -0,0 +1,62 @@ +{ + "eval_set_id": "basic_criteria", + "name": "Basic deterministic criteria", + "description": "Single-turn cases scored by tool trajectory and ROUGE-1.", + "eval_cases": [ + { + "eval_id": "turn_off_bedroom_device", + "conversation": [ + { + "invocation_id": "basic-1", + "user_content": { + "parts": [{"text": "Turn off device_2."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "device_2 is now turned off."}], + "role": "model" + }, + "intermediate_data": { + "tool_uses": [ + {"name": "set_device_info", + "args": {"device_id": "device_2", "status": "OFF"}} + ], + "intermediate_responses": [] + } + } + ], + "session_input": { + "app_name": "home_automation_agent", + "user_id": "user", + "state": {} + } + }, + { + "eval_id": "get_living_room_temperature", + "conversation": [ + { + "invocation_id": "basic-2", + "user_content": { + "parts": [{"text": "What is the temperature in the Living Room?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "The Living Room is 22 degrees Celsius."}], + "role": "model" + }, + "intermediate_data": { + "tool_uses": [ + {"name": "get_temperature", "args": {"location": "Living Room"}} + ], + "intermediate_responses": [] + } + } + ], + "session_input": { + "app_name": "home_automation_agent", + "user_id": "user", + "state": {} + } + } + ] +}