Haran Rajkumar
481fe21cff
Add contributing/samples/evaluation/user_simulation/, demonstrating hallucinations_v1 and per_turn_user_simulator_quality_v1 with a dynamically simulated user, run via `adk eval` over the shared home-automation agent. Co-authored-by: Haran Rajkumar <haranrk@google.com> PiperOrigin-RevId: 952613097
19 lines
480 B
JSON
19 lines
480 B
JSON
{
|
|
"criteria": {
|
|
"hallucinations_v1": {
|
|
"threshold": 0.8,
|
|
"judge_model_options": {"judge_model": "gemini-2.5-flash"},
|
|
"evaluate_intermediate_nl_responses": true
|
|
},
|
|
"per_turn_user_simulator_quality_v1": {
|
|
"threshold": 0.8,
|
|
"judge_model_options": {"judge_model": "gemini-2.5-flash", "num_samples": 5}
|
|
}
|
|
},
|
|
"user_simulator_config": {
|
|
"type": "llm_backed",
|
|
"model": "gemini-2.5-flash",
|
|
"max_allowed_invocations": 8
|
|
}
|
|
}
|