From e84b5a07c150dbe4122afb0fa171d441d4ddd220 Mon Sep 17 00:00:00 2001 From: CTW_CTWalk Date: Tue, 4 Aug 2026 08:23:08 +0800 Subject: [PATCH] Python: fix LocalEvaluator reporting zero-check items as passed (#7399) LocalEvaluator.evaluate initialized item_passed to True and only ever cleared it inside the loop over check results. With no checks configured the loop never runs, so an item with zero scores was recorded as passed: result_counts reported one pass, all_passed was True, and raise_for_status() did not raise. Initialize item_passed from bool(check_results) so an item with no evaluated checks fails closed. This matches the .NET contract in this repository, where AgentEvaluationResults.ItemPassed ends with 'return result.Metrics.Count > 0' and is pinned by LocalEvaluator_WithZeroChecks_ItemsHaveZeroMetricsAndFailAsync. Add a focused regression covering the counts, all_passed, the empty score list, and raise_for_status(). Update the LocalEvaluator class and evaluate() docstrings, which previously described the pass rule without the zero-check case. Fixes #7397 --- python/packages/core/agent_framework/_evaluation.py | 10 ++++++---- python/packages/core/tests/core/test_local_eval.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py index 40190fc0e..cac000ca0 100644 --- a/python/packages/core/agent_framework/_evaluation.py +++ b/python/packages/core/agent_framework/_evaluation.py @@ -1520,7 +1520,8 @@ class LocalEvaluator: """Evaluation provider that runs checks locally without API calls. Implements the ``Evaluator`` protocol. Each check function is applied - to every item. An item passes only if all checks pass. + to every item. An item passes only if at least one check was evaluated + and all evaluated checks pass. Examples: Basic usage: @@ -1560,8 +1561,9 @@ class LocalEvaluator: ) -> EvalResults: """Run all checks on each item and return aggregated results. - An item passes only if every check passes for that item. Per-check - breakdowns are available in ``per_evaluator``. + An item passes only if every check passes for that item. An item with + no checks to evaluate fails, since a pass would carry no evidence. + Per-check breakdowns are available in ``per_evaluator``. Supports both sync and async check functions (from :func:`evaluator`). @@ -1574,7 +1576,7 @@ class LocalEvaluator: for item_idx, item in enumerate(items): check_results = await asyncio.gather(*[_run_check(fn, item) for fn in self._checks]) - item_passed = True + item_passed = bool(check_results) item_scores: list[EvalScoreResult] = [] for result in check_results: counts = per_check.setdefault(result.check_name, {"passed": 0, "failed": 0, "errored": 0}) diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index 1c640859d..f3713506c 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -374,6 +374,18 @@ class TestErrorHandling: class TestLocalEvaluatorIntegration: + @pytest.mark.asyncio + async def test_zero_checks(self): + """A LocalEvaluator with no checks produces items with 0 scores, which count as failed.""" + local = LocalEvaluator() + results = await local.evaluate([_make_item()]) + + assert results.result_counts == {"passed": 0, "failed": 1, "errored": 0} + assert results.all_passed is False + assert results.items[0].scores == [] + with pytest.raises(EvalNotPassedError): + results.raise_for_status() + @pytest.mark.asyncio async def test_mixed_checks(self): """Function evaluators mix with built-in checks in LocalEvaluator."""