Max
106 lines
4.8 KiB
Python
106 lines
4.8 KiB
Python
"""Enforces the contract between the requirements manifest and the test suite.
|
|
|
|
The contract runs in both directions: every non-deferred entry in :data:`REQUIREMENTS` must be
|
|
exercised by at least one test, and every test in the suite must carry at least one
|
|
`@requirement(...)` mark referencing a manifest entry. Deferral reasons that point at coverage
|
|
elsewhere in the repo must point at paths that exist. Test modules are imported directly
|
|
(rather than relying on pytest collection) so the check holds even when only this file is run.
|
|
"""
|
|
|
|
import importlib
|
|
import re
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
|
|
import pytest
|
|
|
|
from tests.interaction._requirements import REQUIREMENTS, Requirement, covered_by, requirement
|
|
|
|
_SUITE_ROOT = Path(__file__).parent
|
|
_REPO_ROOT = _SUITE_ROOT.parent.parent
|
|
|
|
# Repo paths cited inside deferral reasons ("Covered by tests/... ").
|
|
_CITED_PATH = re.compile(r"(?:tests|src)/[\w./-]*\w")
|
|
|
|
# Tests that exercise the suite's own helpers rather than an interaction-model behaviour.
|
|
# Anything listed here is exempt from the every-test-has-a-requirement check.
|
|
_HARNESS_SELF_TESTS = {
|
|
"tests.interaction.lowlevel.test_wire.test_recording_read_stream_ends_iteration_when_the_sender_closes",
|
|
"tests.interaction.transports.test_bridge.test_response_chunks_arrive_as_the_application_sends_them",
|
|
"tests.interaction.transports.test_bridge.test_closing_the_response_delivers_a_disconnect_to_the_application",
|
|
"tests.interaction.transports.test_bridge.test_an_application_failure_before_the_response_starts_fails_the_request",
|
|
"tests.interaction.transports.test_bridge.test_disabling_cancel_on_close_lets_the_application_finish_after_disconnect",
|
|
"tests.interaction.auth.test_flow.test_shimmed_app_serves_overrides_404s_and_otherwise_forwards_to_the_wrapped_app",
|
|
}
|
|
|
|
|
|
def _import_all_test_modules() -> list[ModuleType]:
|
|
"""Import every other test module in the suite so their `@requirement` decorators register."""
|
|
modules: list[ModuleType] = []
|
|
for path in sorted(_SUITE_ROOT.rglob("test_*.py")):
|
|
relative = path.relative_to(_SUITE_ROOT).with_suffix("")
|
|
name = f"{__package__}.{'.'.join(relative.parts)}"
|
|
if name != __name__:
|
|
modules.append(importlib.import_module(name))
|
|
return modules
|
|
|
|
|
|
def test_every_requirement_is_exercised() -> None:
|
|
"""Each non-deferred requirement is covered by at least one test (deferred ones by none)."""
|
|
_import_all_test_modules()
|
|
|
|
uncovered = [
|
|
requirement_id
|
|
for requirement_id, spec in sorted(REQUIREMENTS.items())
|
|
if spec.deferred is None and not covered_by(requirement_id)
|
|
]
|
|
assert not uncovered, f"Requirements with no test and no deferred reason: {uncovered}"
|
|
|
|
stale_deferrals = [
|
|
requirement_id
|
|
for requirement_id, spec in sorted(REQUIREMENTS.items())
|
|
if spec.deferred is not None and covered_by(requirement_id)
|
|
]
|
|
assert not stale_deferrals, f"Deferred requirements that now have tests (remove deferred): {stale_deferrals}"
|
|
|
|
|
|
def test_every_test_exercises_a_requirement() -> None:
|
|
"""Each test in the suite carries at least one `@requirement` mark (harness self-tests excepted)."""
|
|
all_tests = {
|
|
f"{module.__name__}.{name}"
|
|
for module in _import_all_test_modules()
|
|
for name in vars(module)
|
|
if name.startswith("test_")
|
|
}
|
|
linked_tests = {test_name for requirement_id in REQUIREMENTS for test_name in covered_by(requirement_id)}
|
|
|
|
unlinked = sorted(all_tests - linked_tests - _HARNESS_SELF_TESTS)
|
|
assert not unlinked, f"Tests with no @requirement mark: {unlinked}"
|
|
|
|
stale_exemptions = sorted(_HARNESS_SELF_TESTS - all_tests)
|
|
assert not stale_exemptions, f"Harness self-test exemptions that no longer exist: {stale_exemptions}"
|
|
|
|
|
|
def test_deferral_reasons_cite_existing_paths() -> None:
|
|
"""Every repo path named in a deferral reason exists, so coverage pointers cannot rot."""
|
|
missing = sorted(
|
|
f"{requirement_id}: {cited}"
|
|
for requirement_id, spec in REQUIREMENTS.items()
|
|
if spec.deferred is not None
|
|
for cited in _CITED_PATH.findall(spec.deferred)
|
|
if not (_REPO_ROOT / cited).exists()
|
|
)
|
|
assert not missing, f"Deferral reasons citing paths that do not exist: {missing}"
|
|
|
|
|
|
def test_unknown_requirement_id_is_rejected() -> None:
|
|
"""Marking a test with an ID that is not in the manifest fails at decoration time."""
|
|
with pytest.raises(KeyError, match="Unknown requirement id 'tools:call:does-not-exist'"):
|
|
requirement("tools:call:does-not-exist")
|
|
|
|
|
|
def test_invalid_requirement_source_is_rejected() -> None:
|
|
"""A requirement whose source is not a spec URL, 'sdk', or an issue reference fails at construction."""
|
|
with pytest.raises(ValueError, match="source must be a specification URL"):
|
|
Requirement(source="https://example.com/not-the-spec", behavior="Never constructed.")
|