Christopher Haugen (from Dev Box)
6bac6f408b
_split() fell back to al = train when no validation task existed, so on a thin mine the gate scored the very tasks the edits were derived from. That comparison cannot detect overfitting, yet the night reports it as a validated held-out improvement -- a silent wrong answer rather than a visible failure. Observed: a single mined task whose rule judge was section_present=Results. The optimizer added a '## Results' heading, the gate scored the same task it had just optimized, and the night reported held-out 0.250 -> 1.000 and accepted the edit. Any batch where every task hashes to one split reaches the same state, in both directions -- with only a val task, train borrows from it. _split now reports whether val is disjoint from train (including overlap by id), and consolidate fails closed: non-disjoint slices yield the action eject_unverified instead of a verdict the evidence cannot support. Edits are still staged and report.md states that they are unverified suggestions, so a thin night still surfaces material for a human to read -- it just stops claiming the material was validated. tests/test_holdout_integrity.py covers both halves. The second matters as much as the first: a gate that rejects everything would be trivially safe and useless, so there is an explicit test that a candidate which genuinely scores better on tasks the optimizer never saw is still accepted.
91 lines
3.4 KiB
Python
91 lines
3.4 KiB
Python
"""Regression tests for consolidate._split hold-out contract.
|
|
|
|
test-split tasks must never enter train or val. An all-test batch must not
|
|
silently fall back to using the held-out set for consolidation.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
|
|
from skillopt_sleep.consolidate import _split
|
|
from skillopt_sleep.tasks_file import load_tasks_file, write_tasks_file
|
|
from skillopt_sleep.types import TaskRecord
|
|
|
|
|
|
def _ids(tasks):
|
|
return sorted(t.id for t in tasks)
|
|
|
|
|
|
class TestConsolidateSplit(unittest.TestCase):
|
|
def test_all_test_batch_does_not_leak_into_train_or_val(self):
|
|
tasks = [
|
|
TaskRecord(id="t0", project="p", intent="do X", split="test"),
|
|
TaskRecord(id="t1", project="p", intent="do Y", split="test"),
|
|
TaskRecord(id="t2", project="p", intent="do Z", split="test"),
|
|
]
|
|
train, val, _leaked = _split(tasks)
|
|
self.assertEqual(train, [])
|
|
self.assertEqual(val, [])
|
|
|
|
def test_all_test_via_tasks_file_path_does_not_leak(self):
|
|
tasks = [
|
|
TaskRecord(id="t0", project="p", intent="do X", split="test"),
|
|
TaskRecord(id="t1", project="p", intent="do Y", split="test"),
|
|
TaskRecord(id="t2", project="p", intent="do Z", split="test"),
|
|
]
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = write_tasks_file(
|
|
os.path.join(tmp, "tasks.json"),
|
|
{"tasks": [t.to_dict() for t in tasks]},
|
|
)
|
|
loaded, _ = load_tasks_file(path)
|
|
train, val, _leaked = _split(loaded)
|
|
self.assertEqual(_ids(train), [])
|
|
self.assertEqual(_ids(val), [])
|
|
self.assertEqual({t.split for t in loaded}, {"test"})
|
|
|
|
def test_train_only_falls_back_val_to_train(self):
|
|
tasks = [
|
|
TaskRecord(id="a", project="p", intent="A", split="train"),
|
|
TaskRecord(id="b", project="p", intent="B", split="train"),
|
|
]
|
|
train, val, _leaked = _split(tasks)
|
|
self.assertEqual(_ids(train), ["a", "b"])
|
|
self.assertEqual(_ids(val), ["a", "b"])
|
|
|
|
def test_train_plus_test_without_val_gates_on_train_not_test(self):
|
|
tasks = [
|
|
TaskRecord(id="tr", project="p", intent="train", split="train"),
|
|
TaskRecord(id="te", project="p", intent="test", split="test"),
|
|
]
|
|
train, val, _leaked = _split(tasks)
|
|
self.assertEqual(_ids(train), ["tr"])
|
|
self.assertEqual(_ids(val), ["tr"])
|
|
self.assertNotIn("te", _ids(train) + _ids(val))
|
|
|
|
def test_explicit_train_val_test_keeps_partitions(self):
|
|
tasks = [
|
|
TaskRecord(id="tr", project="p", intent="train", split="train"),
|
|
TaskRecord(id="va", project="p", intent="val", split="val"),
|
|
TaskRecord(id="te", project="p", intent="test", split="test"),
|
|
]
|
|
train, val, _leaked = _split(tasks)
|
|
self.assertEqual(_ids(train), ["tr"])
|
|
self.assertEqual(_ids(val), ["va"])
|
|
|
|
def test_legacy_holdout_name_maps_to_val(self):
|
|
tasks = [
|
|
TaskRecord(id="tr", project="p", intent="train", split="replay"),
|
|
TaskRecord(id="va", project="p", intent="val", split="holdout"),
|
|
TaskRecord(id="te", project="p", intent="test", split="test"),
|
|
]
|
|
train, val, _leaked = _split(tasks)
|
|
self.assertEqual(_ids(train), ["tr"])
|
|
self.assertEqual(_ids(val), ["va"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|