| 1 | from __future__ import annotations
|
| 2 |
|
| 3 | import json
|
| 4 | import shutil
|
| 5 | import subprocess
|
| 6 | import sys
|
| 7 | import tempfile
|
| 8 | import unittest
|
| 9 | from pathlib import Path
|
| 10 |
|
| 11 | ROOT = Path(__file__).resolve().parents[1]
|
| 12 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 13 |
|
| 14 | from branch_triage_fixture_lane import scoring_failures # noqa: E402
|
| 15 | from oakbench.branch_triage_shape import ( # noqa: E402
|
| 16 | GENERATOR_VERSION,
|
| 17 | MANIFEST_FILENAME,
|
| 18 | TRIAGE_METRIC_KEYS,
|
| 19 | aggregate_triage_metrics,
|
| 20 | build_manifest,
|
| 21 | expected_triage_for_shape,
|
| 22 | score_branch,
|
| 23 | score_manifest_against_actuals,
|
| 24 | triage_metrics_row_fields,
|
| 25 | unmeasured_triage_metrics,
|
| 26 | validate_manifest,
|
| 27 | validate_triage_metrics,
|
| 28 | )
|
| 29 | from oakbench.fixture_registry import load_fixture_registry # noqa: E402
|
| 30 | from oakbench.rows import SKIP_RETURNCODE, validate_rows # noqa: E402
|
| 31 |
|
| 32 | GENERATOR = ROOT / "scripts" / "make_branch_triage_fixture.py"
|
| 33 | LANE = ROOT / "scripts" / "branch_triage_fixture_lane.py"
|
| 34 |
|
| 35 |
|
| 36 | def run_generator(out: Path, seed: str, branch_count: int) -> subprocess.CompletedProcess:
|
| 37 | return subprocess.run(
|
| 38 | [
|
| 39 | sys.executable,
|
| 40 | str(GENERATOR),
|
| 41 | "--out",
|
| 42 | str(out),
|
| 43 | "--seed",
|
| 44 | seed,
|
| 45 | "--branch-count",
|
| 46 | str(branch_count),
|
| 47 | ],
|
| 48 | capture_output=True,
|
| 49 | text=True,
|
| 50 | )
|
| 51 |
|
| 52 |
|
| 53 | def git(repo: Path, *args: str) -> subprocess.CompletedProcess:
|
| 54 | return subprocess.run(
|
| 55 | ["git", "-C", str(repo), *args],
|
| 56 | capture_output=True,
|
| 57 | text=True,
|
| 58 | check=True,
|
| 59 | )
|
| 60 |
|
| 61 |
|
| 62 | class BranchTriageShapeModuleTest(unittest.TestCase):
|
| 63 | def test_manifest_builder_is_deterministic(self) -> None:
|
| 64 | first = build_manifest(seed="test-seed", branch_count=6)
|
| 65 | second = build_manifest(seed="test-seed", branch_count=6)
|
| 66 | self.assertEqual(first, second)
|
| 67 |
|
| 68 | def test_manifest_labels_validate(self) -> None:
|
| 69 | manifest = build_manifest(seed="test-seed", branch_count=6)
|
| 70 | self.assertEqual(validate_manifest(manifest), [])
|
| 71 | self.assertEqual(len(manifest["branches"]), 6)
|
| 72 | shapes = {branch["shape"] for branch in manifest["branches"]}
|
| 73 | self.assertEqual(
|
| 74 | shapes,
|
| 75 | {
|
| 76 | "empty",
|
| 77 | "superseded_exact",
|
| 78 | "clean_contributes",
|
| 79 | "conflict_additive",
|
| 80 | "contributes_and_reverts_target",
|
| 81 | "missing_data",
|
| 82 | },
|
| 83 | )
|
| 84 |
|
| 85 | def test_expected_labels_match_product_semantics(self) -> None:
|
| 86 | empty = expected_triage_for_shape("empty").as_dict()
|
| 87 | self.assertTrue(empty["close_allowed"])
|
| 88 | self.assertEqual(empty["recommended_action"], "close")
|
| 89 | self.assertIsNone(empty["vcs_merge_safe"])
|
| 90 |
|
| 91 | conflict = expected_triage_for_shape("conflict_additive").as_dict()
|
| 92 | self.assertFalse(conflict["close_allowed"])
|
| 93 | self.assertEqual(conflict["vcs_merge_safe"], False)
|
| 94 | self.assertEqual(conflict["recommended_action"], "resolve")
|
| 95 |
|
| 96 | regressive = expected_triage_for_shape("contributes_and_reverts_target").as_dict()
|
| 97 | self.assertEqual(regressive["recommended_action"], "review")
|
| 98 | self.assertEqual(regressive["vcs_merge_safe"], True)
|
| 99 | self.assertFalse(regressive["merge_allowed"])
|
| 100 | self.assertEqual(regressive["target_risk"], "reverts_target_exact")
|
| 101 |
|
| 102 | def test_false_close_counter_fails_closed(self) -> None:
|
| 103 | expected = expected_triage_for_shape("clean_contributes").as_dict()
|
| 104 | actual = {
|
| 105 | "recommended_action": "close",
|
| 106 | "reason": "wrong",
|
| 107 | "close_allowed": True,
|
| 108 | "vcs_merge_safe": True,
|
| 109 | "merge_allowed": False,
|
| 110 | "mergeability": "clean",
|
| 111 | "contribution": "clean_contributes",
|
| 112 | "target_risk": "none",
|
| 113 | }
|
| 114 | row = score_branch(actual, expected)
|
| 115 | self.assertTrue(row["false_close"])
|
| 116 | self.assertFalse(row["correct"])
|
| 117 |
|
| 118 | def test_regressive_branch_marked_resolve_and_mergeable_fails(self) -> None:
|
| 119 | expected = expected_triage_for_shape("contributes_and_reverts_target").as_dict()
|
| 120 | actual = {
|
| 121 | "recommended_action": "resolve",
|
| 122 | "reason": "wrong",
|
| 123 | "close_allowed": False,
|
| 124 | "vcs_merge_safe": True,
|
| 125 | "merge_allowed": True,
|
| 126 | "mergeability": "clean",
|
| 127 | "contribution": "contributes_and_reverts_target",
|
| 128 | "target_risk": "reverts_target_exact",
|
| 129 | }
|
| 130 | row = score_branch(actual, expected)
|
| 131 | self.assertFalse(row["correct"])
|
| 132 | self.assertTrue(row["false_merge_allowed"])
|
| 133 |
|
| 134 | def test_missing_actual_is_unmeasured_not_correct(self) -> None:
|
| 135 | expected = expected_triage_for_shape("empty").as_dict()
|
| 136 | row = score_branch({}, expected)
|
| 137 | self.assertFalse(row["measured"])
|
| 138 | self.assertFalse(row["correct"])
|
| 139 |
|
| 140 | def test_unmeasured_metrics_are_null(self) -> None:
|
| 141 | metrics = unmeasured_triage_metrics(branches_expected=6)
|
| 142 | self.assertEqual(validate_triage_metrics(metrics), [])
|
| 143 | self.assertIsNone(metrics["triage_recall"])
|
| 144 | self.assertEqual(metrics["branches_expected"], 6)
|
| 145 |
|
| 146 | def test_aggregate_metrics_null_when_nothing_measured(self) -> None:
|
| 147 | expected = expected_triage_for_shape("empty").as_dict()
|
| 148 | branch_results = [
|
| 149 | {
|
| 150 | **score_branch({}, expected),
|
| 151 | "expected_close_allowed": expected["close_allowed"],
|
| 152 | "expected_vcs_merge_safe": expected["vcs_merge_safe"],
|
| 153 | "expected_merge_allowed": expected["merge_allowed"],
|
| 154 | }
|
| 155 | ]
|
| 156 | metrics = aggregate_triage_metrics(branch_results, branches_expected=1)
|
| 157 | self.assertIsNone(metrics["triage_recall"])
|
| 158 | self.assertIsNone(metrics["false_close_count"])
|
| 159 |
|
| 160 | def test_triage_metrics_row_contract(self) -> None:
|
| 161 | fields = triage_metrics_row_fields(
|
| 162 | {
|
| 163 | "triage_recall": 1.0,
|
| 164 | "triage_precision": 1.0,
|
| 165 | "false_close_count": 0,
|
| 166 | "close_eligible_n": 2,
|
| 167 | "false_vcs_merge_safe_count": 0,
|
| 168 | "vcs_merge_safe_n": 2,
|
| 169 | "false_merge_allowed_count": 0,
|
| 170 | "merge_allowed_n": 1,
|
| 171 | "branches_scored": 6,
|
| 172 | "branches_expected": 6,
|
| 173 | }
|
| 174 | )
|
| 175 | self.assertEqual(validate_triage_metrics(fields), [])
|
| 176 | row = {
|
| 177 | "bench_id": "20260614T000000Z",
|
| 178 | "profile": "branch-triage-fixture",
|
| 179 | "scenario": "branch_triage_shape",
|
| 180 | "operation": "triage.score",
|
| 181 | "run": 0,
|
| 182 | "subject": "oak",
|
| 183 | "subject_kind": "oak",
|
| 184 | "elapsed_ms": 1.0,
|
| 185 | "returncode": 0,
|
| 186 | "command": [],
|
| 187 | **fields,
|
| 188 | }
|
| 189 | self.assertEqual(validate_rows([row], "task-loop"), [])
|
| 190 |
|
| 191 |
|
| 192 | class MakeBranchTriageFixtureTest(unittest.TestCase):
|
| 193 | @classmethod
|
| 194 | def setUpClass(cls) -> None:
|
| 195 | if shutil.which("git") is None:
|
| 196 | raise unittest.SkipTest("git not available")
|
| 197 | cls.tmp = tempfile.TemporaryDirectory()
|
| 198 | cls.addClassCleanup(cls.tmp.cleanup)
|
| 199 | cls.out_a = Path(cls.tmp.name) / "a"
|
| 200 | cls.out_b = Path(cls.tmp.name) / "b"
|
| 201 | for out in (cls.out_a, cls.out_b):
|
| 202 | proc = run_generator(out, "test-seed", 6)
|
| 203 | if proc.returncode != 0:
|
| 204 | raise AssertionError(proc.stderr or proc.stdout)
|
| 205 |
|
| 206 | def test_manifest_present_and_valid(self) -> None:
|
| 207 | manifest = json.loads((self.out_a / MANIFEST_FILENAME).read_text())
|
| 208 | self.assertEqual(manifest["generator"], GENERATOR_VERSION)
|
| 209 | self.assertEqual(validate_manifest(manifest), [])
|
| 210 |
|
| 211 | def test_generator_is_deterministic(self) -> None:
|
| 212 | first = (self.out_a / MANIFEST_FILENAME).read_text()
|
| 213 | second = (self.out_b / MANIFEST_FILENAME).read_text()
|
| 214 | self.assertEqual(first, second)
|
| 215 |
|
| 216 | def test_all_manifest_branches_exist_in_repo(self) -> None:
|
| 217 | manifest = json.loads((self.out_a / MANIFEST_FILENAME).read_text())
|
| 218 | branches = git(self.out_a / "repo", "branch", "--format=%(refname:short)")
|
| 219 | present = {line.strip() for line in branches.stdout.splitlines() if line.strip()}
|
| 220 | for branch in manifest["branches"]:
|
| 221 | self.assertIn(branch["name"], present)
|
| 222 |
|
| 223 | def test_manifest_shapes_have_expected_tree_diffs_against_main(self) -> None:
|
| 224 | manifest = json.loads((self.out_a / MANIFEST_FILENAME).read_text())
|
| 225 | repo = self.out_a / "repo"
|
| 226 | expected_paths = {
|
| 227 | "empty": set(),
|
| 228 | "superseded_exact": set(),
|
| 229 | "clean_contributes": {"contrib/clean-2.txt"},
|
| 230 | "conflict_additive": {"conflict/shared-3.txt"},
|
| 231 | "contributes_and_reverts_target": {
|
| 232 | "contrib/reverts-target-4.txt",
|
| 233 | "target/risk-4.txt",
|
| 234 | },
|
| 235 | "missing_data": {
|
| 236 | "missing/manual-5.txt",
|
| 237 | "missing/payload-5.bin",
|
| 238 | },
|
| 239 | }
|
| 240 | for entry in manifest["branches"]:
|
| 241 | diff = git(repo, "diff", "--name-only", "main", entry["name"])
|
| 242 | actual = {line.strip() for line in diff.stdout.splitlines() if line.strip()}
|
| 243 | self.assertEqual(actual, expected_paths[entry["shape"]], entry["name"])
|
| 244 |
|
| 245 | def test_medium_fixture_generates_sixty_branches(self) -> None:
|
| 246 | out = Path(self.tmp.name) / "medium"
|
| 247 | proc = run_generator(out, "test-seed", 60)
|
| 248 | self.assertEqual(proc.returncode, 0, proc.stderr)
|
| 249 | manifest = json.loads((out / MANIFEST_FILENAME).read_text())
|
| 250 | self.assertEqual(len(manifest["branches"]), 60)
|
| 251 |
|
| 252 |
|
| 253 | class BranchTriageFixtureLaneTest(unittest.TestCase):
|
| 254 | def test_lane_skips_without_oak(self) -> None:
|
| 255 | if shutil.which("git") is None:
|
| 256 | self.skipTest("git not available")
|
| 257 | with tempfile.TemporaryDirectory() as tmp:
|
| 258 | workdir = Path(tmp) / "work"
|
| 259 | proc = subprocess.run(
|
| 260 | [
|
| 261 | sys.executable,
|
| 262 | str(LANE),
|
| 263 | "--workdir",
|
| 264 | str(workdir),
|
| 265 | "--dry-run",
|
| 266 | "--quick",
|
| 267 | "--oak-bin",
|
| 268 | str(Path(tmp) / "missing-oak"),
|
| 269 | ],
|
| 270 | capture_output=True,
|
| 271 | text=True,
|
| 272 | )
|
| 273 | self.assertEqual(proc.returncode, 0, proc.stderr)
|
| 274 | rows = json.loads(proc.stdout)
|
| 275 | score_rows = [row for row in rows if row["operation"] == "triage.score"]
|
| 276 | self.assertEqual(len(score_rows), 1)
|
| 277 | row = score_rows[0]
|
| 278 | self.assertEqual(row["returncode"], SKIP_RETURNCODE)
|
| 279 | self.assertTrue(row["skipped"])
|
| 280 | self.assertIn("oak_binary_unavailable", row["skip_reason"])
|
| 281 | for key in TRIAGE_METRIC_KEYS:
|
| 282 | if key == "branches_expected":
|
| 283 | self.assertEqual(row[key], 6)
|
| 284 | else:
|
| 285 | self.assertIsNone(row[key])
|
| 286 |
|
| 287 | def test_lane_writes_skip_result_without_oak(self) -> None:
|
| 288 | if shutil.which("git") is None:
|
| 289 | self.skipTest("git not available")
|
| 290 | with tempfile.TemporaryDirectory() as tmp:
|
| 291 | workdir = Path(tmp) / "work"
|
| 292 | results = Path(tmp) / "results"
|
| 293 | proc = subprocess.run(
|
| 294 | [
|
| 295 | sys.executable,
|
| 296 | str(LANE),
|
| 297 | "--workdir",
|
| 298 | str(workdir),
|
| 299 | "--results",
|
| 300 | str(results),
|
| 301 | "--quick",
|
| 302 | "--oak-bin",
|
| 303 | str(Path(tmp) / "missing-oak"),
|
| 304 | ],
|
| 305 | capture_output=True,
|
| 306 | text=True,
|
| 307 | )
|
| 308 | self.assertEqual(proc.returncode, 0, proc.stderr)
|
| 309 | latest = results / "latest.jsonl"
|
| 310 | self.assertTrue(latest.is_file())
|
| 311 | rows = [json.loads(line) for line in latest.read_text().splitlines() if line.strip()]
|
| 312 | self.assertEqual(len(rows), 2)
|
| 313 | self.assertEqual(rows[-1]["operation"], "triage.score")
|
| 314 | self.assertEqual(rows[-1]["returncode"], SKIP_RETURNCODE)
|
| 315 |
|
| 316 | def test_score_manifest_with_fake_actuals(self) -> None:
|
| 317 | manifest = build_manifest(seed="score-test", branch_count=6)
|
| 318 | actuals = {
|
| 319 | branch["name"]: branch["expected"]
|
| 320 | for branch in manifest["branches"]
|
| 321 | }
|
| 322 | scored = score_manifest_against_actuals(manifest, actuals)
|
| 323 | self.assertEqual(scored["metrics"]["triage_recall"], 1.0)
|
| 324 | self.assertEqual(scored["metrics"]["triage_precision"], 1.0)
|
| 325 | self.assertEqual(scored["metrics"]["false_close_count"], 0)
|
| 326 | self.assertEqual(scoring_failures(scored), [])
|
| 327 |
|
| 328 | def test_regressive_branch_score_failure_is_exposed_to_lane(self) -> None:
|
| 329 | manifest = build_manifest(seed="score-test", branch_count=6)
|
| 330 | actuals = {
|
| 331 | branch["name"]: dict(branch["expected"])
|
| 332 | for branch in manifest["branches"]
|
| 333 | }
|
| 334 | regressive = manifest["branches"][4]
|
| 335 | actuals[regressive["name"]].update(
|
| 336 | {
|
| 337 | "recommended_action": "resolve",
|
| 338 | "merge_allowed": True,
|
| 339 | }
|
| 340 | )
|
| 341 | scored = score_manifest_against_actuals(manifest, actuals)
|
| 342 | failures = scoring_failures(scored)
|
| 343 | self.assertEqual(scored["metrics"]["false_merge_allowed_count"], 1)
|
| 344 | self.assertEqual(len(failures), 1)
|
| 345 | self.assertEqual(failures[0]["shape"], "contributes_and_reverts_target")
|
| 346 | self.assertLess(scored["metrics"]["triage_recall"], 1.0)
|
| 347 | self.assertLess(scored["metrics"]["triage_precision"], 1.0)
|
| 348 |
|
| 349 |
|
| 350 | class FixtureRegistryTest(unittest.TestCase):
|
| 351 | def test_registry_declares_branch_triage_fixture(self) -> None:
|
| 352 | registry = load_fixture_registry()
|
| 353 | self.assertIn("bench-branch-triage-shape-v2", registry)
|
| 354 | spec = registry["bench-branch-triage-shape-v2"]
|
| 355 | self.assertEqual(spec.generator, "scripts/make_branch_triage_fixture.py")
|
| 356 |
|
| 357 |
|
| 358 | if __name__ == "__main__":
|
| 359 | unittest.main()
|