| 1 | from __future__ import annotations
|
| 2 |
|
| 3 | import json
|
| 4 | import subprocess
|
| 5 | import sys
|
| 6 | import tempfile
|
| 7 | import unittest
|
| 8 | from pathlib import Path
|
| 9 |
|
| 10 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 11 |
|
| 12 | from oakbench import perf_loop
|
| 13 |
|
| 14 |
|
| 15 | def row(
|
| 16 | subject: str,
|
| 17 | scenario: str,
|
| 18 | operation: str,
|
| 19 | elapsed_ms: float,
|
| 20 | *,
|
| 21 | profile: str = "large",
|
| 22 | remote_transport: str | None = None,
|
| 23 | run: int = 0,
|
| 24 | ) -> dict:
|
| 25 | payload = {
|
| 26 | "bench_id": "b",
|
| 27 | "profile": profile,
|
| 28 | "scenario": scenario,
|
| 29 | "operation": operation,
|
| 30 | "run": run,
|
| 31 | "subject": subject,
|
| 32 | "subject_kind": "oak" if subject.startswith("oak") else "git",
|
| 33 | "elapsed_ms": elapsed_ms,
|
| 34 | "returncode": 0,
|
| 35 | "command": [subject, operation],
|
| 36 | }
|
| 37 | if remote_transport is not None:
|
| 38 | payload["remote_transport"] = remote_transport
|
| 39 | return payload
|
| 40 |
|
| 41 |
|
| 42 | class PerfScorecardTest(unittest.TestCase):
|
| 43 | def test_scorecard_reports_ratios_and_baseline_diff(self) -> None:
|
| 44 | rows = [
|
| 45 | row("git", "many_small_files_50k", "status.dirty", 90.0, run=0),
|
| 46 | row("git", "many_small_files_50k", "status.dirty", 100.0, run=1),
|
| 47 | row("oak_installed", "many_small_files_50k", "status.dirty", 150.0, run=0),
|
| 48 | row("oak_installed", "many_small_files_50k", "status.dirty", 160.0, run=1),
|
| 49 | ]
|
| 50 | baseline = perf_loop.build_scorecard(rows, oak_subject="oak_installed")
|
| 51 | improved_rows = [
|
| 52 | row("git", "many_small_files_50k", "status.dirty", 95.0, run=0),
|
| 53 | row("oak_installed", "many_small_files_50k", "status.dirty", 120.0, run=0),
|
| 54 | ]
|
| 55 |
|
| 56 | scorecard = perf_loop.build_scorecard(
|
| 57 | improved_rows,
|
| 58 | oak_subject="oak_installed",
|
| 59 | baseline=baseline,
|
| 60 | change_threshold_pct=3.0,
|
| 61 | )
|
| 62 |
|
| 63 | entry = scorecard["entries"][0]
|
| 64 | self.assertEqual(entry["scenario"], "many_small_files_50k")
|
| 65 | self.assertEqual(entry["operation"], "status.dirty")
|
| 66 | self.assertAlmostEqual(entry["ratio"], 120.0 / 95.0)
|
| 67 | self.assertEqual(entry["verdict"], "oak_slower")
|
| 68 | self.assertEqual(entry["baseline"]["verdict"], "better")
|
| 69 | self.assertEqual(scorecard["summary"]["gate_verdict"], "better")
|
| 70 |
|
| 71 | def test_transport_mismatch_is_not_compared(self) -> None:
|
| 72 | rows = [
|
| 73 | row("git", "tiny_text", "remote.clone.cold", 10.0, remote_transport="local_file"),
|
| 74 | row("oak_installed", "tiny_text", "remote.clone.cold", 20.0, remote_transport="network"),
|
| 75 | ]
|
| 76 |
|
| 77 | scorecard = perf_loop.build_scorecard(rows, oak_subject="oak_installed")
|
| 78 |
|
| 79 | self.assertEqual(scorecard["entries"], [])
|
| 80 | self.assertEqual(scorecard["skipped_comparisons"][0]["reason"], "transport_mismatch")
|
| 81 |
|
| 82 | def test_scorecard_cli_exit_codes_are_tri_state(self) -> None:
|
| 83 | self.assertEqual(perf_loop.scorecard_exit_code("better"), perf_loop.SCORECARD_EXIT_BETTER)
|
| 84 | self.assertEqual(perf_loop.scorecard_exit_code("no-change"), perf_loop.SCORECARD_EXIT_NO_CHANGE)
|
| 85 | self.assertEqual(perf_loop.scorecard_exit_code("regressed"), perf_loop.SCORECARD_EXIT_REGRESSED)
|
| 86 |
|
| 87 |
|
| 88 | class PerfTargetSelectionTest(unittest.TestCase):
|
| 89 | def test_selects_worst_non_diagnostic_target_and_respects_no_payoff_ledger(self) -> None:
|
| 90 | scorecard = {
|
| 91 | "entries": [
|
| 92 | {
|
| 93 | "lane": "core",
|
| 94 | "profile": "large",
|
| 95 | "scenario": "many_small_files_50k",
|
| 96 | "operation": "status.dirty.determinism",
|
| 97 | "metric": "elapsed_ms",
|
| 98 | "ratio": 9.0,
|
| 99 | },
|
| 100 | {
|
| 101 | "lane": "core",
|
| 102 | "profile": "large",
|
| 103 | "scenario": "many_small_files_50k",
|
| 104 | "operation": "status.dirty",
|
| 105 | "metric": "elapsed_ms",
|
| 106 | "ratio": 1.65,
|
| 107 | },
|
| 108 | {
|
| 109 | "lane": "core",
|
| 110 | "profile": "large",
|
| 111 | "scenario": "wide_dirty_tree",
|
| 112 | "operation": "status.dirty",
|
| 113 | "metric": "elapsed_ms",
|
| 114 | "ratio": 1.8,
|
| 115 | },
|
| 116 | ]
|
| 117 | }
|
| 118 | ledger = [
|
| 119 | {
|
| 120 | "target": {
|
| 121 | "lane": "core",
|
| 122 | "profile": "large",
|
| 123 | "scenario": "wide_dirty_tree",
|
| 124 | "operation": "status.dirty",
|
| 125 | },
|
| 126 | "landed_or_parked": "parked_no_payoff",
|
| 127 | }
|
| 128 | ]
|
| 129 |
|
| 130 | target = perf_loop.select_target(scorecard, ledger)
|
| 131 |
|
| 132 | self.assertIsNotNone(target)
|
| 133 | self.assertEqual(target["scenario"], "many_small_files_50k")
|
| 134 | self.assertEqual(target["operation"], "status.dirty")
|
| 135 |
|
| 136 | def test_ledger_roundtrip_parses_appended_entries(self) -> None:
|
| 137 | with tempfile.TemporaryDirectory() as tmp:
|
| 138 | path = Path(tmp) / "LEDGER.md"
|
| 139 | entry = {
|
| 140 | "timestamp": "t",
|
| 141 | "target": {
|
| 142 | "lane": "core",
|
| 143 | "profile": "large",
|
| 144 | "scenario": "s",
|
| 145 | "operation": "op",
|
| 146 | },
|
| 147 | "landed_or_parked": "parked",
|
| 148 | "reason": "dry_run",
|
| 149 | }
|
| 150 |
|
| 151 | perf_loop.append_ledger(path, entry)
|
| 152 |
|
| 153 | parsed = perf_loop.parse_ledger(path)
|
| 154 | self.assertEqual(len(parsed), 1)
|
| 155 | self.assertEqual(parsed[0]["reason"], "dry_run")
|
| 156 |
|
| 157 |
|
| 158 | class PerfVerificationTest(unittest.TestCase):
|
| 159 | def _green_ab_result(self) -> dict:
|
| 160 | return {
|
| 161 | "passed": True,
|
| 162 | "before_ms": 100.0,
|
| 163 | "after_ms": 90.0,
|
| 164 | "delta_pct": -10.0,
|
| 165 | }
|
| 166 |
|
| 167 | def test_interleaved_ab_requires_twenty_pairs_and_alternation(self) -> None:
|
| 168 | too_few = [
|
| 169 | perf_loop.ABSample("before", 100.0),
|
| 170 | perf_loop.ABSample("after", 90.0),
|
| 171 | ]
|
| 172 | result = perf_loop.evaluate_interleaved_ab(too_few)
|
| 173 | self.assertFalse(result["passed"])
|
| 174 | self.assertIn("20", result["reason"])
|
| 175 |
|
| 176 | not_alternating = [
|
| 177 | perf_loop.ABSample("before", 100.0),
|
| 178 | perf_loop.ABSample("before", 99.0),
|
| 179 | ] * 20
|
| 180 | result = perf_loop.evaluate_interleaved_ab(not_alternating, min_pairs=20)
|
| 181 | self.assertFalse(result["passed"])
|
| 182 | self.assertIn("alternating", result["reason"])
|
| 183 |
|
| 184 | def test_interleaved_ab_passes_only_when_after_median_improves(self) -> None:
|
| 185 | samples: list[perf_loop.ABSample] = []
|
| 186 | for _ in range(20):
|
| 187 | samples.append(perf_loop.ABSample("before", 100.0))
|
| 188 | samples.append(perf_loop.ABSample("after", 80.0))
|
| 189 |
|
| 190 | result = perf_loop.evaluate_interleaved_ab(samples, min_pairs=20, min_improvement_pct=1.0)
|
| 191 |
|
| 192 | self.assertTrue(result["passed"])
|
| 193 | self.assertEqual(result["before_ms"], 100.0)
|
| 194 | self.assertEqual(result["after_ms"], 80.0)
|
| 195 | self.assertEqual(result["reason"], "target_metric_improved")
|
| 196 |
|
| 197 | def test_interleaved_ab_rejects_noisy_overlapping_improvement(self) -> None:
|
| 198 | samples: list[perf_loop.ABSample] = []
|
| 199 | before_values = [90.0] * 5 + [100.0] * 15
|
| 200 | after_values = [98.5] * 20
|
| 201 | for before_ms, after_ms in zip(before_values, after_values):
|
| 202 | samples.append(perf_loop.ABSample("before", before_ms))
|
| 203 | samples.append(perf_loop.ABSample("after", after_ms))
|
| 204 |
|
| 205 | result = perf_loop.evaluate_interleaved_ab(samples, min_pairs=20, min_improvement_pct=1.0)
|
| 206 |
|
| 207 | self.assertFalse(result["passed"])
|
| 208 | self.assertEqual(result["reason"], "target_metric_improvement_not_consistent")
|
| 209 |
|
| 210 | def test_verification_entry_is_parked_in_propose_mode_even_when_green(self) -> None:
|
| 211 | target = {
|
| 212 | "lane": "core",
|
| 213 | "profile": "large",
|
| 214 | "scenario": "many_small_files_50k",
|
| 215 | "operation": "status.dirty",
|
| 216 | "ratio": 1.65,
|
| 217 | }
|
| 218 |
|
| 219 | entry = perf_loop.verification_entry(
|
| 220 | target=target,
|
| 221 | diagnosis="measured walk/cache/hash/diff phases",
|
| 222 | change_summary="remove one full pass",
|
| 223 | correctness_results=[{"command": ["cargo", "test"], "returncode": 0}],
|
| 224 | ab_result=self._green_ab_result(),
|
| 225 | oak_gate_verdict="PASS",
|
| 226 | fleet_gate_verdict="PASS",
|
| 227 | )
|
| 228 |
|
| 229 | self.assertEqual(entry["landed_or_parked"], "parked")
|
| 230 | self.assertEqual(entry["reason"], "verified_propose_mode_human_review_required")
|
| 231 |
|
| 232 | def test_missing_diagnosis_parks_candidate(self) -> None:
|
| 233 | entry = perf_loop.verification_entry(
|
| 234 | target={"scenario": "s", "operation": "op"},
|
| 235 | diagnosis="",
|
| 236 | change_summary="summary",
|
| 237 | correctness_results=[{"returncode": 0}],
|
| 238 | ab_result={"passed": True},
|
| 239 | )
|
| 240 |
|
| 241 | self.assertEqual(entry["reason"], "missing_diagnosis_measurements")
|
| 242 |
|
| 243 | def test_failed_correctness_gate_parks_candidate(self) -> None:
|
| 244 | entry = perf_loop.verification_entry(
|
| 245 | target={"scenario": "s", "operation": "op"},
|
| 246 | diagnosis="measured phase timings",
|
| 247 | change_summary="candidate change",
|
| 248 | correctness_results=[{"command": ["cargo", "test"], "returncode": 1}],
|
| 249 | ab_result=self._green_ab_result(),
|
| 250 | oak_gate_verdict="PASS",
|
| 251 | fleet_gate_verdict="PASS",
|
| 252 | )
|
| 253 |
|
| 254 | self.assertEqual(entry["landed_or_parked"], "parked")
|
| 255 | self.assertEqual(entry["reason"], "correctness_gate_failed")
|
| 256 |
|
| 257 | def test_missing_oak_efficiency_gate_parks_candidate(self) -> None:
|
| 258 | entry = perf_loop.verification_entry(
|
| 259 | target={"scenario": "s", "operation": "op"},
|
| 260 | diagnosis="measured phase timings",
|
| 261 | change_summary="candidate change",
|
| 262 | correctness_results=[{"returncode": 0}],
|
| 263 | ab_result=self._green_ab_result(),
|
| 264 | oak_gate_verdict="MISSING",
|
| 265 | fleet_gate_verdict="PASS",
|
| 266 | )
|
| 267 |
|
| 268 | self.assertEqual(entry["landed_or_parked"], "parked")
|
| 269 | self.assertEqual(entry["reason"], "oak_efficiency_gate_regressed")
|
| 270 |
|
| 271 | def test_missing_fleet_reliability_gate_parks_candidate(self) -> None:
|
| 272 | entry = perf_loop.verification_entry(
|
| 273 | target={"scenario": "s", "operation": "op"},
|
| 274 | diagnosis="measured phase timings",
|
| 275 | change_summary="candidate change",
|
| 276 | correctness_results=[{"returncode": 0}],
|
| 277 | ab_result=self._green_ab_result(),
|
| 278 | oak_gate_verdict="PASS",
|
| 279 | fleet_gate_verdict="MISSING",
|
| 280 | )
|
| 281 |
|
| 282 | self.assertEqual(entry["landed_or_parked"], "parked")
|
| 283 | self.assertEqual(entry["reason"], "fleet_reliability_gate_regressed")
|
| 284 |
|
| 285 | def test_verification_entry_gate_defaults_fail_closed(self) -> None:
|
| 286 | entry = perf_loop.verification_entry(
|
| 287 | target={"scenario": "s", "operation": "op"},
|
| 288 | diagnosis="measured phase timings",
|
| 289 | change_summary="candidate change",
|
| 290 | correctness_results=[{"returncode": 0}],
|
| 291 | ab_result=self._green_ab_result(),
|
| 292 | )
|
| 293 |
|
| 294 | self.assertEqual(entry["landed_or_parked"], "parked")
|
| 295 | self.assertEqual(entry["reason"], "oak_efficiency_gate_regressed")
|
| 296 |
|
| 297 |
|
| 298 | class PerfLoopCliTest(unittest.TestCase):
|
| 299 | def test_dry_run_selects_and_appends_parked_ledger_entry(self) -> None:
|
| 300 | with tempfile.TemporaryDirectory() as tmp:
|
| 301 | root = Path(tmp)
|
| 302 | scorecard = root / "scorecard.json"
|
| 303 | ledger = root / "LEDGER.md"
|
| 304 | scorecard.write_text(
|
| 305 | json.dumps(
|
| 306 | {
|
| 307 | "entries": [
|
| 308 | {
|
| 309 | "lane": "core",
|
| 310 | "profile": "large",
|
| 311 | "scenario": "many_small_files_50k",
|
| 312 | "operation": "status.dirty",
|
| 313 | "metric": "elapsed_ms",
|
| 314 | "ratio": 1.65,
|
| 315 | "oak_median_ms": 152.5,
|
| 316 | "git_median_ms": 92.4,
|
| 317 | }
|
| 318 | ]
|
| 319 | }
|
| 320 | )
|
| 321 | )
|
| 322 |
|
| 323 | proc = subprocess.run(
|
| 324 | [
|
| 325 | sys.executable,
|
| 326 | str(Path(__file__).resolve().parents[1] / "scripts" / "perf_loop.py"),
|
| 327 | "dry-run",
|
| 328 | "--scorecard",
|
| 329 | str(scorecard),
|
| 330 | "--ledger",
|
| 331 | str(ledger),
|
| 332 | ],
|
| 333 | text=True,
|
| 334 | stdout=subprocess.PIPE,
|
| 335 | stderr=subprocess.PIPE,
|
| 336 | check=False,
|
| 337 | )
|
| 338 |
|
| 339 | self.assertEqual(proc.returncode, 0, proc.stderr)
|
| 340 | entries = perf_loop.parse_ledger(ledger)
|
| 341 | self.assertEqual(len(entries), 1)
|
| 342 | self.assertEqual(entries[0]["reason"], "dry_run_no_candidate_branch")
|
| 343 | self.assertEqual(entries[0]["target"]["operation"], "status.dirty")
|
| 344 |
|
| 345 | def test_dry_run_stops_on_red_scorecard(self) -> None:
|
| 346 | with tempfile.TemporaryDirectory() as tmp:
|
| 347 | root = Path(tmp)
|
| 348 | scorecard = root / "scorecard.json"
|
| 349 | ledger = root / "LEDGER.md"
|
| 350 | scorecard.write_text(
|
| 351 | json.dumps(
|
| 352 | {
|
| 353 | "summary": {"gate_verdict": "regressed"},
|
| 354 | "entries": [
|
| 355 | {
|
| 356 | "lane": "core",
|
| 357 | "profile": "large",
|
| 358 | "scenario": "many_small_files_50k",
|
| 359 | "operation": "status.dirty",
|
| 360 | "metric": "elapsed_ms",
|
| 361 | "ratio": 1.65,
|
| 362 | }
|
| 363 | ],
|
| 364 | }
|
| 365 | )
|
| 366 | )
|
| 367 |
|
| 368 | proc = subprocess.run(
|
| 369 | [
|
| 370 | sys.executable,
|
| 371 | str(Path(__file__).resolve().parents[1] / "scripts" / "perf_loop.py"),
|
| 372 | "dry-run",
|
| 373 | "--scorecard",
|
| 374 | str(scorecard),
|
| 375 | "--ledger",
|
| 376 | str(ledger),
|
| 377 | ],
|
| 378 | text=True,
|
| 379 | stdout=subprocess.PIPE,
|
| 380 | stderr=subprocess.PIPE,
|
| 381 | check=False,
|
| 382 | )
|
| 383 |
|
| 384 | self.assertEqual(proc.returncode, 2)
|
| 385 | self.assertIn("stop-on-first-RED", proc.stderr)
|
| 386 | self.assertFalse(ledger.exists())
|
| 387 |
|
| 388 |
|
| 389 | if __name__ == "__main__":
|
| 390 | unittest.main()
|