| 1 | """Tests for the opt-in dirty-tree spectrum (--dirty-spectrum).
|
| 2 |
|
| 3 | Covers: deterministic seeded file selection in dirty_fraction, byte-identical
|
| 4 | restore, flag plumbing (argparse default off), operation-name expansion, and
|
| 5 | an integration-lite run proving the flag-off operation sequence is unchanged
|
| 6 | when the flag is on (spectrum rows are purely additive).
|
| 7 | """
|
| 8 |
|
| 9 | from __future__ import annotations
|
| 10 |
|
| 11 | import sys
|
| 12 | from pathlib import Path
|
| 13 |
|
| 14 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 15 |
|
| 16 | import hashlib
|
| 17 | import shutil
|
| 18 | import subprocess
|
| 19 | import tempfile
|
| 20 | import unittest
|
| 21 | from unittest import mock
|
| 22 |
|
| 23 | import bench
|
| 24 | from oakbench.rows import validate_rows
|
| 25 | from oakbench.subjects import Subject
|
| 26 |
|
| 27 | GIT = shutil.which("git")
|
| 28 |
|
| 29 | SPECTRUM_OPS_FULL = [
|
| 30 | "status.dirty.p01",
|
| 31 | "diff.dirty.p01",
|
| 32 | "status.dirty.p10",
|
| 33 | "diff.dirty.p10",
|
| 34 | "status.dirty.p50",
|
| 35 | "diff.dirty.p50",
|
| 36 | ]
|
| 37 |
|
| 38 |
|
| 39 | def tree_digest(root: Path) -> dict[str, str]:
|
| 40 | return {
|
| 41 | path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
|
| 42 | for path in bench.tracked_fixture_files(root)
|
| 43 | }
|
| 44 |
|
| 45 |
|
| 46 | @unittest.skipIf(GIT is None, "git not installed")
|
| 47 | class DirtyFractionTest(unittest.TestCase):
|
| 48 | def setUp(self) -> None:
|
| 49 | self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-dirty-spectrum-"))
|
| 50 | self.repo = self.tmp / "repo"
|
| 51 | self.repo.mkdir()
|
| 52 | for index in range(100):
|
| 53 | subdir = self.repo / f"group-{index // 10:03d}"
|
| 54 | subdir.mkdir(exist_ok=True)
|
| 55 | (subdir / f"file-{index:04d}.txt").write_text(f"content {index}\n")
|
| 56 | env = bench.base_env()
|
| 57 | for command in (
|
| 58 | [GIT, "init", "-q"],
|
| 59 | [GIT, "add", "-A"],
|
| 60 | [GIT, "commit", "-q", "-m", "initial"],
|
| 61 | ):
|
| 62 | subprocess.run(command, cwd=self.repo, env=env, check=True, capture_output=True)
|
| 63 |
|
| 64 | def tearDown(self) -> None:
|
| 65 | shutil.rmtree(self.tmp, ignore_errors=True)
|
| 66 |
|
| 67 | def test_fraction_counts_use_ceil(self) -> None:
|
| 68 | for fraction, expected in ((0.01, 1), (0.10, 10), (0.50, 50)):
|
| 69 | changed = bench.dirty_fraction(self.repo, fraction, seed=f"seed:{fraction}")
|
| 70 | self.assertEqual(len(changed), expected)
|
| 71 | bench.restore_dirty_fraction(self.repo, changed, seed=f"seed:{fraction}")
|
| 72 |
|
| 73 | def test_same_seed_same_file_set(self) -> None:
|
| 74 | first = bench.dirty_fraction(self.repo, 0.10, seed="stable-seed")
|
| 75 | bench.restore_dirty_fraction(self.repo, first, seed="stable-seed")
|
| 76 | second = bench.dirty_fraction(self.repo, 0.10, seed="stable-seed")
|
| 77 | bench.restore_dirty_fraction(self.repo, second, seed="stable-seed")
|
| 78 | self.assertEqual(first, second)
|
| 79 |
|
| 80 | def test_selection_is_sorted_and_tracked_only(self) -> None:
|
| 81 | changed = bench.dirty_fraction(self.repo, 0.50, seed="sorted-seed")
|
| 82 | self.assertEqual(changed, sorted(changed))
|
| 83 | self.assertFalse(any(rel.startswith(".git") for rel in changed))
|
| 84 | bench.restore_dirty_fraction(self.repo, changed, seed="sorted-seed")
|
| 85 |
|
| 86 | def test_dirty_appends_fixed_line_and_restore_is_byte_identical(self) -> None:
|
| 87 | before = tree_digest(self.repo)
|
| 88 | changed = bench.dirty_fraction(self.repo, 0.01, seed="roundtrip-seed")
|
| 89 | self.assertEqual(len(changed), 1)
|
| 90 | dirtied = self.repo / changed[0]
|
| 91 | marker = bench.dirty_spectrum_marker("roundtrip-seed")
|
| 92 | self.assertTrue(dirtied.read_bytes().endswith(marker))
|
| 93 | status = subprocess.run(
|
| 94 | [GIT, "status", "--porcelain"],
|
| 95 | cwd=self.repo,
|
| 96 | env=bench.base_env(),
|
| 97 | check=True,
|
| 98 | capture_output=True,
|
| 99 | text=True,
|
| 100 | )
|
| 101 | self.assertIn(changed[0], status.stdout)
|
| 102 | bench.restore_dirty_fraction(self.repo, changed, seed="roundtrip-seed")
|
| 103 | self.assertEqual(tree_digest(self.repo), before)
|
| 104 |
|
| 105 | def test_restore_refuses_unexpected_tail(self) -> None:
|
| 106 | changed = bench.dirty_fraction(self.repo, 0.01, seed="tamper-seed")
|
| 107 | path = self.repo / changed[0]
|
| 108 | with path.open("ab") as fh:
|
| 109 | fh.write(b"extra\n")
|
| 110 | with self.assertRaises(RuntimeError):
|
| 111 | bench.restore_dirty_fraction(self.repo, changed, seed="tamper-seed")
|
| 112 |
|
| 113 |
|
| 114 | class FlagPlumbingTest(unittest.TestCase):
|
| 115 | def test_argparse_accepts_dirty_spectrum(self) -> None:
|
| 116 | with mock.patch.object(sys, "argv", ["bench.py", "--dirty-spectrum"]):
|
| 117 | args = bench.parse_args()
|
| 118 | self.assertTrue(args.dirty_spectrum)
|
| 119 |
|
| 120 | def test_flag_defaults_off(self) -> None:
|
| 121 | with mock.patch.object(sys, "argv", ["bench.py"]):
|
| 122 | args = bench.parse_args()
|
| 123 | self.assertFalse(args.dirty_spectrum)
|
| 124 |
|
| 125 | def test_op_list_unchanged_when_off(self) -> None:
|
| 126 | self.assertEqual(bench.dirty_spectrum_operations(False), [])
|
| 127 | self.assertEqual(bench.dirty_spectrum_operations(False, skip_diff=True), [])
|
| 128 |
|
| 129 | def test_op_list_expands_when_on(self) -> None:
|
| 130 | self.assertEqual(bench.dirty_spectrum_operations(True), SPECTRUM_OPS_FULL)
|
| 131 | self.assertEqual(
|
| 132 | bench.dirty_spectrum_operations(True, skip_diff=True),
|
| 133 | ["status.dirty.p01", "status.dirty.p10", "status.dirty.p50"],
|
| 134 | )
|
| 135 |
|
| 136 | def test_run_subject_scenario_defaults_spectrum_off(self) -> None:
|
| 137 | # The flag-off path must not depend on callers passing the new
|
| 138 | # parameter: existing call sites stay byte-identical in behavior.
|
| 139 | import inspect
|
| 140 |
|
| 141 | signature = inspect.signature(bench.run_subject_scenario)
|
| 142 | self.assertIs(signature.parameters["dirty_spectrum"].default, False)
|
| 143 |
|
| 144 |
|
| 145 | @unittest.skipIf(GIT is None, "git not installed")
|
| 146 | class DirtySpectrumIntegrationTest(unittest.TestCase):
|
| 147 | """Integration-lite: drive run_subject_scenario on a tiny git fixture."""
|
| 148 |
|
| 149 | def setUp(self) -> None:
|
| 150 | self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-spectrum-lane-"))
|
| 151 |
|
| 152 | def tearDown(self) -> None:
|
| 153 | shutil.rmtree(self.tmp, ignore_errors=True)
|
| 154 |
|
| 155 | def run_scenario(self, dirty_spectrum: bool) -> list[dict]:
|
| 156 | scenario = bench.Scenario("tiny_text", 8, 256, 2, False, 1)
|
| 157 | fixture = bench.ensure_fixture(self.tmp, scenario)
|
| 158 | metadata = {
|
| 159 | "bench_id": "test",
|
| 160 | "profile": "micro",
|
| 161 | "benchmark_track": "agent-default",
|
| 162 | "command_semantics_version": bench.COMMAND_SEMANTICS_VERSION,
|
| 163 | "timestamp_utc": "test",
|
| 164 | }
|
| 165 | return bench.run_subject_scenario(
|
| 166 | Subject(name="git", kind="git", label="Git", bin=Path(GIT)),
|
| 167 | scenario,
|
| 168 | fixture,
|
| 169 | run_index=0,
|
| 170 | run_root=self.tmp / ("runs-on" if dirty_spectrum else "runs-off"),
|
| 171 | metadata=metadata,
|
| 172 | skip_diff=False,
|
| 173 | admitted_output_chars=20_000,
|
| 174 | track="agent-default",
|
| 175 | skip_determinism_probe=True,
|
| 176 | skip_remote=True,
|
| 177 | dirty_spectrum=dirty_spectrum,
|
| 178 | )
|
| 179 |
|
| 180 | def test_spectrum_rows_additive_and_flag_off_sequence_unchanged(self) -> None:
|
| 181 | rows_off = self.run_scenario(dirty_spectrum=False)
|
| 182 | rows_on = self.run_scenario(dirty_spectrum=True)
|
| 183 |
|
| 184 | ops_off = [row["operation"] for row in rows_off]
|
| 185 | ops_on = [row["operation"] for row in rows_on]
|
| 186 |
|
| 187 | # Flag off: no spectrum operation names anywhere.
|
| 188 | for op in SPECTRUM_OPS_FULL:
|
| 189 | self.assertNotIn(op, ops_off)
|
| 190 |
|
| 191 | # Flag on: every spectrum row present, valid, and successful.
|
| 192 | for op in SPECTRUM_OPS_FULL:
|
| 193 | self.assertIn(op, ops_on)
|
| 194 | self.assertEqual(validate_rows(rows_on, "core"), [])
|
| 195 | failures = [row for row in rows_on if row["returncode"] != 0]
|
| 196 | self.assertEqual(failures, [])
|
| 197 |
|
| 198 | # Purely additive: removing spectrum rows reproduces the flag-off
|
| 199 | # operation sequence exactly, in order.
|
| 200 | self.assertEqual([op for op in ops_on if op not in SPECTRUM_OPS_FULL], ops_off)
|
| 201 |
|
| 202 | # Scale annotations: 8 tracked files -> ceil gives 1, 1, 4.
|
| 203 | by_op = {row["operation"]: row for row in rows_on}
|
| 204 | self.assertEqual(by_op["status.dirty.p01"]["dirty_file_count"], 1)
|
| 205 | self.assertEqual(by_op["status.dirty.p10"]["dirty_file_count"], 1)
|
| 206 | self.assertEqual(by_op["status.dirty.p50"]["dirty_file_count"], 4)
|
| 207 | self.assertEqual(by_op["diff.dirty.p50"]["dirty_fraction"], 0.50)
|
| 208 |
|
| 209 | # Spectrum status rows must observe exactly the dirtied files: the
|
| 210 | # measured command runs on the dirty tree, after untimed setup.
|
| 211 | status_row = by_op["status.dirty.p50"]
|
| 212 | self.assertEqual(status_row["command"], by_op["status.dirty"]["command"])
|
| 213 |
|
| 214 |
|
| 215 | if __name__ == "__main__":
|
| 216 | unittest.main()
|