Viewing files at commit 46ad54da0178Commit detailsView latest

Browse files
tests/test_dirty_spectrum.py 216 lines · 8.5 KB · python
1
"""Tests for the opt-in dirty-tree spectrum (--dirty-spectrum).
2
3
Covers: deterministic seeded file selection in dirty_fraction, byte-identical
4
restore, flag plumbing (argparse default off), operation-name expansion, and
5
an integration-lite run proving the flag-off operation sequence is unchanged
6
when the flag is on (spectrum rows are purely additive).
7
"""
8
9
from __future__ import annotations
10
11
import sys
12
from pathlib import Path
13
14
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
15
16
import hashlib
17
import shutil
18
import subprocess
19
import tempfile
20
import unittest
21
from unittest import mock
22
23
import bench
24
from oakbench.rows import validate_rows
25
from oakbench.subjects import Subject
26
27
GIT = shutil.which("git")
28
29
SPECTRUM_OPS_FULL = [
30
    "status.dirty.p01",
31
    "diff.dirty.p01",
32
    "status.dirty.p10",
33
    "diff.dirty.p10",
34
    "status.dirty.p50",
35
    "diff.dirty.p50",
36
]
37
38
39
def tree_digest(root: Path) -> dict[str, str]:
40
    return {
41
        path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
42
        for path in bench.tracked_fixture_files(root)
43
    }
44
45
46
@unittest.skipIf(GIT is None, "git not installed")
47
class DirtyFractionTest(unittest.TestCase):
48
    def setUp(self) -> None:
49
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-dirty-spectrum-"))
50
        self.repo = self.tmp / "repo"
51
        self.repo.mkdir()
52
        for index in range(100):
53
            subdir = self.repo / f"group-{index // 10:03d}"
54
            subdir.mkdir(exist_ok=True)
55
            (subdir / f"file-{index:04d}.txt").write_text(f"content {index}\n")
56
        env = bench.base_env()
57
        for command in (
58
            [GIT, "init", "-q"],
59
            [GIT, "add", "-A"],
60
            [GIT, "commit", "-q", "-m", "initial"],
61
        ):
62
            subprocess.run(command, cwd=self.repo, env=env, check=True, capture_output=True)
63
64
    def tearDown(self) -> None:
65
        shutil.rmtree(self.tmp, ignore_errors=True)
66
67
    def test_fraction_counts_use_ceil(self) -> None:
68
        for fraction, expected in ((0.01, 1), (0.10, 10), (0.50, 50)):
69
            changed = bench.dirty_fraction(self.repo, fraction, seed=f"seed:{fraction}")
70
            self.assertEqual(len(changed), expected)
71
            bench.restore_dirty_fraction(self.repo, changed, seed=f"seed:{fraction}")
72
73
    def test_same_seed_same_file_set(self) -> None:
74
        first = bench.dirty_fraction(self.repo, 0.10, seed="stable-seed")
75
        bench.restore_dirty_fraction(self.repo, first, seed="stable-seed")
76
        second = bench.dirty_fraction(self.repo, 0.10, seed="stable-seed")
77
        bench.restore_dirty_fraction(self.repo, second, seed="stable-seed")
78
        self.assertEqual(first, second)
79
80
    def test_selection_is_sorted_and_tracked_only(self) -> None:
81
        changed = bench.dirty_fraction(self.repo, 0.50, seed="sorted-seed")
82
        self.assertEqual(changed, sorted(changed))
83
        self.assertFalse(any(rel.startswith(".git") for rel in changed))
84
        bench.restore_dirty_fraction(self.repo, changed, seed="sorted-seed")
85
86
    def test_dirty_appends_fixed_line_and_restore_is_byte_identical(self) -> None:
87
        before = tree_digest(self.repo)
88
        changed = bench.dirty_fraction(self.repo, 0.01, seed="roundtrip-seed")
89
        self.assertEqual(len(changed), 1)
90
        dirtied = self.repo / changed[0]
91
        marker = bench.dirty_spectrum_marker("roundtrip-seed")
92
        self.assertTrue(dirtied.read_bytes().endswith(marker))
93
        status = subprocess.run(
94
            [GIT, "status", "--porcelain"],
95
            cwd=self.repo,
96
            env=bench.base_env(),
97
            check=True,
98
            capture_output=True,
99
            text=True,
100
        )
101
        self.assertIn(changed[0], status.stdout)
102
        bench.restore_dirty_fraction(self.repo, changed, seed="roundtrip-seed")
103
        self.assertEqual(tree_digest(self.repo), before)
104
105
    def test_restore_refuses_unexpected_tail(self) -> None:
106
        changed = bench.dirty_fraction(self.repo, 0.01, seed="tamper-seed")
107
        path = self.repo / changed[0]
108
        with path.open("ab") as fh:
109
            fh.write(b"extra\n")
110
        with self.assertRaises(RuntimeError):
111
            bench.restore_dirty_fraction(self.repo, changed, seed="tamper-seed")
112
113
114
class FlagPlumbingTest(unittest.TestCase):
115
    def test_argparse_accepts_dirty_spectrum(self) -> None:
116
        with mock.patch.object(sys, "argv", ["bench.py", "--dirty-spectrum"]):
117
            args = bench.parse_args()
118
        self.assertTrue(args.dirty_spectrum)
119
120
    def test_flag_defaults_off(self) -> None:
121
        with mock.patch.object(sys, "argv", ["bench.py"]):
122
            args = bench.parse_args()
123
        self.assertFalse(args.dirty_spectrum)
124
125
    def test_op_list_unchanged_when_off(self) -> None:
126
        self.assertEqual(bench.dirty_spectrum_operations(False), [])
127
        self.assertEqual(bench.dirty_spectrum_operations(False, skip_diff=True), [])
128
129
    def test_op_list_expands_when_on(self) -> None:
130
        self.assertEqual(bench.dirty_spectrum_operations(True), SPECTRUM_OPS_FULL)
131
        self.assertEqual(
132
            bench.dirty_spectrum_operations(True, skip_diff=True),
133
            ["status.dirty.p01", "status.dirty.p10", "status.dirty.p50"],
134
        )
135
136
    def test_run_subject_scenario_defaults_spectrum_off(self) -> None:
137
        # The flag-off path must not depend on callers passing the new
138
        # parameter: existing call sites stay byte-identical in behavior.
139
        import inspect
140
141
        signature = inspect.signature(bench.run_subject_scenario)
142
        self.assertIs(signature.parameters["dirty_spectrum"].default, False)
143
144
145
@unittest.skipIf(GIT is None, "git not installed")
146
class DirtySpectrumIntegrationTest(unittest.TestCase):
147
    """Integration-lite: drive run_subject_scenario on a tiny git fixture."""
148
149
    def setUp(self) -> None:
150
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-spectrum-lane-"))
151
152
    def tearDown(self) -> None:
153
        shutil.rmtree(self.tmp, ignore_errors=True)
154
155
    def run_scenario(self, dirty_spectrum: bool) -> list[dict]:
156
        scenario = bench.Scenario("tiny_text", 8, 256, 2, False, 1)
157
        fixture = bench.ensure_fixture(self.tmp, scenario)
158
        metadata = {
159
            "bench_id": "test",
160
            "profile": "micro",
161
            "benchmark_track": "agent-default",
162
            "command_semantics_version": bench.COMMAND_SEMANTICS_VERSION,
163
            "timestamp_utc": "test",
164
        }
165
        return bench.run_subject_scenario(
166
            Subject(name="git", kind="git", label="Git", bin=Path(GIT)),
167
            scenario,
168
            fixture,
169
            run_index=0,
170
            run_root=self.tmp / ("runs-on" if dirty_spectrum else "runs-off"),
171
            metadata=metadata,
172
            skip_diff=False,
173
            admitted_output_chars=20_000,
174
            track="agent-default",
175
            skip_determinism_probe=True,
176
            skip_remote=True,
177
            dirty_spectrum=dirty_spectrum,
178
        )
179
180
    def test_spectrum_rows_additive_and_flag_off_sequence_unchanged(self) -> None:
181
        rows_off = self.run_scenario(dirty_spectrum=False)
182
        rows_on = self.run_scenario(dirty_spectrum=True)
183
184
        ops_off = [row["operation"] for row in rows_off]
185
        ops_on = [row["operation"] for row in rows_on]
186
187
        # Flag off: no spectrum operation names anywhere.
188
        for op in SPECTRUM_OPS_FULL:
189
            self.assertNotIn(op, ops_off)
190
191
        # Flag on: every spectrum row present, valid, and successful.
192
        for op in SPECTRUM_OPS_FULL:
193
            self.assertIn(op, ops_on)
194
        self.assertEqual(validate_rows(rows_on, "core"), [])
195
        failures = [row for row in rows_on if row["returncode"] != 0]
196
        self.assertEqual(failures, [])
197
198
        # Purely additive: removing spectrum rows reproduces the flag-off
199
        # operation sequence exactly, in order.
200
        self.assertEqual([op for op in ops_on if op not in SPECTRUM_OPS_FULL], ops_off)
201
202
        # Scale annotations: 8 tracked files -> ceil gives 1, 1, 4.
203
        by_op = {row["operation"]: row for row in rows_on}
204
        self.assertEqual(by_op["status.dirty.p01"]["dirty_file_count"], 1)
205
        self.assertEqual(by_op["status.dirty.p10"]["dirty_file_count"], 1)
206
        self.assertEqual(by_op["status.dirty.p50"]["dirty_file_count"], 4)
207
        self.assertEqual(by_op["diff.dirty.p50"]["dirty_fraction"], 0.50)
208
209
        # Spectrum status rows must observe exactly the dirtied files: the
210
        # measured command runs on the dirty tree, after untimed setup.
211
        status_row = by_op["status.dirty.p50"]
212
        self.assertEqual(status_row["command"], by_op["status.dirty"]["command"])
213
214
215
if __name__ == "__main__":
216
    unittest.main()