Viewing files at commit f2acb751aa96Commit detailsView latest

Browse files
tests/test_honest_measurement.py 210 lines · 12.5 KB · python
1
"""P1 honest measurement: Git shim resolution, rusage CPU/IO, cache labels,
2
per-row provenance, and the Scalar-tuned Git subject — from real producer rows."""
3
from __future__ import annotations
4
5
import argparse
6
import os
7
import platform
8
import shutil
9
import subprocess
10
import sys
11
import tempfile
12
import unittest
13
from pathlib import Path
14
from unittest import mock
15
16
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
17
18
import bench  # noqa: E402
19
from oakbench import subjects as oakbench_subjects  # noqa: E402
20
from oakbench.execution import run_timed  # noqa: E402
21
from oakbench.rows import validate_rows  # noqa: E402
22
23
GIT = shutil.which("git")
24
25
26
class GitShimTest(unittest.TestCase):
27
    def test_shim_is_detected_only_at_the_darwin_stub_path(self):
28
        self.assertTrue(oakbench_subjects.is_xcrun_git_shim("/usr/bin/git", "Darwin"))
29
        self.assertFalse(oakbench_subjects.is_xcrun_git_shim("/usr/bin/git", "Linux"))
30
        self.assertFalse(oakbench_subjects.is_xcrun_git_shim("/opt/homebrew/bin/git", "Darwin"))
31
32
    def test_shim_is_resolved_and_recorded(self):
33
        real = Path("/real/xcode/git")
34
        path, requested, measured = oakbench_subjects.resolve_git_subject_bin(
35
            Path("/usr/bin/git"), allow_shim=False, system="Darwin", resolver=lambda: real)
36
        self.assertEqual((path, requested, measured), (real, Path("/usr/bin/git"), False))
37
38
    def test_unresolvable_shim_is_refused_unless_allowed(self):
39
        with self.assertRaisesRegex(SystemExit, "xcrun git shim"):
40
            oakbench_subjects.resolve_git_subject_bin(Path("/usr/bin/git"), allow_shim=False,
41
                                                      system="Darwin", resolver=lambda: None)
42
        path, requested, measured = oakbench_subjects.resolve_git_subject_bin(
43
            Path("/usr/bin/git"), allow_shim=True, system="Darwin", resolver=lambda: None)
44
        self.assertEqual((path, measured), (Path("/usr/bin/git"), True))
45
46
    @unittest.skipUnless(platform.system() == "Darwin" and Path("/usr/bin/git").exists(), "macOS xcrun shim required")
47
    def test_real_host_resolution_differs_from_shim_bytes(self):
48
        real = oakbench_subjects.resolve_real_git()
49
        if real is None:
50
            self.skipTest("no developer tools selected")
51
        self.assertNotEqual(str(real), "/usr/bin/git")
52
        self.assertNotEqual(oakbench_subjects.binary_sha256(real), oakbench_subjects.binary_sha256(Path("/usr/bin/git")))
53
        version = subprocess.run([str(real), "--version"], capture_output=True, text=True, check=True).stdout
54
        self.assertIn("git version", version)
55
56
    @unittest.skipUnless(platform.system() == "Darwin" and Path("/usr/bin/git").exists(), "macOS xcrun shim required")
57
    def test_load_subjects_never_times_the_shim_by_default(self):
58
        with tempfile.TemporaryDirectory() as tmp:
59
            config = Path(tmp) / "subjects.toml"
60
            config.write_text('[subjects.git]\nenabled = true\nkind = "git"\nlabel = "Git"\nbin = "/usr/bin/git"\n')
61
            args = argparse.Namespace(config=config, subjects="git", git_bin=None, oak_installed_bin=None,
62
                                      oak_local_bin=None, allow_git_shim=False)
63
            with mock.patch.dict(os.environ, {oakbench_subjects.ALLOW_GIT_SHIM_ENV: ""}):
64
                if oakbench_subjects.resolve_real_git() is None:
65
                    self.skipTest("no developer tools selected")
66
                subject = oakbench_subjects.load_subjects(args)[0]
67
            self.assertNotEqual(str(subject.bin), "/usr/bin/git")
68
            self.assertEqual(subject.requested_bin, Path("/usr/bin/git"))
69
            details = oakbench_subjects.subject_details([subject])["git"]
70
            self.assertTrue(details["git_shim_resolved"])
71
            self.assertFalse(details["git_shim_measured"])
72
            self.assertEqual(details["git_config_profile"], "stock")
73
74
75
class RusageTest(unittest.TestCase):
76
    def test_cpu_and_io_are_measured_for_a_real_child(self):
77
        capture = run_timed([sys.executable, "-c", "sum(i*i for i in range(300000))"], Path.cwd(), dict(os.environ), 100)
78
        self.assertIsNotNone(capture.cpu_user_ms)
79
        self.assertGreater(capture.cpu_user_ms + capture.cpu_sys_ms, 0.0)
80
        self.assertIsInstance(capture.io_read_blocks, int)
81
        self.assertIsInstance(capture.io_write_blocks, int)
82
83
84
@unittest.skipUnless(GIT, "Git required")
85
class CoreRowContractTest(unittest.TestCase):
86
    """Real bench.py producer rows carry the P1 fields and still validate."""
87
88
    def run_scenario(self, subject, metadata=None, skip_determinism_probe=False, name="p1_contract"):
89
        root = Path(tempfile.mkdtemp(prefix="oakbench-p1-"))
90
        self.addCleanup(shutil.rmtree, root, True)
91
        scenario = bench.Scenario(name, 3, 64, 2, False, 1)
92
        fixture = bench.ensure_fixture(root / "fixtures", scenario)
93
        meta = {"bench_id": "p1-test", "profile": "micro", "subject_details": oakbench_subjects.subject_details([subject]), "cache_mode_requested": "warm",
94
                "timing_contract_version": bench.TIMING_CONTRACT_VERSION, **bench.fixture_fingerprint(fixture),
95
                **(metadata or {})}
96
        return bench.run_subject_scenario(subject, scenario, fixture, 0, root / "runs", meta, False, 10000,
97
                                          "core-equivalent", skip_determinism_probe=skip_determinism_probe,
98
                                          skip_remote=True)
99
100
    def test_timed_rows_carry_cache_rusage_and_provenance(self):
101
        subject = bench.Subject("git", "git", "Git", Path(GIT))
102
        rows = self.run_scenario(subject)
103
        self.assertEqual(validate_rows(rows, "core"), [])
104
        timed = [row for row in rows if row["operation"] in bench.MAIN_TIMED_OPS]
105
        self.assertTrue(timed)
106
        for row in timed:
107
            self.assertEqual(row["page_cache_state"], "warm", row["operation"])
108
            self.assertEqual(row["cache_state"], "warm")
109
            self.assertIsNotNone(row["cpu_user_ms"])
110
            self.assertIsInstance(row["io_read_blocks"], int)
111
            self.assertEqual(row["rusage_source"], "os_wait4_child_rusage")
112
            self.assertEqual(row["subject_bin"], GIT)
113
            self.assertRegex(row["binary_sha256"], "^[0-9a-f]{64}$")
114
            self.assertIn("git version", row["subject_version"])
115
            self.assertRegex(row["fixture_sha256"], "^[0-9a-f]{64}$")
116
            self.assertEqual(row["fixture_stats"]["files"], 3)
117
            self.assertEqual(row["git_config_profile"], "stock")
118
        snapshot = next(row for row in rows if row["operation"] == "snapshot.dirty.commit")
119
        self.assertEqual(snapshot["harness_reads_before_op"], ["repository_state_fingerprint", "payload_sample_hashes"])
120
        replay = next(row for row in rows if row["operation"] == "diff.dirty")
121
        self.assertEqual(replay["cache_state"], "hash_warmed_isolated_dirty_replay")  # caller label preserved
122
        self.assertEqual(replay["page_cache_state"], "warm")
123
124
    def test_cold_mode_purges_after_harness_reads_or_labels_unknown(self):
125
        subject = bench.Subject("git", "git", "Git", Path(GIT))
126
        events = []
127
        def fake_purge():
128
            events.append("purge")
129
            return {"attempted": True, "ok": True, "method": "test_purge", "error": None}
130
        original_fingerprint = bench.oakbench_integrity.repository_state_fingerprint
131
        def fingerprint(repo):
132
            events.append("fingerprint")
133
            return original_fingerprint(repo)
134
        original_run_timed = bench.run_timed
135
        def timed(*args, **kwargs):
136
            events.append("timed")
137
            return original_run_timed(*args, **kwargs)
138
        with mock.patch.object(bench.oakbench_cachectl, "purge_fs_caches", side_effect=fake_purge), \
139
             mock.patch.object(bench.oakbench_integrity, "repository_state_fingerprint", side_effect=fingerprint), \
140
             mock.patch.object(bench, "run_timed", side_effect=timed):
141
            # Diagnostic probes are untimed (null elapsed) and exempt; skip them here.
142
            rows = self.run_scenario(subject, {"cache_mode_requested": "cold"}, skip_determinism_probe=True)
143
        commit = next(row for row in rows if row["operation"] == "snapshot.dirty.commit")
144
        self.assertEqual((commit["page_cache_state"], commit["cache_state"]), ("purged", "cold"))
145
        # Every timed command is IMMEDIATELY preceded by a purge, so no harness
146
        # read (fingerprint/payload hash) sits between the purge and the clock.
147
        self.assertIn("fingerprint", events)
148
        for index, event in enumerate(events):
149
            if event == "timed":
150
                self.assertEqual(events[index - 1], "purge", events[max(0, index - 3):index + 1])
151
        with mock.patch.object(bench.oakbench_cachectl, "purge_fs_caches",
152
                               return_value={"attempted": True, "ok": False, "method": "x", "error": "EPERM"}):
153
            rows = self.run_scenario(subject, {"cache_mode_requested": "cold"})
154
        status = next(row for row in rows if row["operation"] == "status.clean")
155
        self.assertEqual((status["page_cache_state"], status["cache_state"]), ("unknown", "unknown"))
156
        self.assertIn("purge_failed", status["cache_state_reason"])
157
158
    @unittest.skipUnless(GIT and subprocess.run([GIT, "fsmonitor--daemon", "--help"], capture_output=True).returncode in (0, 129),
159
                         "Git fsmonitor required")
160
    def test_scalar_tuned_subject_is_labelled_and_configured(self):
161
        stock = bench.Subject("git", "git", "Git", Path(GIT))
162
        tuned = bench.expand_git_modes([stock], "scalar")[1]
163
        self.assertEqual((tuned.name, tuned.git_mode), ("git_scalar", "scalar"))
164
        # Scalar targets large trees; on tiny fixtures it is not a stronger baseline.
165
        applies, _ = bench.git_mode_applies_to_scenario("scalar", bench.Scenario("tiny_text", 1, 1, 1, False, 1))
166
        self.assertFalse(applies)
167
        applies, _ = bench.git_mode_applies_to_scenario("scalar", bench.Scenario("many_small_files", 1, 1, 1, False, 1))
168
        self.assertTrue(applies)
169
        rows = self.run_scenario(tuned, name="many_small_files")
170
        # Daemon start and cache warm-up are recorded setup, before any timed op.
171
        ops = [row["operation"] for row in rows]
172
        self.assertLess(ops.index("mode.setup.scalar_warm_status"), ops.index("snapshot.initial"))
173
        setup = [row["operation"] for row in rows if row["operation"].startswith("mode.setup.scalar_")]
174
        self.assertEqual(len(setup), len(bench.WIRED_GIT_MODES["scalar"]))
175
        self.assertTrue(all(row["returncode"] == 0 for row in rows if row["operation"].startswith("mode.setup.")))
176
        status = next(row for row in rows if row["operation"] == "status.clean")
177
        self.assertEqual(status["git_config_profile"], "scalar_recommended")
178
        self.assertEqual(validate_rows(rows, "core"), [])
179
180
    def test_subject_exit_77_is_a_recorded_failure_not_a_harness_skip(self):
181
        # QA P1 A7: a subject returning the harness sentinel used to crash the
182
        # row contract; it is a subject failure with its real code preserved.
183
        root = Path(tempfile.mkdtemp(prefix="oakbench-p1-77-"))
184
        self.addCleanup(shutil.rmtree, root, True)
185
        stub = root / "git77"
186
        stub.write_text(f'#!/bin/sh\n[ "$1" = status ] && exit 77\nexec {GIT} "$@"\n')
187
        stub.chmod(0o755)
188
        rows = self.run_scenario(bench.Subject("git", "git", "Git", stub))
189
        status = next(row for row in rows if row["operation"] == "status.clean")
190
        self.assertEqual((status["returncode"], status["process_returncode"]), (1, 77))
191
        self.assertFalse(status.get("skipped", False))
192
        self.assertTrue(all(row.get("skipped") for row in rows[rows.index(status) + 1:]))
193
        self.assertEqual(validate_rows(rows, "core"), [])
194
195
    def test_summary_compares_oak_with_the_strongest_git(self):
196
        subjects = [bench.Subject("git", "git", "Git", Path("/g")),
197
                    bench.Subject("git_scalar", "git", "Git scalar", Path("/g"), git_mode="scalar"),
198
                    bench.Subject("oak_installed", "oak", "Oak", Path("/o"))]
199
        by_key = {("git", "s/status.dirty"): 5.0, ("git_scalar", "s/status.dirty"): 11.0,
200
                  ("oak_installed", "s/status.dirty"): 7.0,
201
                  ("git", "s/snapshot.dirty"): 20.0, ("git_scalar", "s/snapshot.dirty"): 12.0,
202
                  ("oak_installed", "s/snapshot.dirty"): 14.0}
203
        best = bench.strongest_git_baseline(by_key, subjects)
204
        self.assertEqual(best["s/status.dirty"], ("git", 5.0))       # tuned-but-slower never flatters Oak
205
        self.assertEqual(best["s/snapshot.dirty"], ("git_scalar", 12.0))
206
        self.assertIn("strongest Git", "\n".join(bench.strongest_git_lines(by_key, subjects)))
207
208
209
if __name__ == "__main__":
210
    unittest.main()