Viewing files at commit d3edd8c5e361Commit detailsView latest

Browse files
tests/test_parallel_hygiene.py 192 lines · 7.4 KB · python
1
"""Parallel-run hygiene: branch names, fixture builds, and latest.* copies."""
2
3
from __future__ import annotations
4
5
import importlib.util
6
import json
7
import os
8
import shutil
9
import sys
10
import tempfile
11
import unittest
12
from pathlib import Path
13
from unittest.mock import patch
14
15
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
16
17
from oakbench import results as oakbench_results  # noqa: E402
18
from oakbench.results import ResultsStore  # noqa: E402
19
from oakbench.rows import RowContractViolation  # noqa: E402
20
21
22
def load_script(name: str):
23
    path = Path(__file__).resolve().parents[1] / "scripts" / f"{name}.py"
24
    spec = importlib.util.spec_from_file_location(name, path)
25
    module = importlib.util.module_from_spec(spec)
26
    sys.modules[name] = module
27
    spec.loader.exec_module(module)
28
    return module
29
30
31
class BranchNameUniquenessTest(unittest.TestCase):
32
    def test_same_inputs_differ_by_uniquifier(self) -> None:
33
        bench = load_script("bench")
34
        shared = {
35
            "bench_id": "20260610T120000",
36
            "scenario_name": "tiny_text",
37
            "subject_name": "git",
38
            "run_index": 0,
39
        }
40
        first = bench.oak_network_branch_name(**shared, pid=1001, token="aabb")
41
        second = bench.oak_network_branch_name(**shared, pid=1002, token="aabb")
42
        third = bench.oak_network_branch_name(**shared, pid=1001, token="ccdd")
43
        self.assertNotEqual(first, second)
44
        self.assertNotEqual(first, third)
45
        self.assertTrue(first.startswith("bench-20260610T120000-tiny_text-git-r0-"))
46
47
48
class EnsureFixtureRaceTest(unittest.TestCase):
49
    def setUp(self) -> None:
50
        self.bench = load_script("bench")
51
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-parallel-fixture-"))
52
53
    def tearDown(self) -> None:
54
        shutil.rmtree(self.tmp, ignore_errors=True)
55
56
    def test_lost_race_reuses_valid_dest_and_cleans_temp(self) -> None:
57
        scenario = self.bench.Scenario("tiny_text", 3, 64, 1, False, 1)
58
        dest = self.bench.ensure_fixture(self.tmp, scenario)
59
        expected_files = sum(1 for _ in dest.rglob("*.txt"))
60
        marker_checks = {"count": 0}
61
        real_exists = Path.exists
62
63
        def exists(self: Path) -> bool:
64
            if self == dest / ".fixture-ready":
65
                marker_checks["count"] += 1
66
                if marker_checks["count"] == 1:
67
                    return False
68
            return real_exists(self)
69
70
        with patch.object(Path, "exists", exists):
71
            with patch.object(os, "rename", side_effect=OSError(17, "exists")):
72
                result = self.bench.ensure_fixture(self.tmp, scenario)
73
74
        self.assertEqual(result, dest)
75
        self.assertEqual(sum(1 for _ in dest.rglob("*.txt")), expected_files)
76
        building_dirs = list(dest.parent.glob(f"{dest.name}.building-*"))
77
        self.assertEqual(building_dirs, [])
78
79
    def test_marker_mismatch_on_lost_race_raises(self) -> None:
80
        scenario = self.bench.Scenario("race", 2, 64, 1, False, 1)
81
        dest = self.bench.fixture_path(self.tmp, scenario)
82
        dest.mkdir(parents=True)
83
        (dest / ".fixture-ready").write_text("wrong-shape\n")
84
        real_rmtree = shutil.rmtree
85
86
        def skip_dest_rmtree(path: Path, *args, **kwargs) -> None:
87
            if path == dest:
88
                return
89
            real_rmtree(path, *args, **kwargs)
90
91
        with patch.object(shutil, "rmtree", side_effect=skip_dest_rmtree):
92
            with patch.object(os, "rename", side_effect=OSError(17, "exists")):
93
                with self.assertRaises(RuntimeError) as ctx:
94
                    self.bench.ensure_fixture(self.tmp, scenario)
95
96
        self.assertIn("wrong shape marker", str(ctx.exception))
97
98
99
class ResultsStoreAtomicLatestTest(unittest.TestCase):
100
    def setUp(self) -> None:
101
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-results-atomic-"))
102
103
    def tearDown(self) -> None:
104
        shutil.rmtree(self.tmp, ignore_errors=True)
105
106
    def test_latest_jsonl_matches_raw_without_tmp_leftovers(self) -> None:
107
        store = ResultsStore(self.tmp, "core")
108
        rows = [
109
            {
110
                "bench_id": "parallel-test",
111
                "profile": "micro",
112
                "scenario": "tiny_text",
113
                "operation": "status.clean",
114
                "run": 0,
115
                "subject": "git",
116
                "subject_kind": "git",
117
                "elapsed_ms": 1.0,
118
                "returncode": 0,
119
                "command": ["git", "status"],
120
            }
121
        ]
122
        raw_path, _ = store.write("parallel-test", rows, summary_text="# summary\n")
123
        latest_path = self.tmp / "latest.jsonl"
124
        self.assertEqual(latest_path.read_text(), raw_path.read_text())
125
        tmp_leftovers = list(self.tmp.glob("*.tmp-*"))
126
        self.assertEqual(tmp_leftovers, [])
127
128
    def test_contract_violation_keeps_raw_but_never_touches_latest(self) -> None:
129
        store = ResultsStore(self.tmp, "core")
130
        valid_row = {
131
            "bench_id": "old-run",
132
            "profile": "micro",
133
            "scenario": "tiny_text",
134
            "operation": "status.clean",
135
            "run": 0,
136
            "subject": "git",
137
            "subject_kind": "git",
138
            "elapsed_ms": 1.0,
139
            "returncode": 0,
140
            "command": ["git", "status"],
141
        }
142
        store.write("old-run", [valid_row], summary_text="# old summary\n")
143
        latest_path = self.tmp / "latest.jsonl"
144
        latest_summary_path = self.tmp / "latest.summary.md"
145
        old_latest = latest_path.read_text()
146
        old_latest_summary = latest_summary_path.read_text()
147
148
        invalid_row = dict(valid_row, bench_id="new-run", returncode=None)
149
        with self.assertRaises(RowContractViolation):
150
            store.write("new-run", [invalid_row], summary_text="# new summary\n")
151
152
        # latest.* pointers still reference the previous, contract-clean run.
153
        self.assertEqual(latest_path.read_text(), old_latest)
154
        self.assertEqual(json.loads(latest_path.read_text())["bench_id"], "old-run")
155
        self.assertEqual(latest_summary_path.read_text(), old_latest_summary)
156
        # The raw bench_id file keeps the invalid rows for emitter debugging.
157
        raw_path = self.tmp / "new-run.jsonl"
158
        self.assertEqual(json.loads(raw_path.read_text()), invalid_row)
159
160
    def test_write_all_retries_partial_os_write(self) -> None:
161
        path = self.tmp / "partial-write.bin"
162
        fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o644)
163
        real_write = os.write
164
165
        def partial_write(target_fd: int, data) -> int:
166
            size = len(data)
167
            return real_write(target_fd, data[: max(1, size // 2)])
168
169
        try:
170
            with patch.object(oakbench_results.os, "write", side_effect=partial_write):
171
                oakbench_results._write_all(fd, b"abcdef")
172
        finally:
173
            os.close(fd)
174
175
        self.assertEqual(path.read_bytes(), b"abcdef")
176
177
178
class ParallelContentionPayloadPrefixTest(unittest.TestCase):
179
    def test_prefix_includes_subject_worker_level_and_run(self) -> None:
180
        contention = load_script("parallel_contention")
181
        prefixes = {
182
            contention.contention_payload_prefix("bench", subject, workers, 0)
183
            for subject in ("git", "oak_installed")
184
            for workers in (2, 8)
185
        }
186
        self.assertEqual(len(prefixes), 4)
187
        self.assertIn("workers/bench/git/w2/r0", prefixes)
188
        self.assertIn("workers/bench/oak_installed/w8/r0", prefixes)
189
190
191
if __name__ == "__main__":
192
    unittest.main()