| 1 | """Parallel-run hygiene: branch names, fixture builds, and latest.* copies."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import importlib.util
|
| 6 | import json
|
| 7 | import os
|
| 8 | import shutil
|
| 9 | import sys
|
| 10 | import tempfile
|
| 11 | import unittest
|
| 12 | from pathlib import Path
|
| 13 | from unittest.mock import patch
|
| 14 |
|
| 15 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 16 |
|
| 17 | from oakbench import results as oakbench_results # noqa: E402
|
| 18 | from oakbench.results import ResultsStore # noqa: E402
|
| 19 | from oakbench.rows import RowContractViolation # noqa: E402
|
| 20 |
|
| 21 |
|
| 22 | def load_script(name: str):
|
| 23 | path = Path(__file__).resolve().parents[1] / "scripts" / f"{name}.py"
|
| 24 | spec = importlib.util.spec_from_file_location(name, path)
|
| 25 | module = importlib.util.module_from_spec(spec)
|
| 26 | sys.modules[name] = module
|
| 27 | spec.loader.exec_module(module)
|
| 28 | return module
|
| 29 |
|
| 30 |
|
| 31 | class BranchNameUniquenessTest(unittest.TestCase):
|
| 32 | def test_same_inputs_differ_by_uniquifier(self) -> None:
|
| 33 | bench = load_script("bench")
|
| 34 | shared = {
|
| 35 | "bench_id": "20260610T120000",
|
| 36 | "scenario_name": "tiny_text",
|
| 37 | "subject_name": "git",
|
| 38 | "run_index": 0,
|
| 39 | }
|
| 40 | first = bench.oak_network_branch_name(**shared, pid=1001, token="aabb")
|
| 41 | second = bench.oak_network_branch_name(**shared, pid=1002, token="aabb")
|
| 42 | third = bench.oak_network_branch_name(**shared, pid=1001, token="ccdd")
|
| 43 | self.assertNotEqual(first, second)
|
| 44 | self.assertNotEqual(first, third)
|
| 45 | self.assertTrue(first.startswith("bench-20260610T120000-tiny_text-git-r0-"))
|
| 46 |
|
| 47 |
|
| 48 | class EnsureFixtureRaceTest(unittest.TestCase):
|
| 49 | def setUp(self) -> None:
|
| 50 | self.bench = load_script("bench")
|
| 51 | self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-parallel-fixture-"))
|
| 52 |
|
| 53 | def tearDown(self) -> None:
|
| 54 | shutil.rmtree(self.tmp, ignore_errors=True)
|
| 55 |
|
| 56 | def test_lost_race_reuses_valid_dest_and_cleans_temp(self) -> None:
|
| 57 | scenario = self.bench.Scenario("tiny_text", 3, 64, 1, False, 1)
|
| 58 | dest = self.bench.ensure_fixture(self.tmp, scenario)
|
| 59 | expected_files = sum(1 for _ in dest.rglob("*.txt"))
|
| 60 | marker_checks = {"count": 0}
|
| 61 | real_exists = Path.exists
|
| 62 |
|
| 63 | def exists(self: Path) -> bool:
|
| 64 | if self == dest / ".fixture-ready":
|
| 65 | marker_checks["count"] += 1
|
| 66 | if marker_checks["count"] == 1:
|
| 67 | return False
|
| 68 | return real_exists(self)
|
| 69 |
|
| 70 | with patch.object(Path, "exists", exists):
|
| 71 | with patch.object(os, "rename", side_effect=OSError(17, "exists")):
|
| 72 | result = self.bench.ensure_fixture(self.tmp, scenario)
|
| 73 |
|
| 74 | self.assertEqual(result, dest)
|
| 75 | self.assertEqual(sum(1 for _ in dest.rglob("*.txt")), expected_files)
|
| 76 | building_dirs = list(dest.parent.glob(f"{dest.name}.building-*"))
|
| 77 | self.assertEqual(building_dirs, [])
|
| 78 |
|
| 79 | def test_marker_mismatch_on_lost_race_raises(self) -> None:
|
| 80 | scenario = self.bench.Scenario("race", 2, 64, 1, False, 1)
|
| 81 | dest = self.bench.fixture_path(self.tmp, scenario)
|
| 82 | dest.mkdir(parents=True)
|
| 83 | (dest / ".fixture-ready").write_text("wrong-shape\n")
|
| 84 | real_rmtree = shutil.rmtree
|
| 85 |
|
| 86 | def skip_dest_rmtree(path: Path, *args, **kwargs) -> None:
|
| 87 | if path == dest:
|
| 88 | return
|
| 89 | real_rmtree(path, *args, **kwargs)
|
| 90 |
|
| 91 | with patch.object(shutil, "rmtree", side_effect=skip_dest_rmtree):
|
| 92 | with patch.object(os, "rename", side_effect=OSError(17, "exists")):
|
| 93 | with self.assertRaises(RuntimeError) as ctx:
|
| 94 | self.bench.ensure_fixture(self.tmp, scenario)
|
| 95 |
|
| 96 | self.assertIn("wrong shape marker", str(ctx.exception))
|
| 97 |
|
| 98 |
|
| 99 | class ResultsStoreAtomicLatestTest(unittest.TestCase):
|
| 100 | def setUp(self) -> None:
|
| 101 | self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-results-atomic-"))
|
| 102 |
|
| 103 | def tearDown(self) -> None:
|
| 104 | shutil.rmtree(self.tmp, ignore_errors=True)
|
| 105 |
|
| 106 | def test_latest_jsonl_matches_raw_without_tmp_leftovers(self) -> None:
|
| 107 | store = ResultsStore(self.tmp, "core")
|
| 108 | rows = [
|
| 109 | {
|
| 110 | "bench_id": "parallel-test",
|
| 111 | "profile": "micro",
|
| 112 | "scenario": "tiny_text",
|
| 113 | "operation": "status.clean",
|
| 114 | "run": 0,
|
| 115 | "subject": "git",
|
| 116 | "subject_kind": "git",
|
| 117 | "elapsed_ms": 1.0,
|
| 118 | "returncode": 0,
|
| 119 | "command": ["git", "status"],
|
| 120 | }
|
| 121 | ]
|
| 122 | raw_path, _ = store.write("parallel-test", rows, summary_text="# summary\n")
|
| 123 | latest_path = self.tmp / "latest.jsonl"
|
| 124 | self.assertEqual(latest_path.read_text(), raw_path.read_text())
|
| 125 | tmp_leftovers = list(self.tmp.glob("*.tmp-*"))
|
| 126 | self.assertEqual(tmp_leftovers, [])
|
| 127 |
|
| 128 | def test_contract_violation_keeps_raw_but_never_touches_latest(self) -> None:
|
| 129 | store = ResultsStore(self.tmp, "core")
|
| 130 | valid_row = {
|
| 131 | "bench_id": "old-run",
|
| 132 | "profile": "micro",
|
| 133 | "scenario": "tiny_text",
|
| 134 | "operation": "status.clean",
|
| 135 | "run": 0,
|
| 136 | "subject": "git",
|
| 137 | "subject_kind": "git",
|
| 138 | "elapsed_ms": 1.0,
|
| 139 | "returncode": 0,
|
| 140 | "command": ["git", "status"],
|
| 141 | }
|
| 142 | store.write("old-run", [valid_row], summary_text="# old summary\n")
|
| 143 | latest_path = self.tmp / "latest.jsonl"
|
| 144 | latest_summary_path = self.tmp / "latest.summary.md"
|
| 145 | old_latest = latest_path.read_text()
|
| 146 | old_latest_summary = latest_summary_path.read_text()
|
| 147 |
|
| 148 | invalid_row = dict(valid_row, bench_id="new-run", returncode=None)
|
| 149 | with self.assertRaises(RowContractViolation):
|
| 150 | store.write("new-run", [invalid_row], summary_text="# new summary\n")
|
| 151 |
|
| 152 | # latest.* pointers still reference the previous, contract-clean run.
|
| 153 | self.assertEqual(latest_path.read_text(), old_latest)
|
| 154 | self.assertEqual(json.loads(latest_path.read_text())["bench_id"], "old-run")
|
| 155 | self.assertEqual(latest_summary_path.read_text(), old_latest_summary)
|
| 156 | # The raw bench_id file keeps the invalid rows for emitter debugging.
|
| 157 | raw_path = self.tmp / "new-run.jsonl"
|
| 158 | self.assertEqual(json.loads(raw_path.read_text()), invalid_row)
|
| 159 |
|
| 160 | def test_write_all_retries_partial_os_write(self) -> None:
|
| 161 | path = self.tmp / "partial-write.bin"
|
| 162 | fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o644)
|
| 163 | real_write = os.write
|
| 164 |
|
| 165 | def partial_write(target_fd: int, data) -> int:
|
| 166 | size = len(data)
|
| 167 | return real_write(target_fd, data[: max(1, size // 2)])
|
| 168 |
|
| 169 | try:
|
| 170 | with patch.object(oakbench_results.os, "write", side_effect=partial_write):
|
| 171 | oakbench_results._write_all(fd, b"abcdef")
|
| 172 | finally:
|
| 173 | os.close(fd)
|
| 174 |
|
| 175 | self.assertEqual(path.read_bytes(), b"abcdef")
|
| 176 |
|
| 177 |
|
| 178 | class ParallelContentionPayloadPrefixTest(unittest.TestCase):
|
| 179 | def test_prefix_includes_subject_worker_level_and_run(self) -> None:
|
| 180 | contention = load_script("parallel_contention")
|
| 181 | prefixes = {
|
| 182 | contention.contention_payload_prefix("bench", subject, workers, 0)
|
| 183 | for subject in ("git", "oak_installed")
|
| 184 | for workers in (2, 8)
|
| 185 | }
|
| 186 | self.assertEqual(len(prefixes), 4)
|
| 187 | self.assertIn("workers/bench/git/w2/r0", prefixes)
|
| 188 | self.assertIn("workers/bench/oak_installed/w8/r0", prefixes)
|
| 189 |
|
| 190 |
|
| 191 | if __name__ == "__main__":
|
| 192 | unittest.main()
|