Viewing files at commit f08b4a249972Commit detailsView latest

Browse files
tests/test_scale_suite.py 895 lines · 49.4 KB · python
1
"""Instrument tests for suites/scale (generator, sizer, tiers, op oracles, rows).
2
3
Hermetic: no Oak binary and no network. Git-backed tests use the resolved
4
real git and skip when it is absent. Oak-side oracles are exercised with
5
canned subject output (a stub timed runner), so every op's untimed oracle is
6
tested for both a passing and a failing observation.
7
"""
8
9
from __future__ import annotations
10
11
import hashlib
12
import io
13
import json
14
import re
15
import shutil
16
import subprocess
17
import sys
18
import tempfile
19
import unittest
20
from pathlib import Path
21
from types import SimpleNamespace
22
23
ROOT = Path(__file__).resolve().parents[1]
24
sys.path.insert(0, str(ROOT / "suites" / "scale"))
25
sys.path.insert(0, str(ROOT / "scripts"))
26
27
from oakbench.execution import TimedCapture, run_timed  # noqa: E402
28
from oakbench.rows import validate_rows  # noqa: E402
29
from scalesuite import generator as gen  # noqa: E402
30
from scalesuite import ops as ops_mod  # noqa: E402
31
from scalesuite import runner as runner_mod  # noqa: E402
32
from scalesuite import sizer as sz  # noqa: E402
33
from scalesuite import tiers as tiers_mod  # noqa: E402
34
from scalesuite import workspace as ws  # noqa: E402
35
36
try:
37
    from oakbench.subjects import which_measured_git
38
    GIT = which_measured_git()
39
except Exception:  # pragma: no cover
40
    GIT = shutil.which("git")
41
42
SMALL = gen.GenSpec(name="t", seed="unit-seed", files=60, file_bytes=96, depth=4, fanout=4, commits=30,
43
                    touches_per_commit=3, branches=4, branch_commit_files=2, tags=3, binaries=1,
44
                    binary_bytes=(2 << 20) + 17, binary_revisions=2, ignore_patterns=("build/",),
45
                    conflict_files=3)
46
47
48
def stream_sha(spec: gen.GenSpec) -> str:
49
    buffer = io.BytesIO()
50
    gen.write_stream(spec, buffer)
51
    return hashlib.sha256(buffer.getvalue()).hexdigest()
52
53
54
def git(*args: str, cwd: Path) -> str:
55
    return subprocess.run([GIT, *args], cwd=cwd, check=True, capture_output=True, text=True,
56
                          env=gen.git_env()).stdout
57
58
59
class GeneratorDeterminismTest(unittest.TestCase):
60
    def test_same_spec_same_stream(self) -> None:
61
        self.assertEqual(stream_sha(SMALL), stream_sha(SMALL))
62
63
    def test_seed_changes_stream_and_spec_hash(self) -> None:
64
        other = gen.GenSpec(**{**SMALL.to_json(), "seed": "other-seed", "ignore_patterns": ("build/",)})
65
        self.assertNotEqual(stream_sha(SMALL), stream_sha(other))
66
        self.assertNotEqual(SMALL.spec_sha256(), other.spec_sha256())
67
68
    def test_spec_hash_is_stable_text(self) -> None:
69
        self.assertEqual(SMALL.spec_sha256(), gen.GenSpec(**{**SMALL.to_json(), "ignore_patterns": ("build/",)}).spec_sha256())
70
        self.assertRegex(SMALL.spec_sha256(), r"^[0-9a-f]{64}$")
71
72
    def test_validation(self) -> None:
73
        with self.assertRaises(ValueError):
74
            gen.GenSpec(name="x", seed="s", files=0).validate()
75
        with self.assertRaises(ValueError):
76
            gen.GenSpec(name="x", seed="s", binary_revisions=2, commits=3).validate()
77
        with self.assertRaises(ValueError):
78
            gen.GenSpec(name="x", seed="s", file_bytes=8).validate()
79
80
    def test_every_path_has_exact_depth_and_is_unique(self) -> None:
81
        for spec in (SMALL, gen.GenSpec(name="d", seed="s", files=5000, depth=40, fanout=8),
82
                     gen.GenSpec(name="w", seed="s", files=5000, depth=2, fanout=10),
83
                     gen.GenSpec(name="r", seed="s", files=7, depth=1, fanout=4)):
84
            cache = gen.layout(spec)
85
            paths = [gen.file_path(spec, i, cache) for i in range(spec.files)]
86
            self.assertEqual(len(set(paths)), spec.files)
87
            self.assertTrue(all(p.count("/") + 1 == spec.depth for p in paths), spec)
88
89
    def test_content_unique_sized_and_deterministic(self) -> None:
90
        factory = gen.ContentFactory(SMALL)
91
        blobs = {factory.text(i, "r1") for i in range(SMALL.files)}
92
        self.assertEqual(len(blobs), SMALL.files)
93
        self.assertTrue(all(len(b) == SMALL.file_bytes for b in blobs))
94
        self.assertEqual(factory.text(3, "r9"), gen.ContentFactory(SMALL).text(3, "r9"))
95
        self.assertNotEqual(factory.text(3, "r9"), factory.text(3, "r8"))
96
97
    def test_binary_is_sparse_friendly_and_exact_size(self) -> None:
98
        factory = gen.ContentFactory(SMALL)
99
        data = b"".join(factory.binary_segments(0, 1))
100
        self.assertEqual(len(data), SMALL.binary_bytes)
101
        self.assertEqual(data[gen.SEGMENT_RANDOM:gen.SEGMENT], bytes(gen.SEGMENT - gen.SEGMENT_RANDOM))
102
        self.assertNotEqual(data[:gen.SEGMENT_RANDOM], bytes(gen.SEGMENT_RANDOM))
103
        sha1, _ = gen.binary_sha1_and_sha256(factory, 0, 1)
104
        self.assertEqual(sha1, gen.git_blob_id(data))
105
106
    def test_branch_changes_match_stream_facts(self) -> None:
107
        buffer = io.BytesIO()
108
        facts = gen.write_stream(SMALL, buffer)
109
        self.assertEqual(facts["branch_sample"]["changed_paths"],
110
                         sorted(path for _, path in gen.branch_changes(SMALL, 0)))
111
        self.assertEqual(facts["total_commits"], SMALL.commits + SMALL.branches + 2)
112
113
114
@unittest.skipUnless(GIT, "git not available")
115
class GeneratedFixtureTest(unittest.TestCase):
116
    @classmethod
117
    def setUpClass(cls) -> None:
118
        cls.tmp = Path(tempfile.mkdtemp(prefix="scale-gen-"))
119
        cls.manifest = gen.generate(SMALL, cls.tmp / "a", GIT)
120
        cls.repo = cls.tmp / "a" / "src.git"
121
122
    @classmethod
123
    def tearDownClass(cls) -> None:
124
        shutil.rmtree(cls.tmp, ignore_errors=True)
125
126
    def test_same_seed_same_fixture_hash(self) -> None:
127
        again = gen.generate(SMALL, self.tmp / "b", GIT)
128
        self.assertEqual(again["refs_sha256"], self.manifest["refs_sha256"])
129
        self.assertEqual(again["head_commit"], self.manifest["head_commit"])
130
131
    def test_reuse_is_detected(self) -> None:
132
        self.assertTrue(gen.generate(SMALL, self.tmp / "a", GIT)["reused"])
133
134
    def test_manifest_facts_agree_with_git(self) -> None:
135
        probe = self.manifest["probe_path"]
136
        log = git("log", "--oneline", "main", "--", probe, cwd=self.repo).splitlines()
137
        self.assertEqual(len(log), self.manifest["probe_path_commit_count"])
138
        self.assertEqual(int(git("rev-list", "--count", "main", cwd=self.repo)), SMALL.commits)
139
        changed = git("diff", "--name-only", "main...b00000", cwd=self.repo).split()
140
        self.assertEqual(sorted(changed), self.manifest["branch_sample"]["changed_paths"])
141
        self.assertEqual(len(git("tag", "--list", cwd=self.repo).split()), SMALL.tags)
142
        conflicts = subprocess.run([GIT, "merge-tree", "--write-tree", "--name-only", "bench-ours", "bench-theirs"],
143
                                   cwd=self.repo, capture_output=True, text=True)
144
        self.assertEqual(conflicts.returncode, 1)
145
        self.assertEqual(len(conflicts.stdout.split("\n\n")[0].splitlines()[1:]), SMALL.conflict_files)
146
147
    def test_binary_blob_id_matches_generator(self) -> None:
148
        blob = git("rev-parse", f"b00000:{gen.binary_path(0)}", cwd=self.repo).strip()
149
        # branch b00000 forks from the tip, where the binary is at revision binary_revisions
150
        sha1, _ = gen.binary_sha1_and_sha256(gen.ContentFactory(SMALL), 0, SMALL.binary_revisions)
151
        self.assertEqual(blob, sha1)
152
153
    def test_sizer_values(self) -> None:
154
        stats = sz.measure(self.repo, GIT)
155
        values = stats["values"]
156
        self.assertEqual(stats["checkout_scope"], "all_commits")
157
        self.assertEqual(values["uniqueCommitCount"], self.manifest["total_commits"])
158
        self.assertEqual(values["maxCheckoutPathDepth"], SMALL.depth)
159
        self.assertEqual(values["maxCheckoutBlobCount"], SMALL.files + SMALL.binaries + 1)  # + .gitignore
160
        self.assertEqual(values["referenceCount"], 1 + SMALL.branches + 2 + SMALL.tags)
161
        self.assertEqual(values["maxBlobSize"], SMALL.binary_bytes)
162
        self.assertEqual(values["maxHistoryDepth"], SMALL.commits + 1)
163
        self.assertEqual(values["maxCommitParentCount"], 1)
164
        self.assertEqual(values["uniqueTagCount"], 0)
165
        longest = max(len(p) for p in git("ls-tree", "-r", "--name-only", "main", cwd=self.repo).split())
166
        self.assertEqual(values["maxCheckoutPathLength"], longest)
167
        self.assertEqual(stats["level_of_concern"]["maxCheckoutPathDepth"],
168
                         round(SMALL.depth / 10, 4))
169
        compact = sz.compact(stats)
170
        self.assertEqual(compact["maxCheckoutPathDepth"], SMALL.depth)
171
172
    def test_sizer_ref_tips_scope_and_tag_depth(self) -> None:
173
        clone = self.tmp / "tags.git"
174
        subprocess.run([GIT, "clone", "-q", "--bare", str(self.repo), str(clone)], check=True, env=gen.git_env())
175
        env = {**gen.git_env(), "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
176
        subprocess.run([GIT, "-C", str(clone), "tag", "-a", "-m", "one", "a1", "main"], check=True, env=env)
177
        subprocess.run([GIT, "-C", str(clone), "tag", "-a", "-m", "two", "a2", "a1"], check=True, env=env)
178
        stats = sz.measure(clone, GIT, max_checkout_roots=2)
179
        self.assertEqual(stats["checkout_scope"], "ref_tips")
180
        self.assertEqual(stats["values"]["maxTagDepth"], 2)
181
        self.assertEqual(stats["values"]["uniqueTagCount"], 2)
182
        self.assertIsNone(stats["values"]["uniqueTreeEntries"])
183
184
185
@unittest.skipUnless(GIT, "git not available")
186
class RegistryPinTest(unittest.TestCase):
187
    """Cross-machine determinism: regenerating a tier reproduces the pinned identity."""
188
189
    def test_pinned_tiers_regenerate_identically(self) -> None:
190
        registry = json.loads((ROOT / "config" / "fixtures.json").read_text())["fixtures"]
191
        tmp = Path(tempfile.mkdtemp(prefix="scale-pin-"))
192
        try:
193
            checked = 0
194
            for name in ("fresh", "default"):
195
                entry = registry[f"scale-{name}-v1"]
196
                tier = tiers_mod.TIERS[name]
197
                self.assertEqual(entry["spec_sha256"], tier.spec.spec_sha256())
198
                self.assertEqual(entry["seed"], tier.spec.seed)
199
                manifest = gen.generate(tier.spec, tmp / name, GIT)
200
                self.assertEqual(manifest["refs_sha256"], entry["refs_sha256"], name)
201
                self.assertEqual(manifest["head_commit"], entry["head_commit"], name)
202
                checked += 1
203
            self.assertEqual(checked, 2)
204
        finally:
205
            shutil.rmtree(tmp, ignore_errors=True)
206
207
    def test_every_tier_is_registered_with_its_spec_hash(self) -> None:
208
        registry = json.loads((ROOT / "config" / "fixtures.json").read_text())["fixtures"]
209
        for name, tier in tiers_mod.TIERS.items():
210
            self.assertEqual(registry[f"scale-{name}-v1"]["spec_sha256"], tier.spec.spec_sha256(), name)
211
212
213
class SizerPureTest(unittest.TestCase):
214
    def test_parse_tree(self) -> None:
215
        oid = bytes(range(20))
216
        raw = b"100644 a.txt\0" + oid + b"40000 dir\0" + oid[::-1]
217
        self.assertEqual(sz.parse_tree(raw), [(b"100644", b"a.txt", oid), (b"40000", b"dir", oid[::-1])])
218
219
    def test_level_of_concern_uses_git_sizer_reference_values(self) -> None:
220
        self.assertEqual(sz.level_of_concern("maxCheckoutBlobCount", 1_000_000), 20.0)
221
        self.assertEqual(sz.level_of_concern("referenceCount", 25_000), 1.0)
222
        self.assertIsNone(sz.level_of_concern("maxBlobSize", None))
223
224
225
class TierTest(unittest.TestCase):
226
    def test_all_14_core_shapes_are_runnable_specs(self) -> None:
227
        shapes = tiers_mod.core_shape_tiers(0.05)
228
        self.assertEqual(len(shapes), 14)
229
        for shape_id, tier in shapes.items():
230
            tier.spec.validate()
231
            self.assertGreaterEqual(tier.spec.branches, 1, shape_id)
232
            self.assertEqual(tier.kind, "shape")
233
        self.assertEqual(shapes["deep-tree"].spec.depth, 64)
234
        self.assertEqual(shapes["conflict-heavy"].spec.conflict_files, 250)
235
        self.assertEqual(shapes["binary-history"].spec.binary_revisions, shapes["binary-history"].spec.commits)
236
        self.assertIn("build/", shapes["ignored-heavy"].spec.ignore_patterns)
237
238
    def test_scale_is_part_of_identity(self) -> None:
239
        a = tiers_mod.core_shape_tiers(0.05)["normal"]
240
        b = tiers_mod.core_shape_tiers(0.1)["normal"]
241
        self.assertNotEqual(a.fixture_id, b.fixture_id)
242
243
    def test_tier_ids_unique_and_xl_opt_in(self) -> None:
244
        ids = [tier.fixture_id for tier in tiers_mod.TIERS.values()]
245
        self.assertEqual(len(ids), len(set(ids)))
246
        with self.assertRaises(SystemExit):
247
            tiers_mod.resolve("xl-tree")
248
        self.assertEqual(tiers_mod.resolve("xl-tree", allow_xl=True).spec.files, 1_000_000)
249
        with self.assertRaises(SystemExit):
250
            tiers_mod.resolve("xl-combined", allow_xl=True)
251
        combined = tiers_mod.resolve("xl-combined", allow_xl=True, allow_xl_combined=True).spec
252
        self.assertGreaterEqual(combined.commits, 100_000)
253
        self.assertGreaterEqual(combined.files, 1_000_000)
254
        self.assertGreaterEqual(combined.depth, 32)
255
        self.assertGreaterEqual(combined.branches, 10_000)
256
        self.assertGreaterEqual(combined.binary_bytes, 2 << 30)
257
        for tier in tiers_mod.TIERS.values():
258
            tier.spec.validate()
259
260
261
class CatalogTest(unittest.TestCase):
262
    def test_scale_yaml_matches_catalog(self) -> None:
263
        self.assertEqual((ROOT / "scenarios" / "scale.yaml").read_text(), ops_mod.catalog_yaml())
264
265
    def test_tperf_refs_are_file_names_only(self) -> None:
266
        pattern = re.compile(r"^p\d{4}-[a-z0-9-]+\.sh$")
267
        for info in ops_mod.OP_INFO.values():
268
            for ref in info.tperf:
269
                self.assertRegex(ref, pattern)
270
271
    def test_units_cover_ops(self) -> None:
272
        self.assertEqual(sorted(ops_mod.ALL_OPS), sorted(ops_mod.OPS))
273
274
    def test_porcelain_paths(self) -> None:
275
        text = ' M a/b.txt\n D c.txt\nR  old.txt -> new.txt\n?? added/x/\nM "sp ace.txt"\n'
276
        self.assertEqual(ops_mod.porcelain_paths(text),
277
                         ["a/b.txt", "c.txt", "old.txt", "new.txt", "added/x/", "sp ace.txt"])
278
279
280
# ---------------------------------------------------------------------------
281
# Oracle tests with canned subject output
282
# ---------------------------------------------------------------------------
283
284
285
def cap(stdout: str = "", returncode: int = 0, raw: bytes | None = None) -> TimedCapture:
286
    data = stdout.encode()
287
    return TimedCapture(elapsed_ms=1.0, returncode=returncode, stdout_text=stdout[:4000], stdout_bytes=len(data),
288
                        stdout_truncated=False, stderr_text="", stderr_bytes=0, stderr_truncated=False,
289
                        stdout_full_text=stdout, stdout_raw=raw if raw is not None else data)
290
291
292
class OracleTest(unittest.TestCase):
293
    def setUp(self) -> None:
294
        self.tmp = Path(tempfile.mkdtemp(prefix="scale-oracle-"))
295
        self.spec = gen.GenSpec(name="o", seed="oracle", files=40, file_bytes=64, depth=3, fanout=4, commits=12,
296
                                branches=2, branch_commit_files=2)
297
        facts = gen.write_stream(self.spec, io.BytesIO())
298
        factory = gen.ContentFactory(self.spec)
299
        for kind in ("git", "oak"):
300
            work = self.tmp / kind / "work"
301
            for i in range(self.spec.files):
302
                path = work / gen.file_path(self.spec, i)
303
                path.parent.mkdir(parents=True, exist_ok=True)
304
                path.write_bytes(factory.text(i, "r1"))
305
        branch_paths = facts["branch_sample"]["changed_paths"]
306
        self.expected = ws.Expected(
307
            main_file_count=self.spec.files, sample={}, probe_path=facts["probe_path"], probe_bytes=b"probe\nx\n",
308
            probe_lines=2, probe_commit_count=facts["probe_path_commit_count"], main_commits=self.spec.commits,
309
            branch="b00000", branch_paths=branch_paths, branch_bytes={}, main_bytes_of_branch_paths={},
310
            conflict={}, protected=[facts["probe_path"], *branch_paths], all_paths_sample_size=0)
311
        tier = tiers_mod.Tier(tier_id="t", kind="tier", spec=self.spec, dirty=tiers_mod.DirtyShape(3, 2, 1, 1))
312
        root = self.tmp
313
        self.prepared = SimpleNamespace(tier=tier, expected=self.expected, state={}, env={}, git_bin=GIT or "git",
314
                                        oak_base_branch="base", oak_branches_created=2,
315
                                        path=lambda *parts: root.joinpath(*parts))
316
        self.untimed_calls: list = []
317
318
    def tearDown(self) -> None:
319
        shutil.rmtree(self.tmp, ignore_errors=True)
320
321
    def ctx(self, kind: str, *outputs, run: int = 1) -> ops_mod.Ctx:
322
        queue = list(outputs)
323
        subject = SimpleNamespace(name=kind, kind=kind, bin=kind)
324
325
        def fake_runner(command, cwd, env, admitted, **kwargs):
326
            item = queue.pop(0)
327
            return item() if callable(item) else item
328
329
        ctx = ops_mod.Ctx(self.prepared, subject, run, fake_runner, self.tmp / "scratch", 60.0)
330
        ctx.untimed = lambda args, cwd, check=True, binary=None: (  # type: ignore[assignment]
331
            self.untimed_calls.append(list(args)) or SimpleNamespace(returncode=0, stdout=b"", stderr=b""))
332
        ctx.git = lambda args, cwd, check=True: (  # type: ignore[assignment]
333
            self.untimed_calls.append(["git", *args]) or SimpleNamespace(returncode=0, stdout=b"", stderr=b""))
334
        return ctx
335
336
    def run_op(self, op: str, kind: str, *outputs: TimedCapture) -> ops_mod.Outcome:
337
        return ops_mod.execute(self.ctx(kind, *outputs), op)
338
339
    def test_status_clean(self) -> None:
340
        self.assertTrue(self.run_op("status.clean", "git", cap("")).oracle_ok)
341
        self.assertFalse(self.run_op("status.clean", "oak", cap("M a.txt\n")).oracle_ok)
342
343
    def test_log_counts(self) -> None:
344
        lines = "".join(f"h{i} c{i}\n" for i in range(self.spec.commits))
345
        self.assertTrue(self.run_op("log.full", "git", cap(lines)).oracle_ok)
346
        self.assertFalse(self.run_op("log.full", "oak", cap(lines + "extra\n")).oracle_ok)
347
        self.assertTrue(self.run_op("log.n", "oak", cap(lines)).oracle_ok)
348
        self.assertFalse(self.run_op("log.n", "oak", cap("h1 c1\n")).oracle_ok)
349
        probe = "".join(f"p{i}\n" for i in range(self.expected.probe_commit_count))
350
        self.assertTrue(self.run_op("log.path", "git", cap(probe)).oracle_ok)
351
        self.assertFalse(self.run_op("log.path", "oak", cap("No matching commits\n")).oracle_ok)
352
353
    def test_blame(self) -> None:
354
        self.assertTrue(self.run_op("blame", "git", cap("a 1) probe\nb 2) x\n")).oracle_ok)
355
        self.assertFalse(self.run_op("blame", "git", cap("a 1) probe\n")).oracle_ok)
356
        self.assertEqual(self.run_op("blame", "oak").skip_reason, "oak_cli_has_no_blame_or_annotate_command")
357
358
    def test_file_show(self) -> None:
359
        self.assertTrue(self.run_op("file.show", "git", cap(raw=b"probe\nx\n")).oracle_ok)
360
        self.assertFalse(self.run_op("file.show", "git", cap(raw=b"other\n")).oracle_ok)
361
        # oak writes --output; nothing written -> mismatch
362
        self.assertFalse(self.run_op("file.show", "oak", cap("{}")).oracle_ok)
363
364
    def test_branch_list(self) -> None:
365
        self.assertTrue(self.run_op("branch.list", "git", cap("* main\n  b00000\n  b00001\n")).oracle_ok)
366
        self.assertFalse(self.run_op("branch.list", "git", cap("* main\n")).oracle_ok)
367
        listing = json.dumps([{"name": "main"}, {"name": "b00000"}, {"name": "b00001"}])
368
        self.assertTrue(self.run_op("branch.list", "oak", cap(listing)).oracle_ok)
369
        self.assertFalse(self.run_op("branch.list", "oak", cap("not json")).oracle_ok)
370
371
    def test_review_branch_diff(self) -> None:
372
        paths = self.expected.branch_paths
373
        self.assertTrue(self.run_op("review.branch_diff", "git",
374
                                    cap("".join(f"M\t{p}\n" for p in paths))).oracle_ok)
375
        oak_ok = json.dumps({"changed_files": [{"path": p} for p in paths]})
376
        self.assertTrue(self.run_op("review.branch_diff", "oak", cap(oak_ok)).oracle_ok)
377
        self.assertFalse(self.run_op("review.branch_diff", "oak", cap(json.dumps({"changed_files": []}))).oracle_ok)
378
379
    def test_merge_preview_oak(self) -> None:
380
        good = json.dumps({"clean": True, "changed_files": [{"path": p} for p in self.expected.branch_paths],
381
                           "merge_safety": {"certified": False}})
382
        outcome = self.run_op("merge.preview", "oak", cap(good, returncode=7))
383
        self.assertTrue(outcome.oracle_ok)
384
        self.assertFalse(outcome.extra["merge_certified"])
385
        self.assertIn(["switch", "b00000"], self.untimed_calls)
386
        self.assertEqual(self.untimed_calls[-1], ["switch", "base"])  # always switched back
387
        bad = json.dumps({"clean": False, "changed_files": []})
388
        self.assertFalse(self.run_op("merge.preview", "oak", cap(bad)).oracle_ok)
389
        self.assertIsNotNone(self.run_op("merge.preview", "oak", cap(good, returncode=2)).failed)
390
391
    def test_local_merge_ops_skip_for_oak(self) -> None:
392
        self.assertIn("oak_serve_lacks_merge_to_main", self.run_op("merge.local", "oak").skip_reason)
393
        self.assertEqual(self.run_op("merge.conflict", "git").skip_reason, "fixture_has_no_conflict_shape")
394
395
    def _listing(self, kind: str = "git") -> str:
396
        state = self.prepared.state[kind]["dirty"]
397
        text = "".join(f" M {p}\n" for p in state["modified"]) + "".join(f" D {p}\n" for p in state["deleted"])
398
        return text + "".join(f" D {old}\n" for old, _ in state["renamed"]) + "?? added/\n?? renamed/\n"
399
400
    def test_dirty_cycle_status_and_diff_oracles(self) -> None:
401
        good = ops_mod.execute(self.ctx("git", lambda: cap(self._listing())), "status.dirty")
402
        self.assertTrue(good.oracle_ok, good.oracle_detail)
403
        dirty = self.prepared.state["git"]["dirty"]
404
        self.assertEqual(dirty["applied"], [3, 2, 1, 1])
405
        self.assertFalse(set(dirty["modified"]) & set(self.expected.protected))
406
        diff = "".join(f"diff --git a/{p} b/{p}\n" for p in dirty["modified"] + dirty["deleted"])
407
        self.assertTrue(ops_mod.execute(self.ctx("git", cap(diff)), "diff.dirty").oracle_ok)
408
        self.assertFalse(ops_mod.execute(self.ctx("git", cap("diff --git a/x b/x\n")), "diff.dirty").oracle_ok)
409
        partial = ops_mod.execute(self.ctx("oak", lambda: cap(self._listing().split("\n", 1)[1]), run=2),
410
                                  "status.dirty")
411
        self.assertFalse(partial.oracle_ok)
412
413
    def test_dirty_cycle_runs_on_throwaway_branch(self) -> None:
414
        ops_mod.execute(self.ctx("oak", lambda: cap(self._listing("oak"))), "status.dirty")
415
        self.assertIn(["switch", "-c", "dirty-oak-r1"], self.untimed_calls)
416
        self.untimed_calls.clear()
417
        out = ops_mod.execute(self.ctx("oak", cap("{}")), "commit.dirty")
418
        self.assertIsNone(out.failed, out.failed)
419
        self.assertEqual(self.untimed_calls[-2:], [["switch", "base"], ["close", "dirty-oak-r1"]])
420
        self.assertEqual(self.prepared.state["oak"]["harness_commits"], 0)
421
422
    def test_oak_harness_branches_are_closed_on_teardown(self) -> None:
423
        self.prepared.state["oak"] = {"harness_commits": 0, "deleted": set(), "server_extra": 0, "branch_seq": 0}
424
        ops_mod.execute(self.ctx("oak", cap("")), "branch.create")
425
        self.assertIn(["close", "new-oak-r1-1"], self.untimed_calls)
426
        listing = json.dumps([{"name": "main"}, {"name": "b00000"}, {"name": "b00001"}])
427
        self.untimed_calls.clear()
428
        seen = []
429
        ctx = self.ctx("oak", cap(listing))
430
        original = ctx._run_timed
431
        ctx._run_timed = lambda command, *a, **k: (seen.append(command), original(command, *a, **k))[1]
432
        ops_mod.execute(ctx, "branch.list")
433
        self.assertIn("--status", seen[0])
434
435
    def test_recover_returns_to_base(self) -> None:
436
        ctx = self.ctx("oak")
437
        self.assertIsNone(ops_mod.recover(ctx))
438
        self.assertEqual(self.untimed_calls[-1], ["switch", "base"])
439
440
    def test_oak_state_oracles_use_independent_reader(self) -> None:
441
        from unittest import mock
442
        from oakbench import oak_sqlite_reader as reader
443
        probe = self.expected.probe_path
444
        self.prepared.state["oak"] = {"harness_commits": 0, "deleted": set(), "server_extra": 0, "branch_seq": 0}
445
        with mock.patch.object(reader, "committed_payloads",
446
                               return_value=({probe: self.expected.probe_bytes}, {"head": "h", "reader_version": "v"})):
447
            outcome = ops_mod.execute(self.ctx("oak", cap("")), "gc")
448
        self.assertEqual(outcome.oracle_reader, "oak_sqlite_reader_independent")
449
        self.assertTrue(outcome.oracle_ok, outcome.oracle_detail)
450
        with mock.patch.object(reader, "committed_payloads", return_value=({probe: b"torn"}, {"head": "h"})):
451
            self.assertFalse(ops_mod.execute(self.ctx("oak", cap("")), "gc").oracle_ok)
452
        with mock.patch.object(reader, "committed_payloads", side_effect=reader.Unavailable("database too big")):
453
            unmeasured = ops_mod.execute(self.ctx("oak", cap("")), "gc")
454
        self.assertIsNone(unmeasured.oracle_ok)
455
        self.assertIsNone(unmeasured.failed)
456
        self.assertIn("reader_unavailable", unmeasured.oracle_detail)
457
458
    def test_commit_needs_dirty_state(self) -> None:
459
        self.assertIn("needs status.dirty", self.run_op("commit.dirty", "git").failed)
460
461
    def test_timed_failure_is_recorded_not_raised(self) -> None:
462
        outcome = self.run_op("status.clean", "oak", cap("", returncode=3))
463
        self.assertIn("exit 3", outcome.failed)
464
        self.assertIsNone(outcome.oracle_ok)
465
466
    def test_check_worktree(self) -> None:
467
        root = self.tmp / "git" / "work"
468
        path = gen.file_path(self.spec, 5)
469
        self.expected.sample[path] = gen.git_blob_id((root / path).read_bytes())
470
        ctx = self.ctx("git")
471
        ok, detail = ops_mod.check_worktree(ctx, root, self.spec.files)
472
        self.assertTrue(ok, detail)
473
        (root / path).write_bytes(b"changed")
474
        ok, detail = ops_mod.check_worktree(ctx, root, self.spec.files)
475
        self.assertFalse(ok)
476
        self.assertEqual(detail["sample_mismatch_count"], 1)
477
478
479
class RowTest(unittest.TestCase):
480
    def prepared(self) -> SimpleNamespace:
481
        tier = tiers_mod.TIERS["fresh"]
482
        fields = {"tier": "fresh", "tier_kind": "tier", "fixture_id": tier.fixture_id,
483
                  "fixture_manifest_sha256": "a" * 64, "fixture_spec_sha256": "b" * 64}
484
        return SimpleNamespace(tier=tier, fixture_fields=lambda: fields, phases={"ingest.oak_import": 1.5},
485
                               notes=[], manifest={"refs_sha256": "a" * 64, "spec_sha256": "b" * 64})
486
487
    def row(self, subject: str, kind: str, op: str, run: int, outcome: ops_mod.Outcome) -> dict:
488
        return runner_mod.build_row(bench_id="b", prepared=self.prepared(),
489
                                    subject=SimpleNamespace(name=subject, kind=kind), op=op, run=run,
490
                                    outcome=outcome, details={}, env_sample={}, lock_wait_ms=0.0,
491
                                    order_position=0, runner_fields={})
492
493
    def test_rows_validate_for_all_outcomes(self) -> None:
494
        ok = ops_mod.Outcome(timed=[ops_mod.Timed("main", ["git", "status"], cap(""))], oracle_ok=True)
495
        skip = ops_mod.Outcome(skip_reason="no_equivalent")
496
        fail = ops_mod.Outcome(timed=[ops_mod.Timed("main", ["oak", "status"], cap("", 2))], failed="exit 2")
497
        bad = ops_mod.Outcome(timed=[ops_mod.Timed("main", ["oak", "log"], cap(""))], oracle_ok=False)
498
        rows = [self.row("git", "git", "status.clean", 1, ok), self.row("oak_installed", "oak", "blame", 1, skip),
499
                self.row("oak_installed", "oak", "status.clean", 1, fail),
500
                self.row("oak_installed", "oak", "log.path", 1, bad)]
501
        rows.append(runner_mod.abort_row(rows[0], "diff.dirty", "status.dirty"))
502
        prepared = self.prepared()
503
        prepared.oak_identity = {"binary_sha256": "c" * 64}
504
        rows += runner_mod.ingest_rows(prepared, "b", {})
505
        self.assertEqual(validate_rows(rows, "scale"), [])
506
        self.assertEqual([r["returncode"] for r in rows[:5]], [0, 77, 1, 1, 77])
507
        self.assertEqual(rows[3]["failure"], "oracle_failed")
508
        self.assertFalse(any(r["claim_eligible"] for r in rows))
509
        self.assertTrue(all(r["oracle_timed"] is False for r in rows[:4]))
510
        import publish_gate
511
        for row in rows:
512
            if row["subject_kind"] == "oak" and row["operation"].startswith("ingest."):
513
                self.assertEqual(publish_gate.oak_binary_identity(row), "c" * 64)
514
515
    def test_cell_report_gives_floors_and_mde(self) -> None:
516
        rows = []
517
        for run in range(1, 7):
518
            for subject, kind, base in (("git", "git", 10.0), ("git_aa", "git", 10.0),
519
                                        ("oak_installed", "oak", 20.0), ("oak_installed_aa", "oak", 20.0)):
520
                outcome = ops_mod.Outcome(timed=[ops_mod.Timed("main", [kind], cap(""))], oracle_ok=True)
521
                row = self.row(subject, kind, "status.clean", run, outcome)
522
                row["elapsed_ms"] = base + (run % 3) * 0.2
523
                rows.append(row)
524
        report = runner_mod.cell_report(rows, ["git", "git_aa", "oak_installed", "oak_installed_aa"])
525
        item = report[0]
526
        self.assertIsNotNone(item["git_aa_floor_pct"])
527
        self.assertIsNotNone(item["oak_vs_git_mde_pct_80"])
528
        self.assertIsNotNone(item["oak_vs_oak_mde_pct_80"])
529
        self.assertAlmostEqual(item["oak_vs_git_delta_pct_dev_only"], 100.0, delta=5)
530
        self.assertFalse(item["claim_eligible"])
531
        text = runner_mod.summary_markdown(self.prepared(), report,
532
                                           {"bench_id": "b", "runs": 6, "subjects": ["git"]})
533
        self.assertIn("DEVELOPMENT ONLY", text)
534
        self.assertRegex(text, r"\| 1\.9\dx \|")
535
536
537
@unittest.skipUnless(GIT, "git not available")
538
class GitOnlyScheduleTest(unittest.TestCase):
539
    """End to end on git only: prepare, interleave git + git_aa, every oracle passes."""
540
541
    def test_schedule(self) -> None:
542
        from oakbench.environment import base_env
543
        tmp = Path(tempfile.mkdtemp(prefix="scale-e2e-"))
544
        try:
545
            spec = gen.GenSpec(name="e2e", seed="e2e", files=24, file_bytes=64, depth=3, fanout=4, commits=9,
546
                               branches=2, conflict_files=2)
547
            tier = tiers_mod.Tier(tier_id="e2e", kind="tier", spec=spec, dirty=tiers_mod.DirtyShape(2, 1, 1, 1))
548
            env = base_env(home=tmp / "home")
549
            # default logger (stderr) on purpose: it must be importable and callable
550
            prepared = ws.prepare(tier, tmp / "work", git_bin=GIT, oak_bin="oak", env=env, kinds=("git",))
551
            subjects = [SimpleNamespace(name=n, kind="git", bin=Path(GIT)) for n in ("git", "git_aa")]
552
            try:
553
                rows = runner_mod.run_schedule(prepared, subjects, runs=1, ops=ops_mod.ALL_OPS, bench_id="e2e",
554
                                               seed=1, scratch=tmp / "scratch", details={}, runner_fields={},
555
                                               timeout_s=120, lock_wait_ms=None, log=lambda *_: None)
556
            finally:
557
                prepared.close()
558
            self.assertEqual(validate_rows(rows, "scale"), [])
559
            failed = [(r["operation"], r.get("failure"), r.get("oracle_detail")) for r in rows if r["returncode"] == 1]
560
            self.assertEqual(failed, [])
561
            measured = {r["operation"] for r in rows if r["returncode"] == 0}
562
            self.assertEqual(measured, set(ops_mod.ALL_OPS))
563
            self.assertTrue(all(r["oracle_ok"] for r in rows if r["returncode"] == 0))
564
        finally:
565
            shutil.rmtree(tmp, ignore_errors=True)
566
567
568
if __name__ == "__main__":
569
    unittest.main()
570
571
572
class WatchdogTimingTest(unittest.TestCase):
573
    """QA P5-1: a bounded command must not pay Popen.wait(timeout) poll quantization."""
574
575
    def test_watchdog_bound_does_not_change_timing(self) -> None:
576
        import statistics
577
        env = {"PATH": "/usr/bin:/bin"}
578
        plain, bounded = [], []
579
        for _ in range(25):  # interleaved pairs share host drift
580
            plain.append(run_timed(["/bin/sleep", "0.013"], Path("/"), env, 100).elapsed_ms)
581
            bounded.append(run_timed(["/bin/sleep", "0.013"], Path("/"), env, 100, watchdog_seconds=60.0).elapsed_ms)
582
        delta = statistics.median(bounded) - statistics.median(plain)
583
        self.assertLess(abs(delta), 2.0, (statistics.median(plain), statistics.median(bounded)))
584
585
    def test_watchdog_kills_and_reports_timeout(self) -> None:
586
        capture = run_timed(["/bin/sleep", "5"], Path("/"), {"PATH": "/usr/bin:/bin"}, 100, watchdog_seconds=0.3)
587
        self.assertTrue(capture.timed_out)
588
        self.assertEqual(capture.returncode, 124)
589
        self.assertLess(capture.elapsed_ms, 3000)
590
591
    def test_both_bounds_rejected(self) -> None:
592
        with self.assertRaises(ValueError):
593
            run_timed(["/usr/bin/true"], Path("/"), {}, 10, timeout_seconds=1, watchdog_seconds=1)
594
595
    def test_scale_ops_use_the_watchdog(self) -> None:
596
        seen = {}
597
598
        def fake(command, cwd, env, admitted, **kwargs):
599
            seen.update(kwargs)
600
            return cap("")
601
        subject = SimpleNamespace(name="git", kind="git", bin="git")
602
        prepared = SimpleNamespace(env={}, state={}, path=lambda *p: Path("/"))
603
        ops_mod.Ctx(prepared, subject, 1, fake, Path("/"), 30.0).timed(["status"], Path("/"))
604
        self.assertEqual(seen.get("watchdog_seconds"), 30.0)
605
        self.assertNotIn("timeout_seconds", seen)
606
607
608
def _cell_rows(tier: str, op: str, seed, base_git: float = 100.0, base_oak: float = 700.0) -> list:
609
    rows = []
610
    for run in range(1, 7):
611
        for subject, kind, base in (("git", "git", base_git), ("git_aa", "git", base_git),
612
                                    ("oak_installed", "oak", base_oak), ("oak_installed_aa", "oak", base_oak)):
613
            row = {"scenario": tier, "operation": op, "run": run, "subject": subject, "subject_kind": kind,
614
                   "returncode": 0, "elapsed_ms": base + (run % 3) * 0.5}
615
            if kind == "oak" and seed is not None:
616
                row["oak_server_seed"] = seed
617
            rows.append(row)
618
    return rows
619
620
621
class ComparabilityTest(unittest.TestCase):
622
    """QA P5-2: a snapshot-seeded Oak server is not the repository Git serves."""
623
624
    SUBJECTS = ["git", "git_aa", "oak_installed", "oak_installed_aa"]
625
626
    def test_snapshot_seeded_sync_cells_suppress_oak_vs_git(self) -> None:
627
        rows = []
628
        tiers = ("default", "xl-deep", "xl-history", "xl-refs")
629
        ops = ("push.branch", "pull.ff", "pull.diverged", "clone.full")
630
        for tier in tiers:
631
            for op in ops:
632
                rows += _cell_rows(tier, op, "snapshot_1_commit")
633
            rows += _cell_rows(tier, "status.clean", None)
634
        report = {(c["scenario"], c["operation"]): c for c in runner_mod.cell_report(rows, self.SUBJECTS)}
635
        for tier in tiers:
636
            for op in ops:
637
                cell = report[(tier, op)]
638
                self.assertEqual(cell["oak_vs_git_comparability"], "not_comparable:oak_server_seed_snapshot_1_commit")
639
                self.assertIsNone(cell["oak_vs_git_delta_pct_dev_only"])
640
                self.assertIsNone(cell["oak_vs_git_mde_pct_80"])
641
                self.assertIsNotNone(cell["oak_vs_oak_mde_pct_80"])
642
            comparable = report[(tier, "status.clean")]
643
            self.assertEqual(comparable["oak_vs_git_comparability"], "comparable")
644
            self.assertIsNotNone(comparable["oak_vs_git_delta_pct_dev_only"])
645
        prepared = SimpleNamespace(tier=tiers_mod.TIERS["default"], phases={}, notes=[],
646
                                   manifest={"refs_sha256": "a" * 64, "spec_sha256": "b" * 64})
647
        text = runner_mod.summary_markdown(prepared, [report[("default", "pull.ff")], report[("default", "status.clean")]],
648
                                           {"bench_id": "b", "runs": 6, "subjects": self.SUBJECTS})
649
        pull_line = next(line for line in text.splitlines() if line.startswith("| pull.ff "))
650
        self.assertIn("| n/c |", pull_line)
651
        self.assertIn("oak-vs-oak only", pull_line)
652
        self.assertNotIn("7.00x", pull_line)
653
654
    def test_full_history_seed_stays_comparable(self) -> None:
655
        rows = _cell_rows("fresh", "clone.full", "full_history")
656
        cell = runner_mod.cell_report(rows, self.SUBJECTS)[0]
657
        self.assertEqual(cell["oak_vs_git_comparability"], "comparable")
658
659
660
class CalibrationGateTest(unittest.TestCase):
661
    """QA P5-3 (rounds 1-2): claimability is verified by RECOMPUTING the calibration
662
    campaign from its own JSONL; nothing a record states is trusted."""
663
664
    PIN = "scale-fresh-v1"
665
666
    def setUp(self) -> None:
667
        from oakbench import scale_calibration as cal
668
        self.cal = cal
669
        self.tmp = Path(tempfile.mkdtemp(prefix="scale-cal-"))
670
        self.addCleanup(shutil.rmtree, self.tmp, True)
671
        entry = json.loads((ROOT / "config" / "fixtures.json").read_text())["fixtures"][self.PIN]
672
        self.fixture = {"tier": "fresh", "fixture_id": "scale-fresh-x", "fixture_spec_sha256": entry["spec_sha256"],
673
                        "fixture_manifest_sha256": entry["refs_sha256"], "fixture_pin_status": "verified"}
674
        self.volume = {"mount_point": "/System/Volumes/Data", "device": "/dev/disk3s5", "fstype": "apfs",
675
                       "volume_uuid": "VOL-1", "bus_protocol": "Apple Fabric", "physical_store": "disk0s2",
676
                       "medium": "physical"}
677
        self.identity = {"runner_id": "runner-1", "host_cpu_count": 14, "host_mem_bytes": 1 << 35,
678
                         "work_volume": self.volume, "runner_profile": {"system": "Darwin"}}
679
680
    def rows(self, bench_id: str, *, load: float = 0.5, jitter: float = 0.05, fsync_ms: float = 1.0,
681
             identity=None, calibrated=None, flush_kind: str = "F_FULLFSYNC", slow_aa=None,
682
             late_aa_shift: float = 0.0) -> list:
683
        ident = identity or self.identity
684
        rows = []
685
        for run in range(1, 11):
686
            for op_index in range(12):
687
                for subject, kind in (("git", "git"), ("git_aa", "git"), ("oak_installed", "oak"),
688
                                      ("oak_installed_aa", "oak")):
689
                    rows.append({"bench_id": bench_id, "profile": "scale", "scenario": "fresh",
690
                                 "operation": f"op{op_index}", "run": run, "subject": subject, "subject_kind": kind,
691
                                 "elapsed_ms": 10.0 + op_index + ((run + subject.endswith("_aa")) % 3) * jitter,
692
                                 "returncode": 0, "command": [],
693
                                 "binary_sha256": ("a" if kind == "git" else "b") * 64,
694
                                 "environment_sample": {"loadavg": [load, load, load]},
695
                                 **self.fixture, **ident})
696
                    if subject == "oak_installed_aa" and slow_aa and op_index in slow_aa:
697
                        rows[-1]["elapsed_ms"] *= 1.5
698
                    if subject == "oak_installed_aa" and late_aa_shift and run > 5:
699
                        rows[-1]["elapsed_ms"] *= 1.0 + late_aa_shift
700
        base = {"bench_id": bench_id, "profile": "scale", "scenario": "fresh", **self.fixture}
701
        probe = {"fullfsync_ms": [fsync_ms] * 20, "fullfsync_kind": flush_kind, "create_files": 500,
702
                 "create_elapsed_s": 0.05, "work_volume": ident["work_volume"]}
703
        probes = self.cal.probe_rows(probe, base, "start")
704
        for row in probes:
705
            row.update(ident)
706
        rows = probes + rows
707
        if calibrated is not None:
708
            record, facts, reasons = calibrated
709
            for row in rows:
710
                if not row.get("diagnostic"):
711
                    self.cal.stamp(row, record, facts, reasons)
712
        return rows
713
714
    def write(self, name: str, rows: list) -> Path:
715
        path = self.tmp / name
716
        path.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows))
717
        return path
718
719
    def campaign(self, name: str = "calibration.jsonl", **kwargs):
720
        path = self.write(name, self.rows("cal", **kwargs))
721
        rows = self.cal.load_rows(path)
722
        facts, problems = self.cal.recompute(rows, self.registry())
723
        record = self.cal.seal({**{k: facts.get(k) for k in ("runner_id", "host_cpu_count", "host_mem_bytes",
724
                                                              "oak_binary_sha256", "git_binary_sha256",
725
                                                              "work_volume_key", "aa_cells",
726
                                                              "aa_floor_median_pct", "aa_false_claims")},
727
                                "calibration_rows_path": str(path),
728
                                "calibration_rows_sha256": self.cal.file_sha256(path)})
729
        return path, record, facts, problems
730
731
    def registry(self):
732
        return json.loads((ROOT / "config" / "fixtures.json").read_text())["fixtures"]
733
734
    def gate_text(self, *paths: Path) -> str:
735
        import contextlib
736
        import publish_gate
737
        out = io.StringIO()
738
        with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
739
            publish_gate.main([str(p) for p in paths])
740
        return out.getvalue()
741
742
    def published(self, record, facts, problems, **kwargs) -> Path:
743
        return self.write("published.jsonl", self.rows("pub", calibrated=(record, facts, problems), **kwargs))
744
745
    def test_quiet_campaign_recomputes_and_publishes(self) -> None:
746
        cal_path, record, facts, problems = self.campaign()
747
        self.assertEqual(problems, [])
748
        self.assertEqual(facts["aa_cells"], 24)
749
        self.assertEqual(facts["aa_per_arm"]["git"]["held_out_claims"], 0)
750
        self.assertEqual(facts["aa_false_claims"], 0)
751
        pub = self.published(record, facts, problems)
752
        rows = self.cal.load_rows(pub)
753
        self.assertTrue(all(r["claim_eligible"] for r in rows if not r.get("diagnostic")))
754
        self.assertEqual(validate_rows(rows, "scale"), [])
755
        text = self.gate_text(pub, cal_path)
756
        self.assertIn("PASS   scale-calibration", text)
757
        self.assertNotIn("FAIL   scale-calibration", text)
758
759
    def test_campaign_jsonl_missing_from_inputs_fails(self) -> None:
760
        cal_path, record, facts, problems = self.campaign()
761
        self.assertIn("FAIL   scale-calibration", self.gate_text(self.published(record, facts, problems)))
762
763
    def test_calibration_jsonl_not_matching_record_sha_fails(self) -> None:
764
        cal_path, record, facts, problems = self.campaign()
765
        pub = self.published(record, facts, problems)
766
        cal_path.write_text(cal_path.read_text().replace('"elapsed_ms": 10.0', '"elapsed_ms": 10.5', 1))
767
        text = self.gate_text(pub, cal_path)
768
        self.assertIn("FAIL   scale-calibration", text)
769
        self.assertIn("calibration campaign rows not provided", text)  # no input matches the record's sha
770
771
    def test_resealed_edited_record_fails(self) -> None:
772
        cal_path, record, facts, problems = self.campaign(jitter=4.0)  # noisy campaign: floors too wide
773
        self.assertTrue(problems)
774
        forged = self.cal.seal({**{k: v for k, v in record.items() if k not in ("record_sha256",)},
775
                                "aa_floor_median_pct": 0.0})
776
        pub = self.published(forged, facts, [])  # stamped as if valid
777
        text = self.gate_text(pub, cal_path)
778
        self.assertIn("FAIL   scale-calibration", text)
779
780
    def test_resealed_record_numbers_are_never_trusted(self) -> None:
781
        cal_path, record, facts, problems = self.campaign()
782
        forged = self.cal.seal({**{k: v for k, v in record.items() if k != "record_sha256"}, "aa_cells": 999})
783
        text = self.gate_text(self.published(forged, facts, []), cal_path)
784
        self.assertIn("FAIL   scale-calibration", text)
785
        self.assertIn("recomputed", text)
786
787
    def test_copied_runner_record_fails(self) -> None:
788
        cal_path, record, facts, problems = self.campaign()
789
        other = dict(self.identity, runner_id="runner-2")
790
        text = self.gate_text(self.published(record, facts, [], identity=other), cal_path)
791
        self.assertIn("FAIL   scale-calibration", text)
792
        resealed = self.cal.seal({**{k: v for k, v in record.items() if k != "record_sha256"},
793
                                  "runner_id": "runner-2"})
794
        text = self.gate_text(self.published(resealed, facts, [], identity=other), cal_path)
795
        self.assertIn("FAIL   scale-calibration", text)
796
797
    def test_host_mismatch_and_loaded_host_fail(self) -> None:
798
        cal_path, record, facts, problems = self.campaign()
799
        bigger = dict(self.identity, host_cpu_count=100000)
800
        self.assertIn("FAIL   scale-calibration", self.gate_text(self.published(record, facts, [], identity=bigger),
801
                                                                   cal_path))
802
        loaded = self.campaign("loaded.jsonl", load=2.5)
803
        self.assertTrue(any("not quiet" in r for r in loaded[3]))
804
        disk = self.campaign("disk.jsonl", fsync_ms=4.0)
805
        self.assertTrue(any("disk contended" in r for r in disk[3]))
806
        text = self.gate_text(self.published(record, facts, [], fsync_ms=4.0), cal_path)
807
        self.assertIn("published run: disk contended", text)
808
809
    def test_broken_aa_arm_fails(self) -> None:
810
        # QA r3 probe: the identical oak binary 50% slower in 11 of 12 oak_aa cells.
811
        _, _, facts, problems = self.campaign("slow-aa.jsonl", slow_aa=set(range(11)))
812
        self.assertTrue(any("oak_installed" in p and ("floor" in p or "cells have" in p) for p in problems),
813
                        problems)
814
        self.assertEqual(facts["aa_per_arm"]["git"]["cells_over_threshold"], 0)
815
816
    def test_held_out_aa_claims_are_counted(self) -> None:
817
        # A shift that appears only in the held-out half cannot hide inside its own floor.
818
        _, _, facts, problems = self.campaign("late-shift.jsonl", late_aa_shift=0.5)
819
        self.assertGreater(facts["aa_false_claims"], 0)
820
        self.assertTrue(any("held-out claim" in p for p in problems), problems)
821
822
    def test_quiet_campaign_has_no_held_out_claims(self) -> None:
823
        _, _, facts, problems = self.campaign("quiet.jsonl")
824
        self.assertEqual(problems, [])
825
        self.assertEqual(facts["aa_per_arm"]["oak_installed"]["held_out_claims"], 0)
826
827
    def test_too_few_runs_fails(self) -> None:
828
        path = self.write("short.jsonl", [r for r in self.rows("cal") if r.get("diagnostic") or r["run"] <= 6])
829
        self.assertTrue(any("fewer than 10 runs" in p for p in self.cal.recompute(self.cal.load_rows(path),
830
                                                                                   self.registry())[1]))
831
832
    def test_ram_or_image_volume_and_volume_mismatch_fail(self) -> None:
833
        ram = dict(self.volume, medium="ram_or_disk_image", bus_protocol="Disk Image", volume_uuid="RAM")
834
        _, _, _, problems = self.campaign("ram.jsonl", identity=dict(self.identity, work_volume=ram))
835
        self.assertTrue(any("cannot certify" in p for p in problems), problems)
836
        cal_path, record, facts, problems = self.campaign()
837
        self.assertEqual(problems, [])
838
        other = dict(self.volume, volume_uuid="VOL-2", device="/dev/disk9s1")
839
        rows = self.rows("pub", calibrated=(record, facts, []))
840
        self.assertTrue(rows[-1]["claim_eligible"])
841
        for row in rows:  # hand-edit: claim the published run used another (physical) volume
842
            row["work_volume"] = other
843
        text = self.gate_text(self.write("published.jsonl", rows), cal_path)
844
        self.assertIn("FAIL   scale-calibration", text)
845
        self.assertIn("different work volume", text)
846
847
    def test_flush_probe_must_use_platform_full_flush(self) -> None:
848
        _, _, _, problems = self.campaign("plainfsync.jsonl", flush_kind="fsync")
849
        self.assertTrue(any("full-flush primitive" in p for p in problems), problems)
850
        linux = dict(self.identity, runner_profile={"system": "Linux"})
851
        _, _, _, problems = self.campaign("linux.jsonl", identity=linux, flush_kind="fsync")
852
        self.assertFalse(any("full-flush" in p for p in problems), problems)
853
854
    def test_real_volume_identity_is_physical_here(self) -> None:
855
        volume = self.cal.volume_identity(ROOT)
856
        self.assertIn(volume["medium"], ("physical", "unknown"))
857
        self.assertTrue(volume["mount_point"])
858
859
    def test_classify_medium(self) -> None:
860
        self.assertEqual(self.cal.classify_medium({"fstype": "tmpfs"}, {}), "tmpfs")
861
        self.assertEqual(self.cal.classify_medium({"fstype": "apfs"}, {"BusProtocol": "Disk Image"}),
862
                         "ram_or_disk_image")
863
        self.assertEqual(self.cal.classify_medium({"fstype": "apfs"}, {"BusProtocol": "Apple Fabric"}), "physical")
864
        self.assertEqual(self.cal.classify_medium({"fstype": "apfs"}, {}), "unknown")
865
866
    def test_qa_forged_record_fixture_fails(self) -> None:
867
        import gzip
868
        forged = self.tmp / "forgedC.fresh.jsonl"
869
        forged.write_bytes(gzip.decompress((ROOT / "tests" / "fixtures" / "scale" /
870
                                            "qa-r2-forgedC.fresh.jsonl.gz").read_bytes()))
871
        rows = self.cal.load_rows(forged)
872
        self.assertTrue(any(r.get("claim_eligible") is True for r in rows))  # the forgery claims eligibility
873
        self.assertIn("FAIL   scale-calibration", self.gate_text(forged))
874
875
    def test_without_record_rows_are_dev_only(self) -> None:
876
        row = self.cal.stamp({"returncode": 0}, None, None, [])
877
        self.assertFalse(row["claim_eligible"])
878
        self.assertEqual(row["evidence_class"], "development_only_loaded_host")
879
880
881
class PinAndTeardownTest(unittest.TestCase):
882
    def test_check_pin(self) -> None:
883
        registry = json.loads((ROOT / "config" / "fixtures.json").read_text())["fixtures"]["scale-default-v1"]
884
        manifest = {"spec_sha256": registry["spec_sha256"], "refs_sha256": registry["refs_sha256"]}
885
        self.assertEqual(gen.check_pin(manifest), "verified")
886
        with self.assertRaises(gen.FixturePinMismatch):
887
            gen.check_pin(dict(manifest, refs_sha256="0" * 64))
888
        self.assertEqual(gen.check_pin({"spec_sha256": "f" * 64, "refs_sha256": "0" * 64}), "unpinned")
889
890
    def test_results_file_naming(self) -> None:
891
        from oakbench.results import ResultsStore
892
        with tempfile.TemporaryDirectory() as tmp:
893
            store = ResultsStore(Path(tmp), "scale", filename_suffix="fresh")
894
            raw, _ = store.write("20260929T000000Z", [], "x")
895
            self.assertEqual(raw.name, "20260929T000000Z.fresh.jsonl")