| 1 | from __future__ import annotations
|
| 2 |
|
| 3 | import argparse
|
| 4 | import json
|
| 5 | import shutil
|
| 6 | import subprocess
|
| 7 | import sys
|
| 8 | import tempfile
|
| 9 | import unittest
|
| 10 | from pathlib import Path
|
| 11 |
|
| 12 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 13 |
|
| 14 | import baseline_campaign
|
| 15 | from oakbench.baseline_book import (
|
| 16 | DEFAULT_BASELINE_METRICS,
|
| 17 | DEFAULT_BASELINE_STATISTICS,
|
| 18 | sha256_file,
|
| 19 | )
|
| 20 |
|
| 21 |
|
| 22 | def row(run: int, subject: str = "git", elapsed: float = 10.0) -> dict:
|
| 23 | return {
|
| 24 | "bench_id": "b",
|
| 25 | "profile": "smoke",
|
| 26 | "benchmark_track": "core-equivalent",
|
| 27 | "scenario": "tiny_text",
|
| 28 | "operation": "status.clean",
|
| 29 | "run": run,
|
| 30 | "subject": subject,
|
| 31 | "subject_kind": "git",
|
| 32 | "elapsed_ms": elapsed,
|
| 33 | "returncode": 0,
|
| 34 | "command": ["git", "status"],
|
| 35 | }
|
| 36 |
|
| 37 |
|
| 38 | class BaselineCampaignTest(unittest.TestCase):
|
| 39 | def test_book_mode_writes_manifest_book_and_refuses_overwrite(self) -> None:
|
| 40 | with tempfile.TemporaryDirectory() as tmp:
|
| 41 | root = Path(tmp)
|
| 42 | rows = root / "rows.jsonl"
|
| 43 | rows.write_text("".join(json.dumps(row(i, elapsed=10.0 + i)) + "\n" for i in range(2)))
|
| 44 | out = root / "book"
|
| 45 | cmd = [
|
| 46 | sys.executable,
|
| 47 | "scripts/baseline_campaign.py",
|
| 48 | "book",
|
| 49 | "--book-id",
|
| 50 | "test-book",
|
| 51 | "--out",
|
| 52 | str(out),
|
| 53 | "--input-jsonl",
|
| 54 | str(rows),
|
| 55 | "--raw-storage-key",
|
| 56 | "raw/key.jsonl",
|
| 57 | "--bootstrap-samples",
|
| 58 | "20",
|
| 59 | "--noise-floor-pct",
|
| 60 | "4.2",
|
| 61 | "--skip-publish-gate",
|
| 62 | ]
|
| 63 | proc = subprocess.run(
|
| 64 | cmd,
|
| 65 | cwd=Path(__file__).resolve().parents[1],
|
| 66 | text=True,
|
| 67 | stdout=subprocess.PIPE,
|
| 68 | stderr=subprocess.PIPE,
|
| 69 | check=False,
|
| 70 | )
|
| 71 | self.assertEqual(proc.returncode, 0, proc.stderr)
|
| 72 | manifest = json.loads((out / "manifest.json").read_text())
|
| 73 | self.assertEqual(manifest["raw_storage_keys"], ["raw/key.jsonl"])
|
| 74 | self.assertIsNotNone(manifest["runner_id"])
|
| 75 | book = json.loads((out / "book.json").read_text())
|
| 76 | self.assertEqual(book["schema_version"], 2)
|
| 77 | self.assertGreaterEqual(len(book["cells"]), 1)
|
| 78 | self.assertTrue(all(cell["noise_floor_pct"] == 4.2 for cell in book["cells"]))
|
| 79 |
|
| 80 | proc = subprocess.run(
|
| 81 | cmd,
|
| 82 | cwd=Path(__file__).resolve().parents[1],
|
| 83 | text=True,
|
| 84 | stdout=subprocess.PIPE,
|
| 85 | stderr=subprocess.PIPE,
|
| 86 | check=False,
|
| 87 | )
|
| 88 | self.assertEqual(proc.returncode, 2)
|
| 89 | self.assertIn("refusing to overwrite", proc.stderr)
|
| 90 |
|
| 91 | def test_book_mode_rejects_negative_noise_floor_pct(self) -> None:
|
| 92 | with tempfile.TemporaryDirectory() as tmp:
|
| 93 | root = Path(tmp)
|
| 94 | rows = root / "rows.jsonl"
|
| 95 | rows.write_text("".join(json.dumps(row(i, elapsed=10.0 + i)) + "\n" for i in range(2)))
|
| 96 | proc = subprocess.run(
|
| 97 | [
|
| 98 | sys.executable,
|
| 99 | "scripts/baseline_campaign.py",
|
| 100 | "book",
|
| 101 | "--book-id",
|
| 102 | "test-book",
|
| 103 | "--out",
|
| 104 | str(root / "book"),
|
| 105 | "--input-jsonl",
|
| 106 | str(rows),
|
| 107 | "--noise-floor-pct",
|
| 108 | "-1",
|
| 109 | "--skip-publish-gate",
|
| 110 | ],
|
| 111 | cwd=Path(__file__).resolve().parents[1],
|
| 112 | text=True,
|
| 113 | stdout=subprocess.PIPE,
|
| 114 | stderr=subprocess.PIPE,
|
| 115 | check=False,
|
| 116 | )
|
| 117 | self.assertEqual(proc.returncode, 2)
|
| 118 | self.assertIn("must be >= 0", proc.stderr)
|
| 119 |
|
| 120 | def test_resolve_git_binary_prefers_git_bin_then_path_lookup(self) -> None:
|
| 121 | with tempfile.TemporaryDirectory() as tmp:
|
| 122 | fake_git = Path(tmp) / "fake-git"
|
| 123 | fake_git.write_text("#!/bin/sh\necho 'git version 9.9.9-test'\n")
|
| 124 | fake_git.chmod(0o755)
|
| 125 | self.assertEqual(
|
| 126 | baseline_campaign.resolve_git_binary(argparse.Namespace(git_bin=fake_git)),
|
| 127 | fake_git,
|
| 128 | )
|
| 129 | # Without --git-bin (book mode has no such flag) fall back to PATH,
|
| 130 | # with the macOS xcrun shim resolved to the Git that actually runs.
|
| 131 | from oakbench.subjects import which_measured_git
|
| 132 | which = which_measured_git()
|
| 133 | expected = Path(which) if which else None
|
| 134 | if which and shutil.which("git") == "/usr/bin/git" and sys.platform == "darwin":
|
| 135 | self.assertNotEqual(which, "/usr/bin/git")
|
| 136 | self.assertEqual(baseline_campaign.resolve_git_binary(argparse.Namespace()), expected)
|
| 137 |
|
| 138 | def test_manifest_pins_the_resolved_git_binary(self) -> None:
|
| 139 | # The manifest must hash and version the SAME git binary forwarded to
|
| 140 | # bench.py, not whatever `git` happens to be on PATH.
|
| 141 | with tempfile.TemporaryDirectory() as tmp:
|
| 142 | root = Path(tmp)
|
| 143 | fake_git = root / "fake-git"
|
| 144 | fake_git.write_text("#!/bin/sh\necho 'git version 9.9.9-test'\n")
|
| 145 | fake_git.chmod(0o755)
|
| 146 |
|
| 147 | self.assertEqual(
|
| 148 | baseline_campaign.git_version(fake_git), "git version 9.9.9-test"
|
| 149 | )
|
| 150 |
|
| 151 | rows = root / "rows.jsonl"
|
| 152 | rows.write_text("".join(json.dumps(row(i, elapsed=10.0 + i)) + "\n" for i in range(2)))
|
| 153 | out = root / "book"
|
| 154 | args = argparse.Namespace(
|
| 155 | command="book",
|
| 156 | book_id="git-bin-pin-test",
|
| 157 | out=out,
|
| 158 | lane="core",
|
| 159 | metrics=",".join(DEFAULT_BASELINE_METRICS),
|
| 160 | statistics=",".join(DEFAULT_BASELINE_STATISTICS),
|
| 161 | subjects=None,
|
| 162 | include_subops=False,
|
| 163 | bootstrap_samples=20,
|
| 164 | confidence=0.95,
|
| 165 | noise_floor_pct=3.5,
|
| 166 | raw_storage_key=[],
|
| 167 | skip_publish_gate=True,
|
| 168 | force=False,
|
| 169 | update_current=False,
|
| 170 | git_bin=fake_git,
|
| 171 | )
|
| 172 | rc = baseline_campaign.write_campaign_book(args, [rows])
|
| 173 | self.assertEqual(rc, 0)
|
| 174 | manifest = json.loads((out / "manifest.json").read_text())
|
| 175 | self.assertEqual(manifest["git_binary_sha256"], sha256_file(fake_git))
|
| 176 | self.assertEqual(manifest["git_version"], "git version 9.9.9-test")
|
| 177 | book = json.loads((out / "book.json").read_text())
|
| 178 | self.assertTrue(all(cell["noise_floor_pct"] == 3.5 for cell in book["cells"]))
|
| 179 |
|
| 180 |
|
| 181 | if __name__ == "__main__":
|
| 182 | unittest.main()
|