| 1 | """Tests for repro bundle construction."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import json
|
| 6 | import sys
|
| 7 | import tempfile
|
| 8 | import unittest
|
| 9 | from pathlib import Path
|
| 10 |
|
| 11 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 12 |
|
| 13 | import repro_bundle
|
| 14 | from oakbench.baseline_book import sha256_file
|
| 15 |
|
| 16 | ROOT = Path(__file__).resolve().parents[1]
|
| 17 |
|
| 18 | OAK_BINARY_SHA = "a" * 64
|
| 19 | GIT_BINARY_SHA = "b" * 64
|
| 20 |
|
| 21 |
|
| 22 | def make_row(subject: str, run: int, elapsed: float) -> dict:
|
| 23 | return {
|
| 24 | "bench_id": "bundle-test",
|
| 25 | "profile": "standard",
|
| 26 | "scenario": "core-small",
|
| 27 | "operation": "status",
|
| 28 | "run": run,
|
| 29 | "subject": subject,
|
| 30 | "subject_kind": "oak" if subject == "oak" else "git",
|
| 31 | "elapsed_ms": elapsed,
|
| 32 | "returncode": 0,
|
| 33 | "command": [subject, "status"],
|
| 34 | "estimated_tokens_total": 120,
|
| 35 | "raw_output_bytes": 400,
|
| 36 | "tool_call_count": 3,
|
| 37 | "binary_sha256": OAK_BINARY_SHA if subject == "oak" else GIT_BINARY_SHA,
|
| 38 | }
|
| 39 |
|
| 40 |
|
| 41 | def write_rows(path: Path, rows: list[dict]) -> Path:
|
| 42 | path.write_text("\n".join(json.dumps(row) for row in rows) + "\n")
|
| 43 | return path
|
| 44 |
|
| 45 |
|
| 46 | class ReproBundleTest(unittest.TestCase):
|
| 47 | def build(self, tmp: Path, extra_args: list[str] | None = None) -> dict:
|
| 48 | rows = [make_row(subject, run, 10.0 + run) for subject in ("oak", "git") for run in range(3)]
|
| 49 | rows_path = write_rows(tmp / "rows.jsonl", rows)
|
| 50 | out_path = tmp / "bundle.json"
|
| 51 | argv = [
|
| 52 | str(rows_path),
|
| 53 | "--out",
|
| 54 | str(out_path),
|
| 55 | "--suite-revision",
|
| 56 | "deadbeefcafe",
|
| 57 | ] + (extra_args or [])
|
| 58 | self.assertEqual(repro_bundle.main(argv), 0)
|
| 59 | return json.loads(out_path.read_text())
|
| 60 |
|
| 61 | def test_bundle_shape_and_hashes(self) -> None:
|
| 62 | with tempfile.TemporaryDirectory() as tmp_str:
|
| 63 | tmp = Path(tmp_str)
|
| 64 | bundle = self.build(tmp)
|
| 65 |
|
| 66 | self.assertEqual(bundle["schema_version"], 1)
|
| 67 | self.assertEqual(bundle["suite_revision"], "deadbeefcafe")
|
| 68 | self.assertIn("created_utc", bundle)
|
| 69 |
|
| 70 | # Runner identity comes from the live machine profile.
|
| 71 | self.assertIsInstance(bundle["runner_profile"], dict)
|
| 72 | self.assertEqual(bundle["runner_profile"].get("schema_version"), 1)
|
| 73 | self.assertRegex(bundle["runner_id"], r"^[0-9a-f]{16}$")
|
| 74 | self.assertIsInstance(bundle["runner_class"], str)
|
| 75 |
|
| 76 | # Noise floor was not measured: null, never 0 (ADR-0002).
|
| 77 | self.assertIsNone(bundle["noise_floor_pct"])
|
| 78 | self.assertEqual(bundle["r2_keys"], [])
|
| 79 |
|
| 80 | # config_hashes covers every config/**/*.json and config/**/*.toml file with a recorded
|
| 81 | # sha256 that matches an independently computed digest.
|
| 82 | self.assertTrue(bundle["config_hashes"])
|
| 83 | expected_configs = {
|
| 84 | path.relative_to(ROOT).as_posix(): sha256_file(path)
|
| 85 | for path in sorted((ROOT / "config").glob("**/*.json"))
|
| 86 | }
|
| 87 | expected_configs.update(
|
| 88 | {
|
| 89 | path.relative_to(ROOT).as_posix(): sha256_file(path)
|
| 90 | for path in sorted((ROOT / "config").glob("**/*.toml"))
|
| 91 | }
|
| 92 | )
|
| 93 | self.assertEqual(bundle["config_hashes"], expected_configs)
|
| 94 | self.assertIn("config/fixtures.json", bundle["config_hashes"])
|
| 95 | self.assertIn("config/subjects.toml", bundle["config_hashes"])
|
| 96 | self.assertIn("config/agents.toml", bundle["config_hashes"])
|
| 97 | self.assertIn("config/subject_kinds/oak.json", bundle["config_hashes"])
|
| 98 | self.assertEqual(
|
| 99 | bundle["config_hashes"]["config/fixtures.json"],
|
| 100 | sha256_file(ROOT / "config" / "fixtures.json"),
|
| 101 | )
|
| 102 |
|
| 103 | # scenario_hashes covers scenarios/*.yaml.
|
| 104 | expected_scenarios = {
|
| 105 | f"scenarios/{path.name}": sha256_file(path) for path in sorted((ROOT / "scenarios").glob("*.yaml"))
|
| 106 | }
|
| 107 | self.assertEqual(bundle["scenario_hashes"], expected_scenarios)
|
| 108 | self.assertIn("scenarios/core.yaml", bundle["scenario_hashes"])
|
| 109 |
|
| 110 | def test_fixtures_pin_seed_generator_version(self) -> None:
|
| 111 | with tempfile.TemporaryDirectory() as tmp_str:
|
| 112 | bundle = self.build(Path(tmp_str))
|
| 113 | fixtures = {entry["id"]: entry for entry in bundle["fixtures"]}
|
| 114 | registry = json.loads((ROOT / "config" / "fixtures.json").read_text())["fixtures"]
|
| 115 | self.assertEqual(set(fixtures), set(registry))
|
| 116 | large = fixtures["large-mixed-v1"]
|
| 117 | self.assertEqual(large["seed"], "oakbench-large-v1")
|
| 118 | self.assertEqual(large["generator"], "scripts/make_large_fixture.py")
|
| 119 | self.assertEqual(large["version"], "v1")
|
| 120 | for entry in bundle["fixtures"]:
|
| 121 | self.assertEqual(set(entry), {"id", "seed", "generator", "version"})
|
| 122 |
|
| 123 | def test_subject_binaries_and_raw_inputs(self) -> None:
|
| 124 | with tempfile.TemporaryDirectory() as tmp_str:
|
| 125 | tmp = Path(tmp_str)
|
| 126 | bundle = self.build(tmp)
|
| 127 |
|
| 128 | self.assertEqual(
|
| 129 | bundle["subject_binaries"],
|
| 130 | {"git": GIT_BINARY_SHA, "oak": OAK_BINARY_SHA},
|
| 131 | )
|
| 132 |
|
| 133 | rows_path = tmp / "rows.jsonl"
|
| 134 | self.assertEqual(len(bundle["raw_inputs"]), 1)
|
| 135 | raw = bundle["raw_inputs"][0]
|
| 136 | self.assertEqual(raw["path"], str(rows_path))
|
| 137 | self.assertEqual(raw["sha256"], sha256_file(rows_path))
|
| 138 | self.assertEqual(raw["bytes"], rows_path.stat().st_size)
|
| 139 | self.assertEqual(raw["rows"], 6)
|
| 140 |
|
| 141 | def test_content_integrity_summary_records_weakest_source(self) -> None:
|
| 142 | with tempfile.TemporaryDirectory() as tmp_str:
|
| 143 | tmp = Path(tmp_str)
|
| 144 | rows = [
|
| 145 | dict(
|
| 146 | make_row("oak", 0, 10.0),
|
| 147 | content_integrity_check_passed=True,
|
| 148 | content_integrity_source="git_head_payload_sha256",
|
| 149 | ),
|
| 150 | dict(
|
| 151 | make_row("oak", 1, 11.0),
|
| 152 | content_integrity_check_passed=True,
|
| 153 | content_integrity_source="mixed_payload_sources",
|
| 154 | content_integrity_sources=["git_head_payload_sha256", "git_lfs_worktree_payload_sha256"],
|
| 155 | ),
|
| 156 | ]
|
| 157 | rows_path = write_rows(tmp / "rows.jsonl", rows)
|
| 158 | out_path = tmp / "bundle.json"
|
| 159 | self.assertEqual(
|
| 160 | repro_bundle.main([str(rows_path), "--out", str(out_path), "--suite-revision", "rev"]),
|
| 161 | 0,
|
| 162 | )
|
| 163 | bundle = json.loads(out_path.read_text())
|
| 164 | group = bundle["content_integrity"]["groups"]["standard/core-small/status/oak"]
|
| 165 | self.assertEqual(
|
| 166 | group["sources"],
|
| 167 | ["git_head_payload_sha256", "git_lfs_worktree_payload_sha256", "mixed_payload_sources"],
|
| 168 | )
|
| 169 | self.assertEqual(group["weakest_source_strength"], 1)
|
| 170 | self.assertEqual(group["passed"], 2)
|
| 171 |
|
| 172 | def test_rows_without_binary_sha_leave_subject_binaries_empty(self) -> None:
|
| 173 | with tempfile.TemporaryDirectory() as tmp_str:
|
| 174 | tmp = Path(tmp_str)
|
| 175 | rows = [make_row("oak", run, 10.0) for run in range(2)]
|
| 176 | for row in rows:
|
| 177 | row.pop("binary_sha256")
|
| 178 | rows_path = write_rows(tmp / "rows.jsonl", rows)
|
| 179 | out_path = tmp / "bundle.json"
|
| 180 | self.assertEqual(
|
| 181 | repro_bundle.main(
|
| 182 | [str(rows_path), "--out", str(out_path), "--suite-revision", "rev"]
|
| 183 | ),
|
| 184 | 0,
|
| 185 | )
|
| 186 | bundle = json.loads(out_path.read_text())
|
| 187 | self.assertEqual(bundle["subject_binaries"], {})
|
| 188 |
|
| 189 | def test_optional_noise_floor_and_r2_keys(self) -> None:
|
| 190 | with tempfile.TemporaryDirectory() as tmp_str:
|
| 191 | bundle = self.build(
|
| 192 | Path(tmp_str),
|
| 193 | [
|
| 194 | "--noise-floor-pct",
|
| 195 | "4.2",
|
| 196 | "--r2-key",
|
| 197 | "raw/drop-1/rows.jsonl",
|
| 198 | "--r2-key",
|
| 199 | "raw/drop-1/rows.mount.jsonl",
|
| 200 | ],
|
| 201 | )
|
| 202 | self.assertEqual(bundle["noise_floor_pct"], 4.2)
|
| 203 | self.assertEqual(
|
| 204 | bundle["r2_keys"],
|
| 205 | ["raw/drop-1/rows.jsonl", "raw/drop-1/rows.mount.jsonl"],
|
| 206 | )
|
| 207 |
|
| 208 | def test_missing_row_file_is_usage_error(self) -> None:
|
| 209 | with tempfile.TemporaryDirectory() as tmp_str:
|
| 210 | tmp = Path(tmp_str)
|
| 211 | self.assertEqual(
|
| 212 | repro_bundle.main(
|
| 213 | [str(tmp / "nope.jsonl"), "--out", str(tmp / "b.json"), "--suite-revision", "rev"]
|
| 214 | ),
|
| 215 | 2,
|
| 216 | )
|
| 217 |
|
| 218 |
|
| 219 | if __name__ == "__main__":
|
| 220 | unittest.main()
|