88e86b3faccd
Rebuild benchmarks as a clean single-root reposi
2 months ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Build a repro bundle for a published drop.
|
| 3 |
|
| 4 | A repro bundle pins everything a third party needs to reproduce a run:
|
| 5 | suite revision, scenario/config hashes, fixture seeds + generator versions,
|
| 6 | subject binary SHA256s, runner profile, noise floor, and raw-JSONL hashes
|
| 7 | (plus optional R2 storage keys). The bundle carries hashes of the raw rows,
|
| 8 | not the rows themselves.
|
| 9 |
|
| 10 | Usage:
|
| 11 |
|
| 12 | python3 scripts/repro_bundle.py results/rows.jsonl ... \
|
| 13 | --out bundle.json --suite-revision <oak-hash> \
|
| 14 | [--noise-floor-pct 4.2] [--r2-key raw/rows.jsonl ...]
|
| 15 |
|
| 16 | Null means unmeasured (ADR-0002): an absent noise floor is recorded as null,
|
| 17 | never 0.
|
| 18 | """
|
| 19 |
|
| 20 | from __future__ import annotations
|
| 21 |
|
| 22 | import argparse
|
| 23 | import json
|
| 24 | import sys
|
| 25 | from datetime import datetime, timezone
|
| 26 | from pathlib import Path
|
| 27 | from typing import Any
|
| 28 |
|
| 29 | ROOT = Path(__file__).resolve().parents[1]
|
| 30 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 31 |
|
| 32 | from oakbench.baseline_book import sha256_file # noqa: E402
|
| 33 | from oakbench.fixture_registry import load_fixture_registry # noqa: E402
|
| 34 | from oakbench.runner import machine_profile, runner_class, runner_id # noqa: E402
|
| 35 |
|
| 36 | SCHEMA_VERSION = 1
|
| 37 | CONTENT_INTEGRITY_SOURCE_STRENGTH = {
|
| 38 | "git_head_payload_sha256": 3,
|
| 39 | "git_lfs_worktree_payload_sha256": 1,
|
| 40 | "worktree_payload_sha256": 1,
|
| 41 | "mixed_payload_sources": 1,
|
| 42 | }
|
| 43 |
|
| 44 |
|
| 45 | def hash_directory(directory: Path, pattern: str, root: Path) -> dict[str, str]:
|
| 46 | """SHA256 every file matching ``pattern`` under ``directory``."""
|
| 47 | hashes: dict[str, str] = {}
|
| 48 | if directory.is_dir():
|
| 49 | for path in sorted(directory.glob(pattern)):
|
| 50 | if path.is_file():
|
| 51 | hashes[path.relative_to(root).as_posix()] = sha256_file(path)
|
| 52 | return hashes
|
| 53 |
|
| 54 |
|
| 55 | def count_jsonl_rows(path: Path) -> int:
|
| 56 | count = 0
|
| 57 | with path.open() as fh:
|
| 58 | for line in fh:
|
| 59 | if line.strip():
|
| 60 | count += 1
|
| 61 | return count
|
| 62 |
|
| 63 |
|
| 64 | def iter_jsonl_rows(paths: list[Path]) -> list[dict[str, Any]]:
|
| 65 | rows: list[dict[str, Any]] = []
|
| 66 | for path in paths:
|
| 67 | with path.open() as fh:
|
| 68 | for raw_line in fh:
|
| 69 | line = raw_line.strip()
|
| 70 | if line:
|
| 71 | rows.append(json.loads(line))
|
| 72 | return rows
|
| 73 |
|
| 74 |
|
| 75 | def subject_binaries_from_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
| 76 | """Map subject -> binary SHA256 collected from rows' binary_sha256 fields.
|
| 77 |
|
| 78 | A subject that recorded more than one distinct binary hash keeps the full
|
| 79 | sorted list β silently picking one would hide a mixed-binary run.
|
| 80 | """
|
| 81 | found: dict[str, set[str]] = {}
|
| 82 | for row in rows:
|
| 83 | subject = row.get("subject")
|
| 84 | sha = row.get("binary_sha256")
|
| 85 | if isinstance(subject, str) and subject and isinstance(sha, str) and sha:
|
| 86 | found.setdefault(subject, set()).add(sha)
|
| 87 | result: dict[str, Any] = {}
|
| 88 | for subject in sorted(found):
|
| 89 | ordered = sorted(found[subject])
|
| 90 | result[subject] = ordered[0] if len(ordered) == 1 else ordered
|
| 91 | return result
|
| 92 |
|
| 93 |
|
| 94 | def content_integrity_summary_from_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
| 95 | """Summarize content-attestation strength by comparable row group.
|
| 96 |
|
| 97 | The boolean content_integrity_check_passed says only whether the row's
|
| 98 | chosen oracle passed. Publication/replay decisions also need the source
|
| 99 | strength: HEAD-payload attestation is stronger than worktree/LFS-worktree
|
| 100 | attestation, and mixed groups must be treated as their weakest source.
|
| 101 | """
|
| 102 | grouped: dict[str, dict[str, Any]] = {}
|
| 103 | for row in rows:
|
| 104 | source = row.get("content_integrity_source")
|
| 105 | sources = row.get("content_integrity_sources")
|
| 106 | if not isinstance(source, str) and not isinstance(sources, list):
|
| 107 | continue
|
| 108 | profile = row.get("profile")
|
| 109 | key = "/".join(
|
| 110 | [
|
| 111 | "-" if profile is None else str(profile),
|
| 112 | str(row.get("scenario") or ""),
|
| 113 | str(row.get("operation") or ""),
|
| 114 | str(row.get("subject") or ""),
|
| 115 | ]
|
| 116 | )
|
| 117 | entry = grouped.setdefault(
|
| 118 | key,
|
| 119 | {
|
| 120 | "sources": set(),
|
| 121 | "rows": 0,
|
| 122 | "passed": 0,
|
| 123 | "failed": 0,
|
| 124 | "weakest_source_strength": None,
|
| 125 | },
|
| 126 | )
|
| 127 | row_sources = [value for value in ([source] + sources if isinstance(sources, list) else [source]) if isinstance(value, str)]
|
| 128 | for value in row_sources:
|
| 129 | entry["sources"].add(value)
|
| 130 | strength = CONTENT_INTEGRITY_SOURCE_STRENGTH.get(value, 0)
|
| 131 | current = entry["weakest_source_strength"]
|
| 132 | entry["weakest_source_strength"] = strength if current is None else min(current, strength)
|
| 133 | entry["rows"] += 1
|
| 134 | if row.get("content_integrity_check_passed") is True:
|
| 135 | entry["passed"] += 1
|
| 136 | elif row.get("content_integrity_check_passed") is False:
|
| 137 | entry["failed"] += 1
|
| 138 |
|
| 139 | return {
|
| 140 | key: {
|
| 141 | "sources": sorted(value["sources"]),
|
| 142 | "rows": value["rows"],
|
| 143 | "passed": value["passed"],
|
| 144 | "failed": value["failed"],
|
| 145 | "weakest_source_strength": value["weakest_source_strength"],
|
| 146 | }
|
| 147 | for key, value in sorted(grouped.items())
|
| 148 | }
|
| 149 |
|
| 150 |
|
| 151 | def build_bundle(
|
| 152 | row_paths: list[Path],
|
| 153 | *,
|
| 154 | suite_revision: str,
|
| 155 | noise_floor_pct: float | None = None,
|
| 156 | r2_keys: list[str] | None = None,
|
| 157 | root: Path = ROOT,
|
| 158 | ) -> dict[str, Any]:
|
| 159 | profile = machine_profile()
|
| 160 | registry = load_fixture_registry(root / "config" / "fixtures.json")
|
| 161 | rows = iter_jsonl_rows(row_paths)
|
| 162 | return {
|
| 163 | "schema_version": SCHEMA_VERSION,
|
| 164 | "created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
| 165 | "suite_revision": suite_revision,
|
| 166 | "runner_profile": profile,
|
| 167 | "runner_id": runner_id(profile),
|
| 168 | "runner_class": runner_class(profile),
|
| 169 | "noise_floor_pct": noise_floor_pct,
|
| 170 | "config_hashes": {
|
| 171 | **hash_directory(root / "config", "**/*.json", root),
|
| 172 | **hash_directory(root / "config", "**/*.toml", root),
|
| 173 | },
|
| 174 | "scenario_hashes": hash_directory(root / "scenarios", "*.yaml", root),
|
| 175 | "fixtures": [
|
| 176 | {
|
| 177 | "id": spec.fixture_id,
|
| 178 | "seed": spec.seed,
|
| 179 | "generator": spec.generator,
|
| 180 | "version": spec.version,
|
| 181 | }
|
| 182 | for _, spec in sorted(registry.items())
|
| 183 | ],
|
| 184 | "subject_binaries": subject_binaries_from_rows(rows),
|
| 185 | "content_integrity": {
|
| 186 | "source_strength": dict(sorted(CONTENT_INTEGRITY_SOURCE_STRENGTH.items())),
|
| 187 | "groups": content_integrity_summary_from_rows(rows),
|
| 188 | },
|
| 189 | "raw_inputs": [
|
| 190 | {
|
| 191 | "path": str(path),
|
| 192 | "sha256": sha256_file(path),
|
| 193 | "bytes": path.stat().st_size,
|
| 194 | "rows": count_jsonl_rows(path),
|
| 195 | }
|
| 196 | for path in row_paths
|
| 197 | ],
|
| 198 | "r2_keys": list(r2_keys or []),
|
| 199 | }
|
| 200 |
|
| 201 |
|
| 202 | def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
| 203 | parser = argparse.ArgumentParser(description="Build a repro bundle for a published drop.")
|
| 204 | parser.add_argument("rows", nargs="+", type=Path, help="raw row JSONL paths covered by the drop")
|
| 205 | parser.add_argument("--out", type=Path, required=True, help="output bundle JSON path")
|
| 206 | parser.add_argument(
|
| 207 | "--suite-revision",
|
| 208 | required=True,
|
| 209 | help="caller-supplied suite revision (e.g. the oak hash of the harness checkout)",
|
| 210 | )
|
| 211 | parser.add_argument(
|
| 212 | "--noise-floor-pct",
|
| 213 | type=float,
|
| 214 | default=None,
|
| 215 | help="measured same-binary noise floor percentage; omitted means unmeasured (null)",
|
| 216 | )
|
| 217 | parser.add_argument(
|
| 218 | "--r2-key",
|
| 219 | action="append",
|
| 220 | default=None,
|
| 221 | dest="r2_keys",
|
| 222 | help="R2 storage key for an uploaded raw artifact (repeatable)",
|
| 223 | )
|
| 224 | return parser.parse_args(argv)
|
| 225 |
|
| 226 |
|
| 227 | def main(argv: list[str] | None = None) -> int:
|
| 228 | args = parse_args(argv)
|
| 229 | missing = [str(path) for path in args.rows if not path.is_file()]
|
| 230 | if missing:
|
| 231 | print(f"error: row file(s) not found: {', '.join(missing)}", file=sys.stderr)
|
| 232 | return 2
|
| 233 | bundle = build_bundle(
|
| 234 | args.rows,
|
| 235 | suite_revision=args.suite_revision,
|
| 236 | noise_floor_pct=args.noise_floor_pct,
|
| 237 | r2_keys=args.r2_keys,
|
| 238 | )
|
| 239 | args.out.parent.mkdir(parents=True, exist_ok=True)
|
| 240 | args.out.write_text(json.dumps(bundle, indent=2, sort_keys=True) + "\n")
|
| 241 | print(
|
| 242 | f"wrote {args.out}: {len(bundle['raw_inputs'])} raw input(s), "
|
| 243 | f"{len(bundle['config_hashes'])} config hash(es), "
|
| 244 | f"{len(bundle['scenario_hashes'])} scenario hash(es), "
|
| 245 | f"{len(bundle['fixtures'])} fixture(s)"
|
| 246 | )
|
| 247 | return 0
|
| 248 |
|
| 249 |
|
| 250 | if __name__ == "__main__":
|
| 251 | sys.exit(main())
|