88e86b3faccd
Rebuild benchmarks as a clean single-root reposi
2 months ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Build a committed Git Baseline Book from benchmark JSONL rows."""
|
| 3 |
|
| 4 | from __future__ import annotations
|
| 5 |
|
| 6 | import argparse
|
| 7 | import json
|
| 8 | import sys
|
| 9 | from pathlib import Path
|
| 10 |
|
| 11 | ROOT = Path(__file__).resolve().parents[1]
|
| 12 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 13 |
|
| 14 | from oakbench.baseline_book import ( # noqa: E402
|
| 15 | DEFAULT_BASELINE_METRICS,
|
| 16 | DEFAULT_BASELINE_STATISTICS,
|
| 17 | book_json,
|
| 18 | book_markdown,
|
| 19 | cells_from_rows,
|
| 20 | iter_jsonl,
|
| 21 | manifest_json,
|
| 22 | )
|
| 23 | from oakbench.runner import machine_profile, runner_class, runner_id # noqa: E402
|
| 24 |
|
| 25 |
|
| 26 | def non_negative_float(raw: str) -> float:
|
| 27 | value = float(raw)
|
| 28 | if value < 0:
|
| 29 | raise argparse.ArgumentTypeError("must be >= 0")
|
| 30 | return value
|
| 31 |
|
| 32 |
|
| 33 | def parse_args() -> argparse.Namespace:
|
| 34 | parser = argparse.ArgumentParser(description=__doc__)
|
| 35 | parser.add_argument("jsonl", nargs="+", type=Path, help="Git-only or mixed benchmark row JSONL inputs")
|
| 36 | parser.add_argument("--book-id", required=True, help="Immutable baseline book id")
|
| 37 | parser.add_argument("--out", type=Path, help="Output directory; defaults to baselines/git/<book-id>")
|
| 38 | parser.add_argument("--lane", default="core")
|
| 39 | parser.add_argument("--metrics", default=",".join(DEFAULT_BASELINE_METRICS))
|
| 40 | parser.add_argument("--statistics", default=",".join(DEFAULT_BASELINE_STATISTICS))
|
| 41 | parser.add_argument("--subjects", help="Comma-separated Git subject names to include")
|
| 42 | parser.add_argument("--include-subops", action="store_true")
|
| 43 | parser.add_argument("--bootstrap-samples", type=int, default=2000)
|
| 44 | parser.add_argument("--confidence", type=float, default=0.95)
|
| 45 | parser.add_argument("--noise-floor-pct", type=non_negative_float)
|
| 46 | parser.add_argument("--runner-class")
|
| 47 | parser.add_argument("--git-version")
|
| 48 | parser.add_argument("--git-binary-sha256")
|
| 49 | parser.add_argument("--force", action="store_true", help="Overwrite an existing output directory")
|
| 50 | return parser.parse_args()
|
| 51 |
|
| 52 |
|
| 53 | def split_csv(raw: str) -> tuple[str, ...]:
|
| 54 | return tuple(item.strip() for item in raw.split(",") if item.strip())
|
| 55 |
|
| 56 |
|
| 57 | def main() -> int:
|
| 58 | args = parse_args()
|
| 59 | out_dir = args.out or (ROOT / "baselines" / "git" / args.book_id)
|
| 60 | if out_dir.exists() and any(out_dir.iterdir()) and not args.force:
|
| 61 | print(f"refusing to overwrite existing baseline book directory: {out_dir}", file=sys.stderr)
|
| 62 | return 2
|
| 63 | out_dir.mkdir(parents=True, exist_ok=True)
|
| 64 |
|
| 65 | rows = iter_jsonl(args.jsonl)
|
| 66 | subjects = set(split_csv(args.subjects)) if args.subjects else None
|
| 67 | cells = cells_from_rows(
|
| 68 | rows,
|
| 69 | lane=args.lane,
|
| 70 | metrics=split_csv(args.metrics),
|
| 71 | statistics_names=split_csv(args.statistics),
|
| 72 | subjects=subjects,
|
| 73 | include_subops=args.include_subops,
|
| 74 | bootstrap_samples=args.bootstrap_samples,
|
| 75 | confidence=args.confidence,
|
| 76 | noise_floor_pct=args.noise_floor_pct,
|
| 77 | )
|
| 78 | if not cells:
|
| 79 | print("no successful Git metric cells found in inputs", file=sys.stderr)
|
| 80 | return 1
|
| 81 |
|
| 82 | # Books must carry runner identity (ADR-0007); capture this host's profile
|
| 83 | # and let flags override only what the operator explicitly pins.
|
| 84 | profile = machine_profile(ROOT)
|
| 85 | manifest = manifest_json(
|
| 86 | book_id=args.book_id,
|
| 87 | row_paths=args.jsonl,
|
| 88 | runner_id=runner_id(profile),
|
| 89 | runner_class=args.runner_class or runner_class(profile),
|
| 90 | runner_profile=profile,
|
| 91 | git_version=args.git_version,
|
| 92 | git_binary_sha256=args.git_binary_sha256,
|
| 93 | )
|
| 94 | (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
|
| 95 | (out_dir / "book.json").write_text(json.dumps(book_json(cells), indent=2, sort_keys=True) + "\n")
|
| 96 | (out_dir / "book.md").write_text(book_markdown(args.book_id, cells))
|
| 97 | print(f"[baseline-book] {out_dir}")
|
| 98 | print(f"[cells] {len(cells)}")
|
| 99 | return 0
|
| 100 |
|
| 101 |
|
| 102 | if __name__ == "__main__":
|
| 103 | raise SystemExit(main())
|