Log in
scripts/baseline_book.py 103 lines · 3.9 KB · python Blame
1
#!/usr/bin/env python3
2
"""Build a committed Git Baseline Book from benchmark JSONL rows."""
3
4
from __future__ import annotations
5
6
import argparse
7
import json
8
import sys
9
from pathlib import Path
10
11
ROOT = Path(__file__).resolve().parents[1]
12
sys.path.insert(0, str(ROOT / "scripts"))
13
14
from oakbench.baseline_book import (  # noqa: E402
15
    DEFAULT_BASELINE_METRICS,
16
    DEFAULT_BASELINE_STATISTICS,
17
    book_json,
18
    book_markdown,
19
    cells_from_rows,
20
    iter_jsonl,
21
    manifest_json,
22
)
23
from oakbench.runner import machine_profile, runner_class, runner_id  # noqa: E402
24
25
26
def non_negative_float(raw: str) -> float:
27
    value = float(raw)
28
    if value < 0:
29
        raise argparse.ArgumentTypeError("must be >= 0")
30
    return value
31
32
33
def parse_args() -> argparse.Namespace:
34
    parser = argparse.ArgumentParser(description=__doc__)
35
    parser.add_argument("jsonl", nargs="+", type=Path, help="Git-only or mixed benchmark row JSONL inputs")
36
    parser.add_argument("--book-id", required=True, help="Immutable baseline book id")
37
    parser.add_argument("--out", type=Path, help="Output directory; defaults to baselines/git/<book-id>")
38
    parser.add_argument("--lane", default="core")
39
    parser.add_argument("--metrics", default=",".join(DEFAULT_BASELINE_METRICS))
40
    parser.add_argument("--statistics", default=",".join(DEFAULT_BASELINE_STATISTICS))
41
    parser.add_argument("--subjects", help="Comma-separated Git subject names to include")
42
    parser.add_argument("--include-subops", action="store_true")
43
    parser.add_argument("--bootstrap-samples", type=int, default=2000)
44
    parser.add_argument("--confidence", type=float, default=0.95)
45
    parser.add_argument("--noise-floor-pct", type=non_negative_float)
46
    parser.add_argument("--runner-class")
47
    parser.add_argument("--git-version")
48
    parser.add_argument("--git-binary-sha256")
49
    parser.add_argument("--force", action="store_true", help="Overwrite an existing output directory")
50
    return parser.parse_args()
51
52
53
def split_csv(raw: str) -> tuple[str, ...]:
54
    return tuple(item.strip() for item in raw.split(",") if item.strip())
55
56
57
def main() -> int:
58
    args = parse_args()
59
    out_dir = args.out or (ROOT / "baselines" / "git" / args.book_id)
60
    if out_dir.exists() and any(out_dir.iterdir()) and not args.force:
61
        print(f"refusing to overwrite existing baseline book directory: {out_dir}", file=sys.stderr)
62
        return 2
63
    out_dir.mkdir(parents=True, exist_ok=True)
64
65
    rows = iter_jsonl(args.jsonl)
66
    subjects = set(split_csv(args.subjects)) if args.subjects else None
67
    cells = cells_from_rows(
68
        rows,
69
        lane=args.lane,
70
        metrics=split_csv(args.metrics),
71
        statistics_names=split_csv(args.statistics),
72
        subjects=subjects,
73
        include_subops=args.include_subops,
74
        bootstrap_samples=args.bootstrap_samples,
75
        confidence=args.confidence,
76
        noise_floor_pct=args.noise_floor_pct,
77
    )
78
    if not cells:
79
        print("no successful Git metric cells found in inputs", file=sys.stderr)
80
        return 1
81
82
    # Books must carry runner identity (ADR-0007); capture this host's profile
83
    # and let flags override only what the operator explicitly pins.
84
    profile = machine_profile(ROOT)
85
    manifest = manifest_json(
86
        book_id=args.book_id,
87
        row_paths=args.jsonl,
88
        runner_id=runner_id(profile),
89
        runner_class=args.runner_class or runner_class(profile),
90
        runner_profile=profile,
91
        git_version=args.git_version,
92
        git_binary_sha256=args.git_binary_sha256,
93
    )
94
    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
95
    (out_dir / "book.json").write_text(json.dumps(book_json(cells), indent=2, sort_keys=True) + "\n")
96
    (out_dir / "book.md").write_text(book_markdown(args.book_id, cells))
97
    print(f"[baseline-book] {out_dir}")
98
    print(f"[cells] {len(cells)}")
99
    return 0
100
101
102
if __name__ == "__main__":
103
    raise SystemExit(main())