Log in
scripts/baseline_campaign.py 205 lines · 7.1 KB · blame Source
88e86b3faccd Rebuild benchmarks as a clean single-root reposi 2 months ago
1
#!/usr/bin/env python3
2
"""Run or materialize a Git Baseline Book campaign."""
3
4
from __future__ import annotations
5
6
import argparse
7
import json
8
import shutil
9
import subprocess
10
import sys
11
from pathlib import Path
12
13
ROOT = Path(__file__).resolve().parents[1]
14
sys.path.insert(0, str(ROOT / "scripts"))
15
16
import publish_gate  # noqa: E402
17
from oakbench.baseline_book import (  # noqa: E402
18
    DEFAULT_BASELINE_METRICS,
19
    DEFAULT_BASELINE_STATISTICS,
20
    book_json,
21
    book_markdown,
22
    cells_from_rows,
23
    iter_jsonl,
24
    manifest_json,
25
    sha256_file,
26
)
27
from oakbench.runner import machine_profile, runner_class, runner_id  # noqa: E402
28
29
30
def non_negative_float(raw: str) -> float:
31
    value = float(raw)
32
    if value < 0:
33
        raise argparse.ArgumentTypeError("must be >= 0")
34
    return value
35
36
37
def parse_args() -> argparse.Namespace:
38
    parser = argparse.ArgumentParser(description=__doc__)
39
    sub = parser.add_subparsers(dest="command", required=True)
40
41
    run = sub.add_parser("run", help="run bench.py with canonical Git baseline arguments, then write a book")
42
    add_common_book_args(run)
43
    run.add_argument("--profile", default="standard")
44
    run.add_argument("--track", default="core-equivalent")
45
    run.add_argument("--runs", type=int, default=30)
46
    run.add_argument("--results", type=Path, required=True)
47
    run.add_argument("--workdir", type=Path, required=True)
48
    run.add_argument("--git-bin", type=Path)
49
    run.add_argument("--git-modes", default="untracked_cache,fsmonitor")
50
    run.add_argument("--skip-remote", action="store_true")
51
52
    book = sub.add_parser("book", help="write a book from existing benchmark JSONL")
53
    add_common_book_args(book)
54
    book.add_argument("--input-jsonl", nargs="+", type=Path, required=True)
55
    return parser.parse_args()
56
57
58
def add_common_book_args(parser: argparse.ArgumentParser) -> None:
59
    parser.add_argument("--book-id", required=True)
60
    parser.add_argument("--out", type=Path, help="Output directory; defaults to baselines/git/<book-id>")
61
    parser.add_argument("--lane", default="core")
62
    parser.add_argument("--metrics", default=",".join(DEFAULT_BASELINE_METRICS))
63
    parser.add_argument("--statistics", default=",".join(DEFAULT_BASELINE_STATISTICS))
64
    parser.add_argument("--subjects", help="Comma-separated Git subject names to include in the book")
65
    parser.add_argument("--include-subops", action="store_true")
66
    parser.add_argument("--bootstrap-samples", type=int, default=2000)
67
    parser.add_argument("--confidence", type=float, default=0.95)
68
    parser.add_argument("--noise-floor-pct", type=non_negative_float)
69
    parser.add_argument("--raw-storage-key", action="append", default=[])
70
    parser.add_argument("--skip-publish-gate", action="store_true")
71
    parser.add_argument("--force", action="store_true")
72
    parser.add_argument("--update-current", action="store_true")
73
74
75
def split_csv(raw: str | None) -> tuple[str, ...]:
76
    if not raw:
77
        return ()
78
    return tuple(item.strip() for item in raw.split(",") if item.strip())
79
80
81
def resolve_git_binary(args: argparse.Namespace) -> Path | None:
82
    """Resolve the single git binary used for benching, pinning, and versioning.
83
84
    --git-bin wins when given; otherwise fall back to PATH lookup. Returning
85
    None means no git binary was found (manifest fields stay null).
86
    """
87
    git_bin = getattr(args, "git_bin", None)
88
    if git_bin:
89
        return Path(git_bin)
90
    found = shutil.which("git")
91
    return Path(found) if found else None
92
93
94
def main() -> int:
95
    args = parse_args()
96
    if args.command == "run":
97
        rc = run_bench(args)
98
        if rc != 0:
99
            return rc
100
        paths = [args.results / "latest.jsonl"]
101
    else:
102
        paths = args.input_jsonl
103
    return write_campaign_book(args, paths)
104
105
106
def run_bench(args: argparse.Namespace) -> int:
107
    cmd = [
108
        sys.executable,
109
        str(ROOT / "scripts" / "bench.py"),
110
        "--profile",
111
        args.profile,
112
        "--track",
113
        args.track,
114
        "--runs",
115
        str(args.runs),
116
        "--subjects",
117
        "git",
118
        "--git-modes",
119
        args.git_modes,
120
        "--randomize-subject-order",
121
        "--results",
122
        str(args.results),
123
        "--workdir",
124
        str(args.workdir),
125
    ]
126
    git_bin = resolve_git_binary(args)
127
    if git_bin:
128
        cmd.extend(["--git-bin", str(git_bin)])
129
    if args.skip_remote:
130
        cmd.append("--skip-remote")
131
    print("[run] " + " ".join(cmd), flush=True)
132
    return subprocess.run(cmd, cwd=ROOT, check=False).returncode
133
134
135
def write_campaign_book(args: argparse.Namespace, paths: list[Path]) -> int:
136
    out_dir = args.out or (ROOT / "baselines" / "git" / args.book_id)
137
    if out_dir.exists() and any(out_dir.iterdir()) and not args.force:
138
        print(f"refusing to overwrite existing baseline book directory: {out_dir}", file=sys.stderr)
139
        return 2
140
    if not args.skip_publish_gate:
141
        gate_rc = publish_gate.main([str(path) for path in paths])
142
        if gate_rc != 0:
143
            return gate_rc
144
145
    profile = machine_profile(ROOT)
146
    git_bin = resolve_git_binary(args)
147
    rows = iter_jsonl(paths)
148
    cells = cells_from_rows(
149
        rows,
150
        lane=args.lane,
151
        metrics=split_csv(args.metrics),
152
        statistics_names=split_csv(args.statistics),
153
        subjects=set(split_csv(args.subjects)) if args.subjects else None,
154
        include_subops=args.include_subops,
155
        bootstrap_samples=args.bootstrap_samples,
156
        confidence=args.confidence,
157
        noise_floor_pct=args.noise_floor_pct,
158
    )
159
    if not cells:
160
        print("no successful Git metric cells found in inputs", file=sys.stderr)
161
        return 1
162
163
    out_dir.mkdir(parents=True, exist_ok=True)
164
    manifest = manifest_json(
165
        book_id=args.book_id,
166
        row_paths=paths,
167
        runner_id=runner_id(profile),
168
        runner_class=runner_class(profile),
169
        runner_profile=profile,
170
        git_version=git_version(git_bin),
171
        git_binary_sha256=sha256_file(git_bin) if git_bin is not None and git_bin.is_file() else None,
172
        raw_storage_keys=list(args.raw_storage_key),
173
        suite_hash=suite_hash(),
174
    )
175
    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
176
    (out_dir / "book.json").write_text(json.dumps(book_json(cells), indent=2, sort_keys=True) + "\n")
177
    (out_dir / "book.md").write_text(book_markdown(args.book_id, cells))
178
    if args.update_current:
179
        current = ROOT / "baselines" / "git" / "CURRENT"
180
        current.parent.mkdir(parents=True, exist_ok=True)
181
        current.write_text(args.book_id + "\n")
182
    print(f"[baseline-book] {out_dir}")
183
    print(f"[cells] {len(cells)}")
184
    return 0
185
186
187
def git_version(git_bin: Path | None) -> str | None:
188
    if git_bin is None:
189
        return None
190
    return command_stdout([str(git_bin), "--version"])
191
192
193
def suite_hash() -> str | None:
194
    return command_stdout(["oak", "hash"])
195
196
197
def command_stdout(command: list[str]) -> str | None:
198
    try:
199
        return subprocess.check_output(command, cwd=ROOT, text=True, stderr=subprocess.DEVNULL, timeout=10).strip() or None
200
    except Exception:
201
        return None
202
203
204
if __name__ == "__main__":
205
    raise SystemExit(main())