Viewing files at commit 2f5107c20e85Commit detailsView latest

Browse files
scripts/baseline_campaign.py 208 lines · 7.2 KB · python
1
#!/usr/bin/env python3
2
"""Run or materialize a Git Baseline Book campaign."""
3
4
from __future__ import annotations
5
6
import argparse
7
import json
8
import shutil
9
import subprocess
10
import sys
11
from pathlib import Path
12
13
ROOT = Path(__file__).resolve().parents[1]
14
sys.path.insert(0, str(ROOT / "scripts"))
15
16
import publish_gate  # noqa: E402
17
from oakbench.baseline_book import (  # noqa: E402
18
    DEFAULT_BASELINE_METRICS,
19
    DEFAULT_BASELINE_STATISTICS,
20
    book_json,
21
    book_markdown,
22
    cells_from_rows,
23
    iter_jsonl,
24
    manifest_json,
25
    sha256_file,
26
)
27
from oakbench.runner import machine_profile, runner_class, runner_id  # noqa: E402
28
29
30
def non_negative_float(raw: str) -> float:
31
    value = float(raw)
32
    if value < 0:
33
        raise argparse.ArgumentTypeError("must be >= 0")
34
    return value
35
36
37
def parse_args() -> argparse.Namespace:
38
    parser = argparse.ArgumentParser(description=__doc__)
39
    sub = parser.add_subparsers(dest="command", required=True)
40
41
    run = sub.add_parser("run", help="run bench.py with canonical Git baseline arguments, then write a book")
42
    add_common_book_args(run)
43
    run.add_argument("--profile", default="standard")
44
    run.add_argument("--track", default="core-equivalent")
45
    run.add_argument("--runs", type=int, default=30)
46
    run.add_argument("--results", type=Path, required=True)
47
    run.add_argument("--workdir", type=Path, required=True)
48
    run.add_argument("--git-bin", type=Path)
49
    run.add_argument("--git-modes", default="untracked_cache,fsmonitor")
50
    run.add_argument("--skip-remote", action="store_true")
51
52
    book = sub.add_parser("book", help="write a book from existing benchmark JSONL")
53
    add_common_book_args(book)
54
    book.add_argument("--input-jsonl", nargs="+", type=Path, required=True)
55
    return parser.parse_args()
56
57
58
def add_common_book_args(parser: argparse.ArgumentParser) -> None:
59
    parser.add_argument("--book-id", required=True)
60
    parser.add_argument("--out", type=Path, help="Output directory; defaults to baselines/git/<book-id>")
61
    parser.add_argument("--lane", default="core")
62
    parser.add_argument("--metrics", default=",".join(DEFAULT_BASELINE_METRICS))
63
    parser.add_argument("--statistics", default=",".join(DEFAULT_BASELINE_STATISTICS))
64
    parser.add_argument("--subjects", help="Comma-separated Git subject names to include in the book")
65
    parser.add_argument("--include-subops", action="store_true")
66
    parser.add_argument("--bootstrap-samples", type=int, default=2000)
67
    parser.add_argument("--confidence", type=float, default=0.95)
68
    parser.add_argument("--noise-floor-pct", type=non_negative_float)
69
    parser.add_argument("--raw-storage-key", action="append", default=[])
70
    parser.add_argument("--skip-publish-gate", action="store_true")
71
    parser.add_argument("--force", action="store_true")
72
    parser.add_argument("--update-current", action="store_true")
73
74
75
def split_csv(raw: str | None) -> tuple[str, ...]:
76
    if not raw:
77
        return ()
78
    return tuple(item.strip() for item in raw.split(",") if item.strip())
79
80
81
def resolve_git_binary(args: argparse.Namespace) -> Path | None:
82
    """Resolve the single git binary used for benching, pinning, and versioning.
83
84
    --git-bin wins when given; otherwise fall back to PATH lookup. Returning
85
    None means no git binary was found (manifest fields stay null).
86
    """
87
    git_bin = getattr(args, "git_bin", None)
88
    if git_bin:
89
        return Path(git_bin)
90
    from oakbench.subjects import which_measured_git
91
    found = which_measured_git()
92
    return Path(found) if found else None
93
94
95
def main() -> int:
96
    args = parse_args()
97
    if args.command == "run":
98
        rc = run_bench(args)
99
        if rc != 0:
100
            return rc
101
        paths = [args.results / "latest.jsonl"]
102
    else:
103
        paths = args.input_jsonl
104
    return write_campaign_book(args, paths)
105
106
107
def run_bench(args: argparse.Namespace) -> int:
108
    cmd = [
109
        sys.executable,
110
        str(ROOT / "scripts" / "bench.py"),
111
        "--profile",
112
        args.profile,
113
        "--track",
114
        args.track,
115
        "--runs",
116
        str(args.runs),
117
        "--subjects",
118
        # git_aa is the same Git under a second name: each book cell's noise
119
        # floor comes from its own interleaved A/A (ADR-0011).
120
        "git,git_aa",
121
        "--git-modes",
122
        args.git_modes,
123
        "--randomize-subject-order",
124
        "--results",
125
        str(args.results),
126
        "--workdir",
127
        str(args.workdir),
128
    ]
129
    git_bin = resolve_git_binary(args)
130
    if git_bin:
131
        cmd.extend(["--git-bin", str(git_bin)])
132
    if args.skip_remote:
133
        cmd.append("--skip-remote")
134
    print("[run] " + " ".join(cmd), flush=True)
135
    return subprocess.run(cmd, cwd=ROOT, check=False).returncode
136
137
138
def write_campaign_book(args: argparse.Namespace, paths: list[Path]) -> int:
139
    out_dir = args.out or (ROOT / "baselines" / "git" / args.book_id)
140
    if out_dir.exists() and any(out_dir.iterdir()) and not args.force:
141
        print(f"refusing to overwrite existing baseline book directory: {out_dir}", file=sys.stderr)
142
        return 2
143
    if not args.skip_publish_gate:
144
        gate_rc = publish_gate.main([str(path) for path in paths])
145
        if gate_rc != 0:
146
            return gate_rc
147
148
    profile = machine_profile(ROOT)
149
    git_bin = resolve_git_binary(args)
150
    rows = iter_jsonl(paths)
151
    cells = cells_from_rows(
152
        rows,
153
        lane=args.lane,
154
        metrics=split_csv(args.metrics),
155
        statistics_names=split_csv(args.statistics),
156
        subjects=set(split_csv(args.subjects)) if args.subjects else None,
157
        include_subops=args.include_subops,
158
        bootstrap_samples=args.bootstrap_samples,
159
        confidence=args.confidence,
160
        noise_floor_pct=args.noise_floor_pct,
161
    )
162
    if not cells:
163
        print("no successful Git metric cells found in inputs", file=sys.stderr)
164
        return 1
165
166
    out_dir.mkdir(parents=True, exist_ok=True)
167
    manifest = manifest_json(
168
        book_id=args.book_id,
169
        row_paths=paths,
170
        runner_id=runner_id(profile),
171
        runner_class=runner_class(profile),
172
        runner_profile=profile,
173
        git_version=git_version(git_bin),
174
        git_binary_sha256=sha256_file(git_bin) if git_bin is not None and git_bin.is_file() else None,
175
        raw_storage_keys=list(args.raw_storage_key),
176
        suite_hash=suite_hash(),
177
    )
178
    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
179
    (out_dir / "book.json").write_text(json.dumps(book_json(cells), indent=2, sort_keys=True) + "\n")
180
    (out_dir / "book.md").write_text(book_markdown(args.book_id, cells))
181
    if args.update_current:
182
        current = ROOT / "baselines" / "git" / "CURRENT"
183
        current.parent.mkdir(parents=True, exist_ok=True)
184
        current.write_text(args.book_id + "\n")
185
    print(f"[baseline-book] {out_dir}")
186
    print(f"[cells] {len(cells)}")
187
    return 0
188
189
190
def git_version(git_bin: Path | None) -> str | None:
191
    if git_bin is None:
192
        return None
193
    return command_stdout([str(git_bin), "--version"])
194
195
196
def suite_hash() -> str | None:
197
    return command_stdout(["oak", "hash"])
198
199
200
def command_stdout(command: list[str]) -> str | None:
201
    try:
202
        return subprocess.check_output(command, cwd=ROOT, text=True, stderr=subprocess.DEVNULL, timeout=10).strip() or None
203
    except Exception:
204
        return None
205
206
207
if __name__ == "__main__":
208
    raise SystemExit(main())