88e86b3faccd
Rebuild benchmarks as a clean single-root reposi
2 months ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Run or materialize a Git Baseline Book campaign."""
|
| 3 |
|
| 4 | from __future__ import annotations
|
| 5 |
|
| 6 | import argparse
|
| 7 | import json
|
| 8 | import shutil
|
| 9 | import subprocess
|
| 10 | import sys
|
| 11 | from pathlib import Path
|
| 12 |
|
| 13 | ROOT = Path(__file__).resolve().parents[1]
|
| 14 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 15 |
|
| 16 | import publish_gate # noqa: E402
|
| 17 | from oakbench.baseline_book import ( # noqa: E402
|
| 18 | DEFAULT_BASELINE_METRICS,
|
| 19 | DEFAULT_BASELINE_STATISTICS,
|
| 20 | book_json,
|
| 21 | book_markdown,
|
| 22 | cells_from_rows,
|
| 23 | iter_jsonl,
|
| 24 | manifest_json,
|
| 25 | sha256_file,
|
| 26 | )
|
| 27 | from oakbench.runner import machine_profile, runner_class, runner_id # noqa: E402
|
| 28 |
|
| 29 |
|
| 30 | def non_negative_float(raw: str) -> float:
|
| 31 | value = float(raw)
|
| 32 | if value < 0:
|
| 33 | raise argparse.ArgumentTypeError("must be >= 0")
|
| 34 | return value
|
| 35 |
|
| 36 |
|
| 37 | def parse_args() -> argparse.Namespace:
|
| 38 | parser = argparse.ArgumentParser(description=__doc__)
|
| 39 | sub = parser.add_subparsers(dest="command", required=True)
|
| 40 |
|
| 41 | run = sub.add_parser("run", help="run bench.py with canonical Git baseline arguments, then write a book")
|
| 42 | add_common_book_args(run)
|
| 43 | run.add_argument("--profile", default="standard")
|
| 44 | run.add_argument("--track", default="core-equivalent")
|
| 45 | run.add_argument("--runs", type=int, default=30)
|
| 46 | run.add_argument("--results", type=Path, required=True)
|
| 47 | run.add_argument("--workdir", type=Path, required=True)
|
| 48 | run.add_argument("--git-bin", type=Path)
|
| 49 | run.add_argument("--git-modes", default="untracked_cache,fsmonitor")
|
| 50 | run.add_argument("--skip-remote", action="store_true")
|
| 51 |
|
| 52 | book = sub.add_parser("book", help="write a book from existing benchmark JSONL")
|
| 53 | add_common_book_args(book)
|
| 54 | book.add_argument("--input-jsonl", nargs="+", type=Path, required=True)
|
| 55 | return parser.parse_args()
|
| 56 |
|
| 57 |
|
| 58 | def add_common_book_args(parser: argparse.ArgumentParser) -> None:
|
| 59 | parser.add_argument("--book-id", required=True)
|
| 60 | parser.add_argument("--out", type=Path, help="Output directory; defaults to baselines/git/<book-id>")
|
| 61 | parser.add_argument("--lane", default="core")
|
| 62 | parser.add_argument("--metrics", default=",".join(DEFAULT_BASELINE_METRICS))
|
| 63 | parser.add_argument("--statistics", default=",".join(DEFAULT_BASELINE_STATISTICS))
|
| 64 | parser.add_argument("--subjects", help="Comma-separated Git subject names to include in the book")
|
| 65 | parser.add_argument("--include-subops", action="store_true")
|
| 66 | parser.add_argument("--bootstrap-samples", type=int, default=2000)
|
| 67 | parser.add_argument("--confidence", type=float, default=0.95)
|
| 68 | parser.add_argument("--noise-floor-pct", type=non_negative_float)
|
| 69 | parser.add_argument("--raw-storage-key", action="append", default=[])
|
| 70 | parser.add_argument("--skip-publish-gate", action="store_true")
|
| 71 | parser.add_argument("--force", action="store_true")
|
| 72 | parser.add_argument("--update-current", action="store_true")
|
| 73 |
|
| 74 |
|
| 75 | def split_csv(raw: str | None) -> tuple[str, ...]:
|
| 76 | if not raw:
|
| 77 | return ()
|
| 78 | return tuple(item.strip() for item in raw.split(",") if item.strip())
|
| 79 |
|
| 80 |
|
| 81 | def resolve_git_binary(args: argparse.Namespace) -> Path | None:
|
| 82 | """Resolve the single git binary used for benching, pinning, and versioning.
|
| 83 |
|
| 84 | --git-bin wins when given; otherwise fall back to PATH lookup. Returning
|
| 85 | None means no git binary was found (manifest fields stay null).
|
| 86 | """
|
| 87 | git_bin = getattr(args, "git_bin", None)
|
| 88 | if git_bin:
|
| 89 | return Path(git_bin)
|
| 90 | found = shutil.which("git")
|
| 91 | return Path(found) if found else None
|
| 92 |
|
| 93 |
|
| 94 | def main() -> int:
|
| 95 | args = parse_args()
|
| 96 | if args.command == "run":
|
| 97 | rc = run_bench(args)
|
| 98 | if rc != 0:
|
| 99 | return rc
|
| 100 | paths = [args.results / "latest.jsonl"]
|
| 101 | else:
|
| 102 | paths = args.input_jsonl
|
| 103 | return write_campaign_book(args, paths)
|
| 104 |
|
| 105 |
|
| 106 | def run_bench(args: argparse.Namespace) -> int:
|
| 107 | cmd = [
|
| 108 | sys.executable,
|
| 109 | str(ROOT / "scripts" / "bench.py"),
|
| 110 | "--profile",
|
| 111 | args.profile,
|
| 112 | "--track",
|
| 113 | args.track,
|
| 114 | "--runs",
|
| 115 | str(args.runs),
|
| 116 | "--subjects",
|
| 117 | "git",
|
| 118 | "--git-modes",
|
| 119 | args.git_modes,
|
| 120 | "--randomize-subject-order",
|
| 121 | "--results",
|
| 122 | str(args.results),
|
| 123 | "--workdir",
|
| 124 | str(args.workdir),
|
| 125 | ]
|
| 126 | git_bin = resolve_git_binary(args)
|
| 127 | if git_bin:
|
| 128 | cmd.extend(["--git-bin", str(git_bin)])
|
| 129 | if args.skip_remote:
|
| 130 | cmd.append("--skip-remote")
|
| 131 | print("[run] " + " ".join(cmd), flush=True)
|
| 132 | return subprocess.run(cmd, cwd=ROOT, check=False).returncode
|
| 133 |
|
| 134 |
|
| 135 | def write_campaign_book(args: argparse.Namespace, paths: list[Path]) -> int:
|
| 136 | out_dir = args.out or (ROOT / "baselines" / "git" / args.book_id)
|
| 137 | if out_dir.exists() and any(out_dir.iterdir()) and not args.force:
|
| 138 | print(f"refusing to overwrite existing baseline book directory: {out_dir}", file=sys.stderr)
|
| 139 | return 2
|
| 140 | if not args.skip_publish_gate:
|
| 141 | gate_rc = publish_gate.main([str(path) for path in paths])
|
| 142 | if gate_rc != 0:
|
| 143 | return gate_rc
|
| 144 |
|
| 145 | profile = machine_profile(ROOT)
|
| 146 | git_bin = resolve_git_binary(args)
|
| 147 | rows = iter_jsonl(paths)
|
| 148 | cells = cells_from_rows(
|
| 149 | rows,
|
| 150 | lane=args.lane,
|
| 151 | metrics=split_csv(args.metrics),
|
| 152 | statistics_names=split_csv(args.statistics),
|
| 153 | subjects=set(split_csv(args.subjects)) if args.subjects else None,
|
| 154 | include_subops=args.include_subops,
|
| 155 | bootstrap_samples=args.bootstrap_samples,
|
| 156 | confidence=args.confidence,
|
| 157 | noise_floor_pct=args.noise_floor_pct,
|
| 158 | )
|
| 159 | if not cells:
|
| 160 | print("no successful Git metric cells found in inputs", file=sys.stderr)
|
| 161 | return 1
|
| 162 |
|
| 163 | out_dir.mkdir(parents=True, exist_ok=True)
|
| 164 | manifest = manifest_json(
|
| 165 | book_id=args.book_id,
|
| 166 | row_paths=paths,
|
| 167 | runner_id=runner_id(profile),
|
| 168 | runner_class=runner_class(profile),
|
| 169 | runner_profile=profile,
|
| 170 | git_version=git_version(git_bin),
|
| 171 | git_binary_sha256=sha256_file(git_bin) if git_bin is not None and git_bin.is_file() else None,
|
| 172 | raw_storage_keys=list(args.raw_storage_key),
|
| 173 | suite_hash=suite_hash(),
|
| 174 | )
|
| 175 | (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
|
| 176 | (out_dir / "book.json").write_text(json.dumps(book_json(cells), indent=2, sort_keys=True) + "\n")
|
| 177 | (out_dir / "book.md").write_text(book_markdown(args.book_id, cells))
|
| 178 | if args.update_current:
|
| 179 | current = ROOT / "baselines" / "git" / "CURRENT"
|
| 180 | current.parent.mkdir(parents=True, exist_ok=True)
|
| 181 | current.write_text(args.book_id + "\n")
|
| 182 | print(f"[baseline-book] {out_dir}")
|
| 183 | print(f"[cells] {len(cells)}")
|
| 184 | return 0
|
| 185 |
|
| 186 |
|
| 187 | def git_version(git_bin: Path | None) -> str | None:
|
| 188 | if git_bin is None:
|
| 189 | return None
|
| 190 | return command_stdout([str(git_bin), "--version"])
|
| 191 |
|
| 192 |
|
| 193 | def suite_hash() -> str | None:
|
| 194 | return command_stdout(["oak", "hash"])
|
| 195 |
|
| 196 |
|
| 197 | def command_stdout(command: list[str]) -> str | None:
|
| 198 | try:
|
| 199 | return subprocess.check_output(command, cwd=ROOT, text=True, stderr=subprocess.DEVNULL, timeout=10).strip() or None
|
| 200 | except Exception:
|
| 201 | return None
|
| 202 |
|
| 203 |
|
| 204 | if __name__ == "__main__":
|
| 205 | raise SystemExit(main())
|