88e86b3faccd
Rebuild benchmarks as a clean single-root reposi
2 months ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Run the branch-triage-shape correctness lane over a local git fixture.
|
| 3 |
|
| 4 | Ensures (or generates) the branch triage fixture, probes Oak for branch review
|
| 5 | or batch triage JSON support, scores measured rows against BRANCH_MANIFEST.json,
|
| 6 | and emits task-loop lane rows. Missing Oak capability yields explicit skip rows
|
| 7 | (returncode 77) with null metrics β never a fabricated pass.
|
| 8 | """
|
| 9 |
|
| 10 | from __future__ import annotations
|
| 11 |
|
| 12 | import argparse
|
| 13 | import json
|
| 14 | import shutil
|
| 15 | import subprocess
|
| 16 | import sys
|
| 17 | import tempfile
|
| 18 | from datetime import datetime, timezone
|
| 19 | from pathlib import Path
|
| 20 | from typing import Any
|
| 21 |
|
| 22 | ROOT = Path(__file__).resolve().parents[1]
|
| 23 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 24 |
|
| 25 | from oakbench import environment as oakbench_environment
|
| 26 | from oakbench import platform_clock
|
| 27 | from oakbench.branch_triage_shape import (
|
| 28 | MANIFEST_FILENAME,
|
| 29 | TRIAGE_METRIC_KEYS,
|
| 30 | load_manifest,
|
| 31 | score_manifest_against_actuals,
|
| 32 | triage_metrics_row_fields,
|
| 33 | unmeasured_triage_metrics,
|
| 34 | validate_manifest,
|
| 35 | validate_triage_metrics,
|
| 36 | )
|
| 37 | from oakbench.fixture_registry import load_fixture_registry
|
| 38 | from oakbench.results import ResultsStore
|
| 39 | from oakbench.rows import SKIP_RETURNCODE, validate_rows
|
| 40 |
|
| 41 | GENERATOR = ROOT / "scripts" / "make_branch_triage_fixture.py"
|
| 42 | DEFAULT_WORKDIR = Path(tempfile.gettempdir()) / "oak-branch-triage-fixture"
|
| 43 | DEFAULT_RESULTS = ROOT / "results" / "branch-triage-fixture"
|
| 44 | PROFILE = "branch-triage-fixture"
|
| 45 | FIXTURE_ID = "bench-branch-triage-shape-v2"
|
| 46 | SCENARIO = "branch_triage_shape"
|
| 47 | MEASUREMENT_SOURCE = "direct_cli_timed_subprocess"
|
| 48 | DEFAULT_SEED = "oakbench-branch-triage-v2"
|
| 49 | DEFAULT_BRANCH_COUNT = 6
|
| 50 |
|
| 51 |
|
| 52 | def utc_bench_id() -> str:
|
| 53 | return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
| 54 |
|
| 55 |
|
| 56 | def ensure_fixture(workdir: Path, seed: str, branch_count: int) -> Path:
|
| 57 | fixture_root = workdir / f"{seed}-n{branch_count}"
|
| 58 | manifest_path = fixture_root / MANIFEST_FILENAME
|
| 59 | if not manifest_path.is_file():
|
| 60 | proc = subprocess.run(
|
| 61 | [
|
| 62 | sys.executable,
|
| 63 | str(GENERATOR),
|
| 64 | "--out",
|
| 65 | str(fixture_root),
|
| 66 | "--seed",
|
| 67 | seed,
|
| 68 | "--branch-count",
|
| 69 | str(branch_count),
|
| 70 | ],
|
| 71 | capture_output=True,
|
| 72 | text=True,
|
| 73 | )
|
| 74 | if proc.returncode != 0:
|
| 75 | raise RuntimeError(proc.stderr or proc.stdout or "fixture generation failed")
|
| 76 | errors = validate_manifest(load_manifest(manifest_path))
|
| 77 | if errors:
|
| 78 | raise RuntimeError("; ".join(errors))
|
| 79 | return fixture_root
|
| 80 |
|
| 81 |
|
| 82 | def oak_supports_review_triage(oak_bin: Path, repo: Path) -> tuple[bool, str]:
|
| 83 | proc = subprocess.run(
|
| 84 | [str(oak_bin), "branch", "review", "main", "--merge-preview", "--json"],
|
| 85 | cwd=repo,
|
| 86 | capture_output=True,
|
| 87 | text=True,
|
| 88 | env=oakbench_environment.base_env(),
|
| 89 | )
|
| 90 | if proc.returncode != 0:
|
| 91 | return False, "branch_review_unavailable"
|
| 92 | try:
|
| 93 | payload = json.loads(proc.stdout)
|
| 94 | except json.JSONDecodeError:
|
| 95 | return False, "branch_review_invalid_json"
|
| 96 | triage = payload.get("triage", payload)
|
| 97 | if not isinstance(triage, dict):
|
| 98 | return False, "branch_review_missing_triage"
|
| 99 | if "recommended_action" not in triage:
|
| 100 | return False, "branch_review_missing_recommended_action"
|
| 101 | return True, "branch_review"
|
| 102 |
|
| 103 |
|
| 104 | def oak_supports_batch_triage(oak_bin: Path, repo: Path) -> tuple[bool, str]:
|
| 105 | proc = subprocess.run(
|
| 106 | [str(oak_bin), "branch", "triage", "--against", "main", "--status", "open", "--json"],
|
| 107 | cwd=repo,
|
| 108 | capture_output=True,
|
| 109 | text=True,
|
| 110 | env=oakbench_environment.base_env(),
|
| 111 | )
|
| 112 | if proc.returncode != 0:
|
| 113 | return False, "branch_triage_unavailable"
|
| 114 | try:
|
| 115 | payload = json.loads(proc.stdout)
|
| 116 | except json.JSONDecodeError:
|
| 117 | return False, "branch_triage_invalid_json"
|
| 118 | if not isinstance(payload, dict):
|
| 119 | return False, "branch_triage_invalid_payload"
|
| 120 | return True, "branch_triage"
|
| 121 |
|
| 122 |
|
| 123 | def review_branch(oak_bin: Path, repo: Path, branch: str) -> dict[str, Any]:
|
| 124 | proc = subprocess.run(
|
| 125 | [str(oak_bin), "branch", "review", branch, "--merge-preview", "--json"],
|
| 126 | cwd=repo,
|
| 127 | capture_output=True,
|
| 128 | text=True,
|
| 129 | check=False,
|
| 130 | env=oakbench_environment.base_env(),
|
| 131 | )
|
| 132 | if proc.returncode != 0:
|
| 133 | return {}
|
| 134 | try:
|
| 135 | return json.loads(proc.stdout)
|
| 136 | except json.JSONDecodeError:
|
| 137 | return {}
|
| 138 |
|
| 139 |
|
| 140 | def batch_triage(oak_bin: Path, repo: Path) -> dict[str, dict[str, Any]]:
|
| 141 | proc = subprocess.run(
|
| 142 | [str(oak_bin), "branch", "triage", "--against", "main", "--status", "open", "--json"],
|
| 143 | cwd=repo,
|
| 144 | capture_output=True,
|
| 145 | text=True,
|
| 146 | check=False,
|
| 147 | env=oakbench_environment.base_env(),
|
| 148 | )
|
| 149 | if proc.returncode != 0:
|
| 150 | return {}
|
| 151 | try:
|
| 152 | payload = json.loads(proc.stdout)
|
| 153 | except json.JSONDecodeError:
|
| 154 | return {}
|
| 155 | rows = payload.get("branches") or payload.get("rows") or []
|
| 156 | actuals: dict[str, dict[str, Any]] = {}
|
| 157 | if isinstance(rows, list):
|
| 158 | for row in rows:
|
| 159 | if isinstance(row, dict) and row.get("branch"):
|
| 160 | actuals[str(row["branch"])] = row
|
| 161 | return actuals
|
| 162 |
|
| 163 |
|
| 164 | def base_row(
|
| 165 | *,
|
| 166 | bench_id: str,
|
| 167 | operation: str,
|
| 168 | run_index: int,
|
| 169 | elapsed_ms: float,
|
| 170 | branch_count: int,
|
| 171 | registry_fields: dict[str, Any],
|
| 172 | ) -> dict[str, Any]:
|
| 173 | return {
|
| 174 | "bench_id": bench_id,
|
| 175 | "profile": PROFILE,
|
| 176 | "scenario": SCENARIO,
|
| 177 | "operation": operation,
|
| 178 | "run": run_index,
|
| 179 | "subject": "oak",
|
| 180 | "subject_kind": "oak",
|
| 181 | "elapsed_ms": round(elapsed_ms, 3),
|
| 182 | "returncode": 0,
|
| 183 | "command": [],
|
| 184 | "measurement_source": MEASUREMENT_SOURCE,
|
| 185 | "branch_count": branch_count,
|
| 186 | **registry_fields,
|
| 187 | }
|
| 188 |
|
| 189 |
|
| 190 | def skip_row(
|
| 191 | meta: dict[str, Any],
|
| 192 | operation: str,
|
| 193 | run_index: int,
|
| 194 | reason: str,
|
| 195 | elapsed_ms: float = 0.0,
|
| 196 | ) -> dict[str, Any]:
|
| 197 | return {
|
| 198 | **meta,
|
| 199 | "operation": operation,
|
| 200 | "run": run_index,
|
| 201 | "elapsed_ms": round(elapsed_ms, 3),
|
| 202 | "returncode": SKIP_RETURNCODE,
|
| 203 | "skipped": True,
|
| 204 | "skip_reason": reason,
|
| 205 | "command": [],
|
| 206 | **triage_metrics_row_fields(unmeasured_triage_metrics(branches_expected=int(meta.get("branch_count") or 0))),
|
| 207 | }
|
| 208 |
|
| 209 |
|
| 210 | def scoring_failures(scored: dict[str, Any]) -> list[dict[str, Any]]:
|
| 211 | return [
|
| 212 | row
|
| 213 | for row in scored["branch_results"]
|
| 214 | if not row.get("measured")
|
| 215 | or not row.get("correct")
|
| 216 | or row.get("false_close")
|
| 217 | or row.get("false_vcs_merge_safe")
|
| 218 | or row.get("false_merge_allowed")
|
| 219 | ]
|
| 220 |
|
| 221 |
|
| 222 | def run_lane(args: argparse.Namespace) -> list[dict[str, Any]]:
|
| 223 | registry = load_fixture_registry()
|
| 224 | spec = registry.get(FIXTURE_ID)
|
| 225 | registry_fields = spec.row_fields() if spec else {
|
| 226 | "fixture_id": FIXTURE_ID,
|
| 227 | "fixture_generator": "scripts/make_branch_triage_fixture.py",
|
| 228 | "fixture_version": "v1",
|
| 229 | "fixture_seed": args.seed,
|
| 230 | "fixture_manifest_sha256": None,
|
| 231 | }
|
| 232 | bench_id = args.bench_id or utc_bench_id()
|
| 233 | meta = base_row(
|
| 234 | bench_id=bench_id,
|
| 235 | operation="triage.fixture.ensure",
|
| 236 | run_index=args.run,
|
| 237 | elapsed_ms=0.0,
|
| 238 | branch_count=args.branch_count,
|
| 239 | registry_fields=registry_fields,
|
| 240 | )
|
| 241 |
|
| 242 | start = platform_clock.monotonic_ms()
|
| 243 | try:
|
| 244 | fixture_root = ensure_fixture(args.workdir, args.seed, args.branch_count)
|
| 245 | except RuntimeError as exc:
|
| 246 | return [
|
| 247 | skip_row(meta, "triage.fixture.ensure", args.run, f"fixture_generation_failed:{exc}",
|
| 248 | platform_clock.monotonic_ms() - start)
|
| 249 | ]
|
| 250 | repo = fixture_root / "repo"
|
| 251 | manifest = load_manifest(fixture_root / MANIFEST_FILENAME)
|
| 252 | rows: list[dict[str, Any]] = [
|
| 253 | {
|
| 254 | **meta,
|
| 255 | "elapsed_ms": round(platform_clock.monotonic_ms() - start, 3),
|
| 256 | "fixture_root": str(fixture_root),
|
| 257 | "branches_expected": len(manifest.get("branches") or []),
|
| 258 | }
|
| 259 | ]
|
| 260 |
|
| 261 | oak_bin = Path(args.oak_bin) if args.oak_bin else shutil.which("oak")
|
| 262 | if oak_bin is None or not Path(oak_bin).exists():
|
| 263 | rows.append(
|
| 264 | skip_row(meta, "triage.score", args.run, "oak_binary_unavailable",
|
| 265 | platform_clock.monotonic_ms() - start)
|
| 266 | )
|
| 267 | return rows
|
| 268 |
|
| 269 | oak_path = Path(oak_bin)
|
| 270 | probe_start = platform_clock.monotonic_ms()
|
| 271 | batch_ok, batch_reason = oak_supports_batch_triage(oak_path, repo)
|
| 272 | review_ok, review_reason = oak_supports_review_triage(oak_path, repo)
|
| 273 | if not batch_ok and not review_ok:
|
| 274 | rows.append(
|
| 275 | skip_row(
|
| 276 | meta,
|
| 277 | "triage.score",
|
| 278 | args.run,
|
| 279 | f"branch_triage_capability_missing:{batch_reason}:{review_reason}",
|
| 280 | platform_clock.monotonic_ms() - probe_start,
|
| 281 | )
|
| 282 | )
|
| 283 | return rows
|
| 284 |
|
| 285 | actuals: dict[str, dict[str, Any]] = {}
|
| 286 | if batch_ok:
|
| 287 | actuals = batch_triage(oak_path, repo)
|
| 288 | if not actuals and review_ok:
|
| 289 | for branch in manifest.get("branches") or []:
|
| 290 | name = str(branch.get("name") or "")
|
| 291 | if name:
|
| 292 | actuals[name] = review_branch(oak_path, repo, name)
|
| 293 |
|
| 294 | score_start = platform_clock.monotonic_ms()
|
| 295 | scored = score_manifest_against_actuals(manifest, actuals)
|
| 296 | metrics = scored["metrics"]
|
| 297 | metric_fields = triage_metrics_row_fields(metrics)
|
| 298 | metric_errors = validate_triage_metrics(metric_fields)
|
| 299 | failures = scoring_failures(scored)
|
| 300 | rows.append(
|
| 301 | {
|
| 302 | **meta,
|
| 303 | "operation": "triage.score",
|
| 304 | "elapsed_ms": round(platform_clock.monotonic_ms() - score_start, 3),
|
| 305 | "returncode": 1 if metric_errors or failures else 0,
|
| 306 | "command": [str(oak_path), "branch", "triage_or_review", "--json"],
|
| 307 | "triage_provider": batch_reason if batch_ok else review_reason,
|
| 308 | "branch_results": scored["branch_results"],
|
| 309 | **metric_fields,
|
| 310 | }
|
| 311 | )
|
| 312 | return rows
|
| 313 |
|
| 314 |
|
| 315 | def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
| 316 | parser = argparse.ArgumentParser(description="Branch triage fixture correctness lane")
|
| 317 | parser.add_argument("--workdir", type=Path, default=DEFAULT_WORKDIR)
|
| 318 | parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
|
| 319 | parser.add_argument("--seed", default=DEFAULT_SEED)
|
| 320 | parser.add_argument("--branch-count", type=int, default=DEFAULT_BRANCH_COUNT)
|
| 321 | parser.add_argument("--quick", action="store_true")
|
| 322 | parser.add_argument("--medium", action="store_true")
|
| 323 | parser.add_argument("--run", type=int, default=0)
|
| 324 | parser.add_argument("--bench-id")
|
| 325 | parser.add_argument("--oak-bin")
|
| 326 | parser.add_argument("--dry-run", action="store_true")
|
| 327 | return parser.parse_args(argv)
|
| 328 |
|
| 329 |
|
| 330 | def main(argv: list[str] | None = None) -> int:
|
| 331 | args = parse_args(argv)
|
| 332 | if args.quick:
|
| 333 | args.branch_count = DEFAULT_BRANCH_COUNT
|
| 334 | if args.medium:
|
| 335 | args.branch_count = 60
|
| 336 | rows = run_lane(args)
|
| 337 | contract_errors = validate_rows(rows, "task-loop")
|
| 338 | for row in rows:
|
| 339 | metric_payload = {key: row.get(key) for key in TRIAGE_METRIC_KEYS}
|
| 340 | contract_errors.extend(validate_triage_metrics(metric_payload))
|
| 341 | if contract_errors:
|
| 342 | print("\n".join(contract_errors), file=sys.stderr)
|
| 343 | return 1
|
| 344 | if args.dry_run:
|
| 345 | print(json.dumps(rows, indent=2, sort_keys=True))
|
| 346 | return 0
|
| 347 | store = ResultsStore(args.results, lane="task-loop")
|
| 348 | bench_id = str(rows[0]["bench_id"])
|
| 349 | store.write(bench_id, rows)
|
| 350 | failures = [row for row in rows if row.get("returncode") not in (0, SKIP_RETURNCODE)]
|
| 351 | return 1 if failures else 0
|
| 352 |
|
| 353 |
|
| 354 | if __name__ == "__main__":
|
| 355 | raise SystemExit(main())
|