88e86b3faccd
Rebuild benchmarks as a clean single-root reposi
2 months ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Propose-mode overnight loop for Oak perf candidates.
|
| 3 |
|
| 4 | The script selects targets, verifies candidate branches, and writes LEDGER.md.
|
| 5 | It never edits Oak and never merges.
|
| 6 | """
|
| 7 |
|
| 8 | from __future__ import annotations
|
| 9 |
|
| 10 | import argparse
|
| 11 | import json
|
| 12 | import shlex
|
| 13 | import sys
|
| 14 | from pathlib import Path
|
| 15 |
|
| 16 | ROOT = Path(__file__).resolve().parents[1]
|
| 17 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 18 |
|
| 19 | from oakbench.perf_loop import ( # noqa: E402
|
| 20 | append_ledger,
|
| 21 | dry_run_entry,
|
| 22 | evaluate_interleaved_ab,
|
| 23 | load_scorecard,
|
| 24 | parse_ledger,
|
| 25 | run_correctness_gate,
|
| 26 | run_interleaved_commands,
|
| 27 | select_target,
|
| 28 | target_key,
|
| 29 | verification_entry,
|
| 30 | )
|
| 31 |
|
| 32 |
|
| 33 | def parse_args() -> argparse.Namespace:
|
| 34 | parser = argparse.ArgumentParser(description=__doc__)
|
| 35 | sub = parser.add_subparsers(dest="command", required=True)
|
| 36 |
|
| 37 | select = sub.add_parser("select", help="Select the highest-ratio target from a scorecard.")
|
| 38 | select.add_argument("--scorecard", required=True, type=Path)
|
| 39 | select.add_argument("--ledger", type=Path, default=ROOT / "LEDGER.md")
|
| 40 | select.add_argument("--output", type=Path)
|
| 41 |
|
| 42 | dry = sub.add_parser("dry-run", help="Exercise select -> park -> ledger without editing Oak.")
|
| 43 | dry.add_argument("--scorecard", required=True, type=Path)
|
| 44 | dry.add_argument("--ledger", type=Path, default=ROOT / "LEDGER.md")
|
| 45 | dry.add_argument("--max-candidates", type=int, default=1)
|
| 46 | dry.add_argument("--stop-on-first-red", action="store_true", default=True)
|
| 47 | dry.add_argument("--output", type=Path)
|
| 48 |
|
| 49 | verify = sub.add_parser("verify", help="Verify a candidate branch in propose mode.")
|
| 50 | verify.add_argument("--target", required=True, type=Path, help="Selected target JSON.")
|
| 51 | verify.add_argument("--ledger", type=Path, default=ROOT / "LEDGER.md")
|
| 52 | verify.add_argument("--oak-repo", required=True, type=Path)
|
| 53 | verify.add_argument("--before-command", required=True, help="Shell-style command for the baseline binary.")
|
| 54 | verify.add_argument("--after-command", required=True, help="Shell-style command for the candidate binary.")
|
| 55 | verify.add_argument("--ab-cwd", required=True, type=Path, help="Same prepared fixture directory for both commands.")
|
| 56 | verify.add_argument("--pairs", type=int, default=20, help="A/B pairs; 20 means 40 subprocesses.")
|
| 57 | verify.add_argument("--min-improvement-pct", type=float, default=3.0)
|
| 58 | verify.add_argument("--diagnosis-file", required=True, type=Path)
|
| 59 | verify.add_argument("--change-summary", required=True)
|
| 60 | verify.add_argument("--oak-gate-verdict", default="MISSING", help="PASS only after the Oak-vs-Oak gate is green.")
|
| 61 | verify.add_argument("--fleet-gate-verdict", default="MISSING", help="PASS only after the fleet reliability gate is green.")
|
| 62 | verify.add_argument("--lever-rejected", action="append", default=[])
|
| 63 | verify.add_argument("--skip-correctness", action="store_true")
|
| 64 | verify.add_argument("--output", type=Path)
|
| 65 | return parser.parse_args()
|
| 66 |
|
| 67 |
|
| 68 | def _write_optional(path: Path | None, payload: dict) -> None:
|
| 69 | if path:
|
| 70 | path.parent.mkdir(parents=True, exist_ok=True)
|
| 71 | path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
| 72 |
|
| 73 |
|
| 74 | def main() -> int:
|
| 75 | args = parse_args()
|
| 76 | if args.command == "select":
|
| 77 | target = select_target(load_scorecard(args.scorecard), parse_ledger(args.ledger))
|
| 78 | if target is None:
|
| 79 | print("no eligible target", file=sys.stderr)
|
| 80 | return 1
|
| 81 | _write_optional(args.output, target)
|
| 82 | print(json.dumps(target, indent=2, sort_keys=True))
|
| 83 | return 0
|
| 84 |
|
| 85 | if args.command == "dry-run":
|
| 86 | if args.max_candidates < 1:
|
| 87 | raise SystemExit("--max-candidates must be >= 1")
|
| 88 | scorecard = load_scorecard(args.scorecard)
|
| 89 | if args.stop_on_first_red and scorecard.get("summary", {}).get("gate_verdict") == "regressed":
|
| 90 | print("scorecard is RED; stop-on-first-RED prevented candidate selection", file=sys.stderr)
|
| 91 | return 2
|
| 92 | target = select_target(scorecard, parse_ledger(args.ledger))
|
| 93 | if target is None:
|
| 94 | print("no eligible target", file=sys.stderr)
|
| 95 | return 1
|
| 96 | entry = dry_run_entry(target)
|
| 97 | append_ledger(args.ledger, entry)
|
| 98 | _write_optional(args.output, entry)
|
| 99 | print(f"parked dry-run candidate: {target_key(target)}")
|
| 100 | return 0
|
| 101 |
|
| 102 | if args.command == "verify":
|
| 103 | target = json.loads(args.target.read_text())
|
| 104 | if not isinstance(target, dict):
|
| 105 | raise SystemExit(f"{args.target}: expected target object")
|
| 106 | diagnosis = args.diagnosis_file.read_text() if args.diagnosis_file.exists() else ""
|
| 107 | correctness = run_correctness_gate(args.oak_repo, skip=args.skip_correctness)
|
| 108 | if correctness and all(int(row.get("returncode", 1)) == 0 for row in correctness):
|
| 109 | samples = run_interleaved_commands(
|
| 110 | shlex.split(args.before_command),
|
| 111 | shlex.split(args.after_command),
|
| 112 | cwd=args.ab_cwd,
|
| 113 | pairs=args.pairs,
|
| 114 | )
|
| 115 | ab_result = evaluate_interleaved_ab(samples, min_pairs=args.pairs, min_improvement_pct=args.min_improvement_pct)
|
| 116 | else:
|
| 117 | ab_result = {"passed": False, "reason": "correctness_gate_failed"}
|
| 118 | entry = verification_entry(
|
| 119 | target=target,
|
| 120 | diagnosis=diagnosis,
|
| 121 | change_summary=args.change_summary,
|
| 122 | correctness_results=correctness,
|
| 123 | ab_result=ab_result,
|
| 124 | oak_gate_verdict=args.oak_gate_verdict,
|
| 125 | fleet_gate_verdict=args.fleet_gate_verdict,
|
| 126 | levers_rejected=args.lever_rejected,
|
| 127 | propose_mode=True,
|
| 128 | )
|
| 129 | append_ledger(args.ledger, entry)
|
| 130 | _write_optional(args.output, entry)
|
| 131 | print(json.dumps(entry, indent=2, sort_keys=True))
|
| 132 | return 0 if entry["reason"] == "verified_propose_mode_human_review_required" else 1
|
| 133 |
|
| 134 | raise AssertionError(args.command)
|
| 135 |
|
| 136 |
|
| 137 | if __name__ == "__main__":
|
| 138 | raise SystemExit(main())
|