Log in
scripts/perf_loop.py 138 lines · 5.9 KB · python Blame
1
#!/usr/bin/env python3
2
"""Propose-mode overnight loop for Oak perf candidates.
3
4
The script selects targets, verifies candidate branches, and writes LEDGER.md.
5
It never edits Oak and never merges.
6
"""
7
8
from __future__ import annotations
9
10
import argparse
11
import json
12
import shlex
13
import sys
14
from pathlib import Path
15
16
ROOT = Path(__file__).resolve().parents[1]
17
sys.path.insert(0, str(ROOT / "scripts"))
18
19
from oakbench.perf_loop import (  # noqa: E402
20
    append_ledger,
21
    dry_run_entry,
22
    evaluate_interleaved_ab,
23
    load_scorecard,
24
    parse_ledger,
25
    run_correctness_gate,
26
    run_interleaved_commands,
27
    select_target,
28
    target_key,
29
    verification_entry,
30
)
31
32
33
def parse_args() -> argparse.Namespace:
34
    parser = argparse.ArgumentParser(description=__doc__)
35
    sub = parser.add_subparsers(dest="command", required=True)
36
37
    select = sub.add_parser("select", help="Select the highest-ratio target from a scorecard.")
38
    select.add_argument("--scorecard", required=True, type=Path)
39
    select.add_argument("--ledger", type=Path, default=ROOT / "LEDGER.md")
40
    select.add_argument("--output", type=Path)
41
42
    dry = sub.add_parser("dry-run", help="Exercise select -> park -> ledger without editing Oak.")
43
    dry.add_argument("--scorecard", required=True, type=Path)
44
    dry.add_argument("--ledger", type=Path, default=ROOT / "LEDGER.md")
45
    dry.add_argument("--max-candidates", type=int, default=1)
46
    dry.add_argument("--stop-on-first-red", action="store_true", default=True)
47
    dry.add_argument("--output", type=Path)
48
49
    verify = sub.add_parser("verify", help="Verify a candidate branch in propose mode.")
50
    verify.add_argument("--target", required=True, type=Path, help="Selected target JSON.")
51
    verify.add_argument("--ledger", type=Path, default=ROOT / "LEDGER.md")
52
    verify.add_argument("--oak-repo", required=True, type=Path)
53
    verify.add_argument("--before-command", required=True, help="Shell-style command for the baseline binary.")
54
    verify.add_argument("--after-command", required=True, help="Shell-style command for the candidate binary.")
55
    verify.add_argument("--ab-cwd", required=True, type=Path, help="Same prepared fixture directory for both commands.")
56
    verify.add_argument("--pairs", type=int, default=20, help="A/B pairs; 20 means 40 subprocesses.")
57
    verify.add_argument("--min-improvement-pct", type=float, default=3.0)
58
    verify.add_argument("--diagnosis-file", required=True, type=Path)
59
    verify.add_argument("--change-summary", required=True)
60
    verify.add_argument("--oak-gate-verdict", default="MISSING", help="PASS only after the Oak-vs-Oak gate is green.")
61
    verify.add_argument("--fleet-gate-verdict", default="MISSING", help="PASS only after the fleet reliability gate is green.")
62
    verify.add_argument("--lever-rejected", action="append", default=[])
63
    verify.add_argument("--skip-correctness", action="store_true")
64
    verify.add_argument("--output", type=Path)
65
    return parser.parse_args()
66
67
68
def _write_optional(path: Path | None, payload: dict) -> None:
69
    if path:
70
        path.parent.mkdir(parents=True, exist_ok=True)
71
        path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
72
73
74
def main() -> int:
75
    args = parse_args()
76
    if args.command == "select":
77
        target = select_target(load_scorecard(args.scorecard), parse_ledger(args.ledger))
78
        if target is None:
79
            print("no eligible target", file=sys.stderr)
80
            return 1
81
        _write_optional(args.output, target)
82
        print(json.dumps(target, indent=2, sort_keys=True))
83
        return 0
84
85
    if args.command == "dry-run":
86
        if args.max_candidates < 1:
87
            raise SystemExit("--max-candidates must be >= 1")
88
        scorecard = load_scorecard(args.scorecard)
89
        if args.stop_on_first_red and scorecard.get("summary", {}).get("gate_verdict") == "regressed":
90
            print("scorecard is RED; stop-on-first-RED prevented candidate selection", file=sys.stderr)
91
            return 2
92
        target = select_target(scorecard, parse_ledger(args.ledger))
93
        if target is None:
94
            print("no eligible target", file=sys.stderr)
95
            return 1
96
        entry = dry_run_entry(target)
97
        append_ledger(args.ledger, entry)
98
        _write_optional(args.output, entry)
99
        print(f"parked dry-run candidate: {target_key(target)}")
100
        return 0
101
102
    if args.command == "verify":
103
        target = json.loads(args.target.read_text())
104
        if not isinstance(target, dict):
105
            raise SystemExit(f"{args.target}: expected target object")
106
        diagnosis = args.diagnosis_file.read_text() if args.diagnosis_file.exists() else ""
107
        correctness = run_correctness_gate(args.oak_repo, skip=args.skip_correctness)
108
        if correctness and all(int(row.get("returncode", 1)) == 0 for row in correctness):
109
            samples = run_interleaved_commands(
110
                shlex.split(args.before_command),
111
                shlex.split(args.after_command),
112
                cwd=args.ab_cwd,
113
                pairs=args.pairs,
114
            )
115
            ab_result = evaluate_interleaved_ab(samples, min_pairs=args.pairs, min_improvement_pct=args.min_improvement_pct)
116
        else:
117
            ab_result = {"passed": False, "reason": "correctness_gate_failed"}
118
        entry = verification_entry(
119
            target=target,
120
            diagnosis=diagnosis,
121
            change_summary=args.change_summary,
122
            correctness_results=correctness,
123
            ab_result=ab_result,
124
            oak_gate_verdict=args.oak_gate_verdict,
125
            fleet_gate_verdict=args.fleet_gate_verdict,
126
            levers_rejected=args.lever_rejected,
127
            propose_mode=True,
128
        )
129
        append_ledger(args.ledger, entry)
130
        _write_optional(args.output, entry)
131
        print(json.dumps(entry, indent=2, sort_keys=True))
132
        return 0 if entry["reason"] == "verified_propose_mode_human_review_required" else 1
133
134
    raise AssertionError(args.command)
135
136
137
if __name__ == "__main__":
138
    raise SystemExit(main())