Log in
scripts/branch_triage_fixture_lane.py 355 lines · 11.7 KB · python Blame
1
#!/usr/bin/env python3
2
"""Run the branch-triage-shape correctness lane over a local git fixture.
3
4
Ensures (or generates) the branch triage fixture, probes Oak for branch review
5
or batch triage JSON support, scores measured rows against BRANCH_MANIFEST.json,
6
and emits task-loop lane rows. Missing Oak capability yields explicit skip rows
7
(returncode 77) with null metrics β€” never a fabricated pass.
8
"""
9
10
from __future__ import annotations
11
12
import argparse
13
import json
14
import shutil
15
import subprocess
16
import sys
17
import tempfile
18
from datetime import datetime, timezone
19
from pathlib import Path
20
from typing import Any
21
22
ROOT = Path(__file__).resolve().parents[1]
23
sys.path.insert(0, str(ROOT / "scripts"))
24
25
from oakbench import environment as oakbench_environment
26
from oakbench import platform_clock
27
from oakbench.branch_triage_shape import (
28
    MANIFEST_FILENAME,
29
    TRIAGE_METRIC_KEYS,
30
    load_manifest,
31
    score_manifest_against_actuals,
32
    triage_metrics_row_fields,
33
    unmeasured_triage_metrics,
34
    validate_manifest,
35
    validate_triage_metrics,
36
)
37
from oakbench.fixture_registry import load_fixture_registry
38
from oakbench.results import ResultsStore
39
from oakbench.rows import SKIP_RETURNCODE, validate_rows
40
41
GENERATOR = ROOT / "scripts" / "make_branch_triage_fixture.py"
42
DEFAULT_WORKDIR = Path(tempfile.gettempdir()) / "oak-branch-triage-fixture"
43
DEFAULT_RESULTS = ROOT / "results" / "branch-triage-fixture"
44
PROFILE = "branch-triage-fixture"
45
FIXTURE_ID = "bench-branch-triage-shape-v2"
46
SCENARIO = "branch_triage_shape"
47
MEASUREMENT_SOURCE = "direct_cli_timed_subprocess"
48
DEFAULT_SEED = "oakbench-branch-triage-v2"
49
DEFAULT_BRANCH_COUNT = 6
50
51
52
def utc_bench_id() -> str:
53
    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
54
55
56
def ensure_fixture(workdir: Path, seed: str, branch_count: int) -> Path:
57
    fixture_root = workdir / f"{seed}-n{branch_count}"
58
    manifest_path = fixture_root / MANIFEST_FILENAME
59
    if not manifest_path.is_file():
60
        proc = subprocess.run(
61
            [
62
                sys.executable,
63
                str(GENERATOR),
64
                "--out",
65
                str(fixture_root),
66
                "--seed",
67
                seed,
68
                "--branch-count",
69
                str(branch_count),
70
            ],
71
            capture_output=True,
72
            text=True,
73
        )
74
        if proc.returncode != 0:
75
            raise RuntimeError(proc.stderr or proc.stdout or "fixture generation failed")
76
    errors = validate_manifest(load_manifest(manifest_path))
77
    if errors:
78
        raise RuntimeError("; ".join(errors))
79
    return fixture_root
80
81
82
def oak_supports_review_triage(oak_bin: Path, repo: Path) -> tuple[bool, str]:
83
    proc = subprocess.run(
84
        [str(oak_bin), "branch", "review", "main", "--merge-preview", "--json"],
85
        cwd=repo,
86
        capture_output=True,
87
        text=True,
88
        env=oakbench_environment.base_env(),
89
    )
90
    if proc.returncode != 0:
91
        return False, "branch_review_unavailable"
92
    try:
93
        payload = json.loads(proc.stdout)
94
    except json.JSONDecodeError:
95
        return False, "branch_review_invalid_json"
96
    triage = payload.get("triage", payload)
97
    if not isinstance(triage, dict):
98
        return False, "branch_review_missing_triage"
99
    if "recommended_action" not in triage:
100
        return False, "branch_review_missing_recommended_action"
101
    return True, "branch_review"
102
103
104
def oak_supports_batch_triage(oak_bin: Path, repo: Path) -> tuple[bool, str]:
105
    proc = subprocess.run(
106
        [str(oak_bin), "branch", "triage", "--against", "main", "--status", "open", "--json"],
107
        cwd=repo,
108
        capture_output=True,
109
        text=True,
110
        env=oakbench_environment.base_env(),
111
    )
112
    if proc.returncode != 0:
113
        return False, "branch_triage_unavailable"
114
    try:
115
        payload = json.loads(proc.stdout)
116
    except json.JSONDecodeError:
117
        return False, "branch_triage_invalid_json"
118
    if not isinstance(payload, dict):
119
        return False, "branch_triage_invalid_payload"
120
    return True, "branch_triage"
121
122
123
def review_branch(oak_bin: Path, repo: Path, branch: str) -> dict[str, Any]:
124
    proc = subprocess.run(
125
        [str(oak_bin), "branch", "review", branch, "--merge-preview", "--json"],
126
        cwd=repo,
127
        capture_output=True,
128
        text=True,
129
        check=False,
130
        env=oakbench_environment.base_env(),
131
    )
132
    if proc.returncode != 0:
133
        return {}
134
    try:
135
        return json.loads(proc.stdout)
136
    except json.JSONDecodeError:
137
        return {}
138
139
140
def batch_triage(oak_bin: Path, repo: Path) -> dict[str, dict[str, Any]]:
141
    proc = subprocess.run(
142
        [str(oak_bin), "branch", "triage", "--against", "main", "--status", "open", "--json"],
143
        cwd=repo,
144
        capture_output=True,
145
        text=True,
146
        check=False,
147
        env=oakbench_environment.base_env(),
148
    )
149
    if proc.returncode != 0:
150
        return {}
151
    try:
152
        payload = json.loads(proc.stdout)
153
    except json.JSONDecodeError:
154
        return {}
155
    rows = payload.get("branches") or payload.get("rows") or []
156
    actuals: dict[str, dict[str, Any]] = {}
157
    if isinstance(rows, list):
158
        for row in rows:
159
            if isinstance(row, dict) and row.get("branch"):
160
                actuals[str(row["branch"])] = row
161
    return actuals
162
163
164
def base_row(
165
    *,
166
    bench_id: str,
167
    operation: str,
168
    run_index: int,
169
    elapsed_ms: float,
170
    branch_count: int,
171
    registry_fields: dict[str, Any],
172
) -> dict[str, Any]:
173
    return {
174
        "bench_id": bench_id,
175
        "profile": PROFILE,
176
        "scenario": SCENARIO,
177
        "operation": operation,
178
        "run": run_index,
179
        "subject": "oak",
180
        "subject_kind": "oak",
181
        "elapsed_ms": round(elapsed_ms, 3),
182
        "returncode": 0,
183
        "command": [],
184
        "measurement_source": MEASUREMENT_SOURCE,
185
        "branch_count": branch_count,
186
        **registry_fields,
187
    }
188
189
190
def skip_row(
191
    meta: dict[str, Any],
192
    operation: str,
193
    run_index: int,
194
    reason: str,
195
    elapsed_ms: float = 0.0,
196
) -> dict[str, Any]:
197
    return {
198
        **meta,
199
        "operation": operation,
200
        "run": run_index,
201
        "elapsed_ms": round(elapsed_ms, 3),
202
        "returncode": SKIP_RETURNCODE,
203
        "skipped": True,
204
        "skip_reason": reason,
205
        "command": [],
206
        **triage_metrics_row_fields(unmeasured_triage_metrics(branches_expected=int(meta.get("branch_count") or 0))),
207
    }
208
209
210
def scoring_failures(scored: dict[str, Any]) -> list[dict[str, Any]]:
211
    return [
212
        row
213
        for row in scored["branch_results"]
214
        if not row.get("measured")
215
        or not row.get("correct")
216
        or row.get("false_close")
217
        or row.get("false_vcs_merge_safe")
218
        or row.get("false_merge_allowed")
219
    ]
220
221
222
def run_lane(args: argparse.Namespace) -> list[dict[str, Any]]:
223
    registry = load_fixture_registry()
224
    spec = registry.get(FIXTURE_ID)
225
    registry_fields = spec.row_fields() if spec else {
226
        "fixture_id": FIXTURE_ID,
227
        "fixture_generator": "scripts/make_branch_triage_fixture.py",
228
        "fixture_version": "v1",
229
        "fixture_seed": args.seed,
230
        "fixture_manifest_sha256": None,
231
    }
232
    bench_id = args.bench_id or utc_bench_id()
233
    meta = base_row(
234
        bench_id=bench_id,
235
        operation="triage.fixture.ensure",
236
        run_index=args.run,
237
        elapsed_ms=0.0,
238
        branch_count=args.branch_count,
239
        registry_fields=registry_fields,
240
    )
241
242
    start = platform_clock.monotonic_ms()
243
    try:
244
        fixture_root = ensure_fixture(args.workdir, args.seed, args.branch_count)
245
    except RuntimeError as exc:
246
        return [
247
            skip_row(meta, "triage.fixture.ensure", args.run, f"fixture_generation_failed:{exc}",
248
                     platform_clock.monotonic_ms() - start)
249
        ]
250
    repo = fixture_root / "repo"
251
    manifest = load_manifest(fixture_root / MANIFEST_FILENAME)
252
    rows: list[dict[str, Any]] = [
253
        {
254
            **meta,
255
            "elapsed_ms": round(platform_clock.monotonic_ms() - start, 3),
256
            "fixture_root": str(fixture_root),
257
            "branches_expected": len(manifest.get("branches") or []),
258
        }
259
    ]
260
261
    oak_bin = Path(args.oak_bin) if args.oak_bin else shutil.which("oak")
262
    if oak_bin is None or not Path(oak_bin).exists():
263
        rows.append(
264
            skip_row(meta, "triage.score", args.run, "oak_binary_unavailable",
265
                     platform_clock.monotonic_ms() - start)
266
        )
267
        return rows
268
269
    oak_path = Path(oak_bin)
270
    probe_start = platform_clock.monotonic_ms()
271
    batch_ok, batch_reason = oak_supports_batch_triage(oak_path, repo)
272
    review_ok, review_reason = oak_supports_review_triage(oak_path, repo)
273
    if not batch_ok and not review_ok:
274
        rows.append(
275
            skip_row(
276
                meta,
277
                "triage.score",
278
                args.run,
279
                f"branch_triage_capability_missing:{batch_reason}:{review_reason}",
280
                platform_clock.monotonic_ms() - probe_start,
281
            )
282
        )
283
        return rows
284
285
    actuals: dict[str, dict[str, Any]] = {}
286
    if batch_ok:
287
        actuals = batch_triage(oak_path, repo)
288
    if not actuals and review_ok:
289
        for branch in manifest.get("branches") or []:
290
            name = str(branch.get("name") or "")
291
            if name:
292
                actuals[name] = review_branch(oak_path, repo, name)
293
294
    score_start = platform_clock.monotonic_ms()
295
    scored = score_manifest_against_actuals(manifest, actuals)
296
    metrics = scored["metrics"]
297
    metric_fields = triage_metrics_row_fields(metrics)
298
    metric_errors = validate_triage_metrics(metric_fields)
299
    failures = scoring_failures(scored)
300
    rows.append(
301
        {
302
            **meta,
303
            "operation": "triage.score",
304
            "elapsed_ms": round(platform_clock.monotonic_ms() - score_start, 3),
305
            "returncode": 1 if metric_errors or failures else 0,
306
            "command": [str(oak_path), "branch", "triage_or_review", "--json"],
307
            "triage_provider": batch_reason if batch_ok else review_reason,
308
            "branch_results": scored["branch_results"],
309
            **metric_fields,
310
        }
311
    )
312
    return rows
313
314
315
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
316
    parser = argparse.ArgumentParser(description="Branch triage fixture correctness lane")
317
    parser.add_argument("--workdir", type=Path, default=DEFAULT_WORKDIR)
318
    parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
319
    parser.add_argument("--seed", default=DEFAULT_SEED)
320
    parser.add_argument("--branch-count", type=int, default=DEFAULT_BRANCH_COUNT)
321
    parser.add_argument("--quick", action="store_true")
322
    parser.add_argument("--medium", action="store_true")
323
    parser.add_argument("--run", type=int, default=0)
324
    parser.add_argument("--bench-id")
325
    parser.add_argument("--oak-bin")
326
    parser.add_argument("--dry-run", action="store_true")
327
    return parser.parse_args(argv)
328
329
330
def main(argv: list[str] | None = None) -> int:
331
    args = parse_args(argv)
332
    if args.quick:
333
        args.branch_count = DEFAULT_BRANCH_COUNT
334
    if args.medium:
335
        args.branch_count = 60
336
    rows = run_lane(args)
337
    contract_errors = validate_rows(rows, "task-loop")
338
    for row in rows:
339
        metric_payload = {key: row.get(key) for key in TRIAGE_METRIC_KEYS}
340
        contract_errors.extend(validate_triage_metrics(metric_payload))
341
    if contract_errors:
342
        print("\n".join(contract_errors), file=sys.stderr)
343
        return 1
344
    if args.dry_run:
345
        print(json.dumps(rows, indent=2, sort_keys=True))
346
        return 0
347
    store = ResultsStore(args.results, lane="task-loop")
348
    bench_id = str(rows[0]["bench_id"])
349
    store.write(bench_id, rows)
350
    failures = [row for row in rows if row.get("returncode") not in (0, SKIP_RETURNCODE)]
351
    return 1 if failures else 0
352
353
354
if __name__ == "__main__":
355
    raise SystemExit(main())