Viewing files at commit 1dc7410826b4Commit detailsView latest

Browse files
scripts/oakbench/branch_triage_shape.py 470 lines · 16.6 KB · python
1
"""Branch triage fixture shapes, manifest validation, and correctness metrics.
2
3
The generator produces labeled git branch states plus ``BRANCH_MANIFEST.json``.
4
Scoring compares measured Oak triage JSON against the manifest oracle. Null means
5
unmeasured (ADR-0002): missing ``oak branch triage`` / review triage support
6
must yield skip rows and null metrics, never fabricated zeros or silent passes.
7
"""
8
9
from __future__ import annotations
10
11
import json
12
import math
13
from dataclasses import dataclass
14
from pathlib import Path
15
from typing import Any, Mapping, Optional
16
17
MANIFEST_FILENAME = "BRANCH_MANIFEST.json"
18
GENERATOR_VERSION = "branch-triage-shape-v2"
19
SCHEMA_VERSION = 1
20
21
RECOMMENDED_ACTIONS = (
22
    "close",
23
    "validate_then_merge",
24
    "resolve",
25
    "rebuild",
26
    "review",
27
    "unknown",
28
)
29
MERGEABILITY = ("clean", "conflicts", "unknown")
30
CONTRIBUTION = (
31
    "empty",
32
    "superseded_exact",
33
    "clean_contributes",
34
    "conflict_additive",
35
    "contributes_and_reverts_target",
36
    "missing_data",
37
    "unknown",
38
)
39
TARGET_RISK = ("none", "reverts_target_exact", "missing_data", "unknown")
40
41
# Six deterministic shapes cycled for larger branch counts.
42
SHAPE_NAMES = (
43
    "empty",
44
    "superseded_exact",
45
    "clean_contributes",
46
    "conflict_additive",
47
    "contributes_and_reverts_target",
48
    "missing_data",
49
)
50
51
TRIAGE_METRIC_KEYS = (
52
    "triage_recall",
53
    "triage_precision",
54
    "false_close_count",
55
    "close_eligible_n",
56
    "false_vcs_merge_safe_count",
57
    "vcs_merge_safe_n",
58
    "false_merge_allowed_count",
59
    "merge_allowed_n",
60
    "branches_scored",
61
    "branches_expected",
62
)
63
64
65
@dataclass(frozen=True)
66
class ExpectedTriage:
67
    recommended_action: str
68
    reason: str
69
    close_allowed: bool
70
    vcs_merge_safe: Optional[bool]
71
    merge_allowed: bool
72
    mergeability: str
73
    contribution: str
74
    target_risk: str
75
76
    def as_dict(self) -> dict[str, Any]:
77
        return {
78
            "recommended_action": self.recommended_action,
79
            "reason": self.reason,
80
            "close_allowed": self.close_allowed,
81
            "vcs_merge_safe": self.vcs_merge_safe,
82
            "merge_allowed": self.merge_allowed,
83
            "mergeability": self.mergeability,
84
            "contribution": self.contribution,
85
            "target_risk": self.target_risk,
86
        }
87
88
89
@dataclass(frozen=True)
90
class BranchManifestEntry:
91
    name: str
92
    shape: str
93
    expected: ExpectedTriage
94
95
    def as_dict(self) -> dict[str, Any]:
96
        return {
97
            "name": self.name,
98
            "shape": self.shape,
99
            "expected": self.expected.as_dict(),
100
        }
101
102
103
def shape_name(index: int) -> str:
104
    return SHAPE_NAMES[index % len(SHAPE_NAMES)]
105
106
107
def branch_name(index: int) -> str:
108
    return f"triage-{shape_name(index)}-{index}"
109
110
111
def expected_triage_for_shape(shape: str) -> ExpectedTriage:
112
    """Manifest oracle for one labeled shape (full merge-preview analysis)."""
113
    if shape == "empty":
114
        return ExpectedTriage(
115
            recommended_action="close",
116
            reason="empty",
117
            close_allowed=True,
118
            vcs_merge_safe=None,
119
            merge_allowed=False,
120
            mergeability="unknown",
121
            contribution="empty",
122
            target_risk="unknown",
123
        )
124
    if shape == "superseded_exact":
125
        return ExpectedTriage(
126
            recommended_action="close",
127
            reason="superseded_exact",
128
            close_allowed=True,
129
            vcs_merge_safe=None,
130
            merge_allowed=False,
131
            mergeability="unknown",
132
            contribution="superseded_exact",
133
            target_risk="unknown",
134
        )
135
    if shape == "clean_contributes":
136
        return ExpectedTriage(
137
            recommended_action="validate_then_merge",
138
            reason="clean_contributes",
139
            close_allowed=False,
140
            vcs_merge_safe=True,
141
            merge_allowed=True,
142
            mergeability="clean",
143
            contribution="clean_contributes",
144
            target_risk="none",
145
        )
146
    if shape == "conflict_additive":
147
        return ExpectedTriage(
148
            recommended_action="resolve",
149
            reason="conflict_additive",
150
            close_allowed=False,
151
            vcs_merge_safe=False,
152
            merge_allowed=False,
153
            mergeability="conflicts",
154
            contribution="conflict_additive",
155
            target_risk="none",
156
        )
157
    if shape == "contributes_and_reverts_target":
158
        return ExpectedTriage(
159
            recommended_action="review",
160
            reason="reverts_target_exact",
161
            close_allowed=False,
162
            vcs_merge_safe=True,
163
            merge_allowed=False,
164
            mergeability="clean",
165
            contribution="contributes_and_reverts_target",
166
            target_risk="reverts_target_exact",
167
        )
168
    if shape == "missing_data":
169
        return ExpectedTriage(
170
            recommended_action="review",
171
            reason="missing_data",
172
            close_allowed=False,
173
            vcs_merge_safe=None,
174
            merge_allowed=False,
175
            mergeability="unknown",
176
            contribution="missing_data",
177
            target_risk="missing_data",
178
        )
179
    raise ValueError(f"unknown branch triage shape {shape!r}")
180
181
182
def manifest_entries(branch_count: int) -> list[BranchManifestEntry]:
183
    if branch_count < 1:
184
        raise ValueError("branch_count must be at least 1")
185
    entries: list[BranchManifestEntry] = []
186
    for index in range(branch_count):
187
        shape = shape_name(index)
188
        entries.append(
189
            BranchManifestEntry(
190
                name=branch_name(index),
191
                shape=shape,
192
                expected=expected_triage_for_shape(shape),
193
            )
194
        )
195
    return entries
196
197
198
def build_manifest(*, seed: str, branch_count: int, against: str = "main") -> dict[str, Any]:
199
    return {
200
        "schema_version": SCHEMA_VERSION,
201
        "generator": GENERATOR_VERSION,
202
        "seed": seed,
203
        "against": against,
204
        "branch_count": branch_count,
205
        "shapes": list(SHAPE_NAMES),
206
        "branches": [entry.as_dict() for entry in manifest_entries(branch_count)],
207
    }
208
209
210
def write_manifest(path: Path, *, seed: str, branch_count: int, against: str = "main") -> dict[str, Any]:
211
    manifest = build_manifest(seed=seed, branch_count=branch_count, against=against)
212
    path.parent.mkdir(parents=True, exist_ok=True)
213
    path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
214
    return manifest
215
216
217
def load_manifest(path: Path) -> dict[str, Any]:
218
    return json.loads(path.read_text())
219
220
221
def validate_manifest(manifest: Mapping[str, Any]) -> list[str]:
222
    errors: list[str] = []
223
    if manifest.get("schema_version") != SCHEMA_VERSION:
224
        errors.append(f"schema_version must be {SCHEMA_VERSION}")
225
    if manifest.get("generator") != GENERATOR_VERSION:
226
        errors.append(f"generator must be {GENERATOR_VERSION!r}")
227
    seed = manifest.get("seed")
228
    if not isinstance(seed, str) or not seed:
229
        errors.append("seed must be a non-empty string")
230
    against = manifest.get("against")
231
    if not isinstance(against, str) or not against:
232
        errors.append("against must be a non-empty string")
233
    branch_count = manifest.get("branch_count")
234
    if not isinstance(branch_count, int) or branch_count < 1:
235
        errors.append("branch_count must be a positive integer")
236
    branches = manifest.get("branches")
237
    if not isinstance(branches, list) or not branches:
238
        errors.append("branches must be a non-empty list")
239
        return errors
240
    if isinstance(branch_count, int) and len(branches) != branch_count:
241
        errors.append("branch_count does not match branches length")
242
    seen_names: set[str] = set()
243
    for index, branch in enumerate(branches, 1):
244
        if not isinstance(branch, dict):
245
            errors.append(f"branch {index}: expected object")
246
            continue
247
        name = branch.get("name")
248
        if not isinstance(name, str) or not name:
249
            errors.append(f"branch {index}: missing name")
250
        elif name in seen_names:
251
            errors.append(f"branch {index}: duplicate name {name!r}")
252
        else:
253
            seen_names.add(name)
254
        shape = branch.get("shape")
255
        if shape not in SHAPE_NAMES:
256
            errors.append(f"branch {index}: unknown shape {shape!r}")
257
        expected = branch.get("expected")
258
        if not isinstance(expected, dict):
259
            errors.append(f"branch {index}: missing expected object")
260
            continue
261
        errors.extend(_validate_expected(index, expected))
262
    return errors
263
264
265
def _validate_expected(index: int, expected: Mapping[str, Any]) -> list[str]:
266
    errors: list[str] = []
267
    action = expected.get("recommended_action")
268
    if action not in RECOMMENDED_ACTIONS:
269
        errors.append(f"branch {index}: invalid recommended_action {action!r}")
270
    for key in ("mergeability", "contribution", "target_risk"):
271
        allowed = MERGEABILITY if key == "mergeability" else CONTRIBUTION if key == "contribution" else TARGET_RISK
272
        value = expected.get(key)
273
        if value not in allowed:
274
            errors.append(f"branch {index}: invalid {key} {value!r}")
275
    for key in ("close_allowed", "merge_allowed"):
276
        if not isinstance(expected.get(key), bool):
277
            errors.append(f"branch {index}: {key} must be boolean")
278
    vms = expected.get("vcs_merge_safe")
279
    if vms is not None and not isinstance(vms, bool):
280
        errors.append(f"branch {index}: vcs_merge_safe must be boolean or null")
281
    reason = expected.get("reason")
282
    if not isinstance(reason, str) or not reason:
283
        errors.append(f"branch {index}: reason must be a non-empty string")
284
    return errors
285
286
287
def normalize_actual_triage(payload: Mapping[str, Any]) -> dict[str, Any]:
288
    """Extract triage fields from branch review/triage JSON payloads."""
289
    triage = payload.get("triage")
290
    source = triage if isinstance(triage, dict) else payload
291
    contribution = source.get("contribution")
292
    target_risk = source.get("target_risk")
293
    return {
294
        "recommended_action": _string_or_none(source.get("recommended_action")),
295
        "reason": _string_or_none(source.get("reason")),
296
        "close_allowed": source.get("close_allowed") if isinstance(source.get("close_allowed"), bool) else None,
297
        "vcs_merge_safe": source.get("vcs_merge_safe") if source.get("vcs_merge_safe") in (True, False) else None,
298
        "merge_allowed": source.get("merge_allowed") if isinstance(source.get("merge_allowed"), bool) else None,
299
        "mergeability": _string_or_none(source.get("mergeability")),
300
        "contribution": _string_or_none(contribution),
301
        "target_risk": _string_or_none(target_risk),
302
    }
303
304
305
def _string_or_none(value: Any) -> Optional[str]:
306
    return str(value) if isinstance(value, str) and value else None
307
308
309
def _matches_expected(actual: Mapping[str, Any], expected: Mapping[str, Any]) -> bool:
310
    for key, expected_value in expected.items():
311
        actual_value = actual.get(key)
312
        if key == "vcs_merge_safe":
313
            if actual_value != expected_value:
314
                return False
315
            continue
316
        if actual_value != expected_value:
317
            return False
318
    return True
319
320
321
def score_branch(actual: Mapping[str, Any], expected: Mapping[str, Any]) -> dict[str, Any]:
322
    normalized = normalize_actual_triage(actual)
323
    measured = normalized["recommended_action"] is not None
324
    correct = measured and _matches_expected(normalized, expected)
325
    false_close = (
326
        measured
327
        and normalized["close_allowed"] is True
328
        and expected.get("close_allowed") is False
329
    )
330
    false_vcs_merge_safe = (
331
        measured
332
        and normalized["vcs_merge_safe"] is True
333
        and expected.get("vcs_merge_safe") is not True
334
    )
335
    false_merge_allowed = (
336
        measured
337
        and normalized["merge_allowed"] is True
338
        and expected.get("merge_allowed") is False
339
    )
340
    return {
341
        "branch": expected.get("branch_name"),
342
        "measured": measured,
343
        "correct": correct,
344
        "false_close": false_close,
345
        "false_vcs_merge_safe": false_vcs_merge_safe,
346
        "false_merge_allowed": false_merge_allowed,
347
        "actual": normalized,
348
    }
349
350
351
def aggregate_triage_metrics(
352
    branch_results: list[dict[str, Any]],
353
    *,
354
    branches_expected: int,
355
) -> dict[str, Any]:
356
    measured = [row for row in branch_results if row.get("measured")]
357
    if not measured:
358
        return unmeasured_triage_metrics(branches_expected=branches_expected)
359
360
    correct = sum(1 for row in measured if row.get("correct"))
361
    false_close_count = sum(1 for row in measured if row.get("false_close"))
362
    close_eligible_n = sum(
363
        1 for row in branch_results if row.get("expected_close_allowed") is True
364
    )
365
    false_vcs_merge_safe_count = sum(1 for row in measured if row.get("false_vcs_merge_safe"))
366
    vcs_merge_safe_n = sum(
367
        1 for row in branch_results if row.get("expected_vcs_merge_safe") is True
368
    )
369
    false_merge_allowed_count = sum(1 for row in measured if row.get("false_merge_allowed"))
370
    merge_allowed_n = sum(
371
        1 for row in branch_results if row.get("expected_merge_allowed") is True
372
    )
373
374
    # Recall is recovery against the oracle set; precision is exact-match
375
    # accuracy among the branches that emitted a triage payload.
376
    recall = round(correct / branches_expected, 4) if branches_expected else None
377
    precision = round(correct / len(measured), 4) if measured else None
378
379
    return {
380
        "triage_recall": recall,
381
        "triage_precision": precision,
382
        "false_close_count": false_close_count,
383
        "close_eligible_n": close_eligible_n,
384
        "false_vcs_merge_safe_count": false_vcs_merge_safe_count,
385
        "vcs_merge_safe_n": vcs_merge_safe_n,
386
        "false_merge_allowed_count": false_merge_allowed_count,
387
        "merge_allowed_n": merge_allowed_n,
388
        "branches_scored": len(measured),
389
        "branches_expected": branches_expected,
390
    }
391
392
393
def unmeasured_triage_metrics(*, branches_expected: int) -> dict[str, Any]:
394
    return {key: None for key in TRIAGE_METRIC_KEYS} | {
395
        "branches_expected": branches_expected,
396
    }
397
398
399
def score_manifest_against_actuals(
400
    manifest: Mapping[str, Any],
401
    actual_by_branch: Mapping[str, Mapping[str, Any]],
402
) -> dict[str, Any]:
403
    branches = manifest.get("branches") or []
404
    branch_results: list[dict[str, Any]] = []
405
    for branch in branches:
406
        if not isinstance(branch, dict):
407
            continue
408
        name = str(branch.get("name") or "")
409
        expected = branch.get("expected") or {}
410
        actual = actual_by_branch.get(name, {})
411
        row = score_branch(actual, expected)
412
        row["branch"] = name
413
        row["shape"] = branch.get("shape")
414
        row["expected_close_allowed"] = expected.get("close_allowed")
415
        row["expected_vcs_merge_safe"] = expected.get("vcs_merge_safe")
416
        row["expected_merge_allowed"] = expected.get("merge_allowed")
417
        row["expected_recommended_action"] = expected.get("recommended_action")
418
        branch_results.append(row)
419
    return {
420
        "branch_results": branch_results,
421
        "metrics": aggregate_triage_metrics(
422
            branch_results,
423
            branches_expected=len(branch_results),
424
        ),
425
    }
426
427
428
def triage_metrics_row_fields(metrics: Mapping[str, Any]) -> dict[str, Any]:
429
    fields: dict[str, Any] = {}
430
    for key in TRIAGE_METRIC_KEYS:
431
        value = metrics.get(key)
432
        if key.endswith("_n") or key in ("branches_scored", "branches_expected"):
433
            fields[key] = value if isinstance(value, int) and value >= 0 else None
434
        elif value is None:
435
            fields[key] = None
436
        elif key.endswith("_count"):
437
            fields[key] = int(value)
438
        elif key in ("triage_recall", "triage_precision"):
439
            number = float(value)
440
            fields[key] = number if math.isfinite(number) and 0.0 <= number <= 1.0 else None
441
        else:
442
            fields[key] = value
443
    return fields
444
445
446
def validate_triage_metrics(metrics: Mapping[str, Any]) -> list[str]:
447
    errors: list[str] = []
448
    for key in TRIAGE_METRIC_KEYS:
449
        if key not in metrics:
450
            errors.append(f"missing triage metric {key!r}")
451
    for key in ("triage_recall", "triage_precision"):
452
        value = metrics.get(key)
453
        if value is not None and (
454
            isinstance(value, bool) or not isinstance(value, (int, float)) or not 0.0 <= float(value) <= 1.0
455
        ):
456
            errors.append(f"{key} must be null or within [0, 1]")
457
    for key in (
458
        "false_close_count",
459
        "close_eligible_n",
460
        "false_vcs_merge_safe_count",
461
        "vcs_merge_safe_n",
462
        "false_merge_allowed_count",
463
        "merge_allowed_n",
464
        "branches_scored",
465
        "branches_expected",
466
    ):
467
        value = metrics.get(key)
468
        if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value < 0):
469
            errors.append(f"{key} must be null or a non-negative integer")
470
    return errors