/oak
Log in
benchmarks
scripts/conflict_resolution_lane.py 949 lines · 34.1 KB · python Blame
1
#!/usr/bin/env python3
2
"""Run the smallest useful conflict-resolution workflow lane.
3
4
The lane consumes the synthetic bench-conflict-corpus-v1 repositories and their
5
*.RESOLUTION.json sidecar oracles. Text oracle content uses UTF-8 bytes exactly
6
as stored in the JSON string; newline normalization is not performed. For each
7
kind it clones the local fixture, attempts the merge from `ours` to `theirs`,
8
presents the unmerged paths, applies the oracle resolution, commits it, and
9
verifies the final HEAD payload hash plus any post-resolution check.
10
11
Rows use the existing workflow lane contract. Unsupported subjects and missing
12
commands/corpus data produce explicit skip rows (returncode 77), not success.
13
"""
14
15
from __future__ import annotations
16
17
import argparse
18
import base64
19
import hashlib
20
import json
21
import platform
22
import shutil
23
import subprocess
24
import sys
25
import tempfile
26
from dataclasses import dataclass
27
from datetime import datetime, timezone
28
from pathlib import Path
29
from typing import Any
30
31
from oakbench import environment as oakbench_environment
32
from oakbench import tokens as oakbench_tokens
33
from oakbench.execution import PEAK_RSS_SOURCE, run_timed
34
from oakbench.results import ResultsStore
35
from oakbench.rows import SKIP_RETURNCODE, row_returncode
36
37
ROOT = Path(__file__).resolve().parents[1]
38
DEFAULT_WORKDIR = Path(tempfile.gettempdir()) / "oak-conflict-resolution"
39
DEFAULT_RESULTS = ROOT / "results" / "conflict-resolution"
40
DEFAULT_KINDS = ("text-overlap-small", "adjacent-lines", "lockfile")
41
ORACLE_SUFFIX = ".RESOLUTION.json"
42
WORKFLOW = "conflict_resolution"
43
WORKFLOW_DESCRIPTION = "Merge a curated branch-pair conflict and apply the resolution oracle."
44
PROFILE = "conflict-resolution"
45
MEASUREMENT_SOURCE = "direct_cli_timed_subprocess"
46
GENERATOR = ROOT / "scripts" / "make_conflict_corpus.py"
47
48
49
@dataclass(frozen=True)
50
class Subject:
51
    name: str
52
    kind: str
53
    label: str
54
    bin: Path | None
55
56
57
def base_env() -> dict[str, str]:
58
    return oakbench_environment.base_env(
59
        author_name="Oak Conflict Bench",
60
        author_email="[email protected]",
61
        oak_author="oak-conflict-bench",
62
    )
63
64
65
def sha256_bytes(data: bytes) -> str:
66
    return hashlib.sha256(data).hexdigest()
67
68
69
def file_sha256(path: Path) -> str:
70
    return sha256_bytes(path.read_bytes())
71
72
73
def resolution_bytes(resolution: dict[str, Any]) -> bytes:
74
    if "content" in resolution:
75
        return str(resolution["content"]).encode("utf-8")
76
    return base64.b64decode(str(resolution["content_b64"]))
77
78
79
def load_oracle(corpus: Path, kind: str) -> tuple[Path | None, dict[str, Any] | None]:
80
    candidates = [
81
        corpus / f"{kind}{ORACLE_SUFFIX}",
82
        corpus / kind / "RESOLUTION.json",
83
    ]
84
    for path in candidates:
85
        if path.is_file():
86
            return path, json.loads(path.read_text())
87
    return None, None
88
89
90
def validate_oracle_schema(oracle: dict[str, Any]) -> str | None:
91
    if not isinstance(oracle.get("expected_conflict"), bool):
92
        return "missing_or_invalid_expected_conflict"
93
    paths = oracle.get("conflicted_paths")
94
    if not isinstance(paths, list) or any(not isinstance(path, str) for path in paths):
95
        return "missing_or_invalid_conflicted_paths"
96
    resolution = oracle.get("resolution")
97
    if not isinstance(resolution, dict):
98
        return "missing_or_invalid_resolution"
99
    if not isinstance(resolution.get("path"), str) or not resolution["path"]:
100
        return "missing_or_invalid_resolution_path"
101
    if not isinstance(resolution.get("content_sha256"), str) or not resolution["content_sha256"]:
102
        return "missing_or_invalid_resolution_content_sha256"
103
    has_text = isinstance(resolution.get("content"), str)
104
    has_b64 = isinstance(resolution.get("content_b64"), str)
105
    if has_text == has_b64:
106
        return "resolution_requires_exactly_one_of_content_or_content_b64"
107
    if has_b64:
108
        try:
109
            base64.b64decode(str(resolution["content_b64"]), validate=True)
110
        except (ValueError, TypeError):
111
            return "invalid_resolution_content_b64"
112
    check = oracle.get("post_merge_check")
113
    if check is not None:
114
        if not isinstance(check, dict):
115
            return "invalid_post_merge_check"
116
        command = check.get("command")
117
        if not isinstance(command, list) or any(not isinstance(part, str) for part in command):
118
            return "invalid_post_merge_check_command"
119
        if not isinstance(check.get("expected_returncode_clean_merge"), int):
120
            return "invalid_post_merge_check_expected_returncode_clean_merge"
121
    return None
122
123
124
def oracle_common_fields(kind: str, oracle_path: Path | None, oracle: dict[str, Any] | None) -> dict[str, Any]:
125
    resolution = oracle.get("resolution") if oracle else None
126
    expected_files = list(oracle.get("conflicted_paths", [])) if oracle else []
127
    check = oracle.get("post_merge_check") if oracle else None
128
    return {
129
        "conflict_kind": kind,
130
        "conflict_files_expected": expected_files,
131
        "conflict_file_count_expected": len(expected_files),
132
        "resolution_oracle_path": str(oracle_path) if oracle_path else None,
133
        "resolution_oracle_sidecar_sha256": file_sha256(oracle_path) if oracle_path else None,
134
        "resolution_oracle_payload_path": resolution.get("path") if isinstance(resolution, dict) else None,
135
        "resolution_oracle_content_sha256": (
136
            resolution.get("content_sha256") if isinstance(resolution, dict) else None
137
        ),
138
        "resolution_oracle_hash_verified": None,
139
        "post_merge_check_command": check.get("command") if isinstance(check, dict) else None,
140
        "post_merge_check_expected_returncode_clean_merge": (
141
            check.get("expected_returncode_clean_merge") if isinstance(check, dict) else None
142
        ),
143
        "post_merge_check_observed_returncode_clean_merge": None,
144
    }
145
146
147
def row_base(
148
    metadata: dict[str, Any],
149
    subject: Subject,
150
    kind: str,
151
    run_index: int,
152
    phase: str,
153
    operation: str,
154
    oracle_path: Path | None,
155
    oracle: dict[str, Any] | None,
156
) -> dict[str, Any]:
157
    return {
158
        **metadata,
159
        "subject": subject.name,
160
        "subject_kind": subject.kind,
161
        "subject_label": subject.label,
162
        "scenario": WORKFLOW,
163
        "workflow": WORKFLOW,
164
        "workflow_description": WORKFLOW_DESCRIPTION,
165
        "run": run_index,
166
        "phase": phase,
167
        "operation": operation,
168
        **oracle_common_fields(kind, oracle_path, oracle),
169
    }
170
171
172
def interaction_fields(command: list[str], capture: Any, *, tool_kind: str) -> dict[str, Any]:
173
    command_text = oakbench_environment.command_display(command)
174
    token_fields = oakbench_tokens.interaction_token_fields(
175
        command_text,
176
        capture.stdout_text,
177
        capture.stderr_text,
178
        capture.stdout_bytes,
179
        capture.stderr_bytes,
180
        capture.stdout_truncated,
181
        capture.stderr_truncated,
182
        tool_call_count=1,
183
    )
184
    is_vcs = tool_kind == "vcs"
185
    is_test = tool_kind == "test"
186
    return {
187
        "measurement_source": MEASUREMENT_SOURCE,
188
        "tool_call_count": 1,
189
        "terminal_tool_call_count": 1,
190
        "vcs_tool_call_count": 1 if is_vcs else 0,
191
        "test_tool_call_count": 1 if is_test else 0,
192
        "peak_rss_bytes": capture.peak_rss_bytes,
193
        "peak_rss_source": PEAK_RSS_SOURCE,
194
        "tool_calls": {
195
            "total": 1,
196
            "terminal": 1,
197
            "vcs": 1 if is_vcs else 0,
198
            "test": 1 if is_test else 0,
199
            tool_kind: 1,
200
        },
201
        **token_fields,
202
    }
203
204
205
def command_row(
206
    *,
207
    metadata: dict[str, Any],
208
    subject: Subject,
209
    kind: str,
210
    run_index: int,
211
    phase: str,
212
    operation: str,
213
    step_kind: str,
214
    command: list[str],
215
    cwd: Path,
216
    expected_returncodes: tuple[int, ...],
217
    oracle_path: Path | None,
218
    oracle: dict[str, Any] | None,
219
    admitted_output_chars: int,
220
) -> dict[str, Any]:
221
    capture = run_timed(command, cwd, base_env(), admitted_output_chars)
222
    expected = capture.returncode in expected_returncodes
223
    row = {
224
        **row_base(metadata, subject, kind, run_index, phase, operation, oracle_path, oracle),
225
        "step_kind": step_kind,
226
        "elapsed_ms": round(capture.elapsed_ms, 3),
227
        "returncode": 0 if expected else (capture.returncode or 1),
228
        "process_returncode": capture.returncode,
229
        "expected_returncodes": list(expected_returncodes),
230
        "command": command,
231
        **interaction_fields(command, capture, tool_kind=step_kind),
232
    }
233
    if not expected:
234
        row["stderr"] = capture.stderr_text[-4000:]
235
    return row
236
237
238
def skip_row(
239
    *,
240
    metadata: dict[str, Any],
241
    subject: Subject,
242
    kind: str,
243
    run_index: int,
244
    reason: str,
245
    oracle_path: Path | None = None,
246
    oracle: dict[str, Any] | None = None,
247
) -> dict[str, Any]:
248
    return {
249
        **row_base(metadata, subject, kind, run_index, "workflow", "workflow.skipped", oracle_path, oracle),
250
        "step_kind": "summary",
251
        "elapsed_ms": 0.0,
252
        "returncode": SKIP_RETURNCODE,
253
        "process_returncode": SKIP_RETURNCODE,
254
        "command": [],
255
        "skipped": True,
256
        "skip_reason": reason,
257
        "tool_call_count": 0,
258
        "terminal_tool_call_count": 0,
259
        "vcs_tool_call_count": 0,
260
        "test_tool_call_count": 0,
261
    }
262
263
264
def observed_unmerged(repo: Path, git_bin: Path) -> list[str]:
265
    proc = subprocess.run(
266
        [str(git_bin), "-C", str(repo), "diff", "--name-only", "--diff-filter=U"],
267
        env=base_env(),
268
        capture_output=True,
269
        text=True,
270
        check=False,
271
    )
272
    if proc.returncode != 0:
273
        return []
274
    return sorted(path for path in proc.stdout.splitlines() if path)
275
276
277
def apply_resolution(repo: Path, oracle_path: Path, *, trust_oracle_commands: bool = False) -> int:
278
    oracle = json.loads(oracle_path.read_text())
279
    schema_error = validate_oracle_schema(oracle)
280
    if schema_error is not None:
281
        print(f"invalid oracle schema: {schema_error}", file=sys.stderr)
282
        return 77
283
    resolution = oracle["resolution"]
284
    resolved_path = Path(str(resolution["path"]))
285
    if resolved_path.is_absolute() or ".." in resolved_path.parts:
286
        print(f"unsafe resolution path: {resolved_path}", file=sys.stderr)
287
        return 1
288
    target = repo / resolved_path
289
    target.parent.mkdir(parents=True, exist_ok=True)
290
    target.write_bytes(resolution_bytes(resolution))
291
292
    # Rename/rename or delete/edit conflicts can leave alternate unmerged paths.
293
    # The oracle declares the canonical surviving path; remove the rest.
294
    for rel in oracle.get("conflicted_paths", []):
295
        candidate = Path(str(rel))
296
        if candidate == resolved_path or candidate.is_absolute() or ".." in candidate.parts:
297
            continue
298
        path = repo / candidate
299
        if path.exists() and path.is_file():
300
            path.unlink()
301
302
    git = shutil.which("git")
303
    if git is None:
304
        print("git not available for staging oracle resolution", file=sys.stderr)
305
        return 3
306
    stage = subprocess.run([git, "-C", str(repo), "add", "-A"], env=base_env(), check=False)
307
    return stage.returncode
308
309
310
def check_oracle(repo: Path, oracle_path: Path, *, trust_oracle_commands: bool = False) -> int:
311
    oracle = json.loads(oracle_path.read_text())
312
    schema_error = validate_oracle_schema(oracle)
313
    if schema_error is not None:
314
        print(f"invalid oracle schema: {schema_error}", file=sys.stderr)
315
        return 77
316
    resolution = oracle["resolution"]
317
    rel = str(resolution["path"])
318
    expected = str(resolution["content_sha256"])
319
    git = shutil.which("git")
320
    if git is None:
321
        print("git not available for HEAD payload verification", file=sys.stderr)
322
        return 3
323
    payload = subprocess.run(
324
        [git, "-C", str(repo), "show", f"HEAD:{rel}"],
325
        env=base_env(),
326
        stdout=subprocess.PIPE,
327
        stderr=subprocess.PIPE,
328
        check=False,
329
    )
330
    if payload.returncode != 0:
331
        sys.stderr.write(payload.stderr.decode("utf-8", "replace"))
332
        return 1
333
    actual = sha256_bytes(payload.stdout)
334
    if actual != expected:
335
        print(f"sha256 mismatch for {rel}: expected {expected}, got {actual}", file=sys.stderr)
336
        return 1
337
338
    check = oracle.get("post_merge_check")
339
    post_resolution_returncode = None
340
    if check is not None and trust_oracle_commands:
341
        proc = subprocess.run(check["command"], cwd=repo, env=base_env(), check=False)
342
        post_resolution_returncode = proc.returncode
343
        if proc.returncode != 0:
344
            print(f"post-resolution check returned {proc.returncode}, expected 0", file=sys.stderr)
345
            return 1
346
    print(
347
        json.dumps(
348
            {
349
                "resolution_oracle_hash_verified": True,
350
                "resolution_oracle_payload_path": rel,
351
                "resolution_oracle_content_sha256": expected,
352
                "post_resolution_check_returncode": post_resolution_returncode,
353
            },
354
            sort_keys=True,
355
        )
356
    )
357
    return 0
358
359
360
def _payload_hash_verified(repo: Path, oracle: dict[str, Any], git_bin: Path) -> bool:
361
    rel = str(oracle["resolution"]["path"])
362
    expected = str(oracle["resolution"]["content_sha256"])
363
    proc = subprocess.run(
364
        [str(git_bin), "-C", str(repo), "show", f"HEAD:{rel}"],
365
        env=base_env(),
366
        stdout=subprocess.PIPE,
367
        stderr=subprocess.DEVNULL,
368
        check=False,
369
    )
370
    return proc.returncode == 0 and sha256_bytes(proc.stdout) == expected
371
372
373
def run_git_kind(
374
    *,
375
    subject: Subject,
376
    corpus: Path,
377
    kind: str,
378
    run_index: int,
379
    run_root: Path,
380
    metadata: dict[str, Any],
381
    admitted_output_chars: int,
382
    trust_oracle_commands: bool = False,
383
) -> list[dict[str, Any]]:
384
    oracle_path, oracle = load_oracle(corpus, kind)
385
    if oracle_path is None or oracle is None:
386
        return [
387
            skip_row(
388
                metadata=metadata,
389
                subject=subject,
390
                kind=kind,
391
                run_index=run_index,
392
                reason=f"conflict_oracle_missing:{corpus / f'{kind}{ORACLE_SUFFIX}'}",
393
            )
394
        ]
395
    schema_error = validate_oracle_schema(oracle)
396
    if schema_error is not None:
397
        return [
398
            skip_row(
399
                metadata=metadata,
400
                subject=subject,
401
                kind=kind,
402
                run_index=run_index,
403
                reason=f"conflict_oracle_malformed:{schema_error}",
404
                oracle_path=oracle_path,
405
                oracle=oracle,
406
            )
407
        ]
408
    repo_src = corpus / kind
409
    if not repo_src.is_dir():
410
        return [
411
            skip_row(
412
                metadata=metadata,
413
                subject=subject,
414
                kind=kind,
415
                run_index=run_index,
416
                reason=f"conflict_fixture_repo_missing:{repo_src}",
417
                oracle_path=oracle_path,
418
                oracle=oracle,
419
            )
420
        ]
421
    if subject.bin is None or not subject.bin.exists():
422
        return [
423
            skip_row(
424
                metadata=metadata,
425
                subject=subject,
426
                kind=kind,
427
                run_index=run_index,
428
                reason="git_command_unavailable",
429
                oracle_path=oracle_path,
430
                oracle=oracle,
431
            )
432
        ]
433
434
    repo = run_root / WORKFLOW / subject.name / kind / f"run-{run_index}"
435
    if repo.exists():
436
        shutil.rmtree(repo)
437
    repo.parent.mkdir(parents=True, exist_ok=True)
438
439
    rows: list[dict[str, Any]] = []
440
    git = str(subject.bin)
441
    clone = command_row(
442
        metadata=metadata,
443
        subject=subject,
444
        kind=kind,
445
        run_index=run_index,
446
        phase="setup",
447
        operation="setup.clone",
448
        step_kind="vcs",
449
        command=[git, "clone", "--quiet", str(repo_src), str(repo)],
450
        cwd=repo.parent,
451
        expected_returncodes=(0,),
452
        oracle_path=oracle_path,
453
        oracle=oracle,
454
        admitted_output_chars=admitted_output_chars,
455
    )
456
    rows.append(clone)
457
    if row_returncode(clone) != 0:
458
        rows.append(sum_rows(rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle))
459
        return rows
460
461
    setup_commands = [
462
        ("setup.checkout_ours", [git, "checkout", "-q", "ours"]),
463
        ("setup.theirs_branch", [git, "branch", "-f", "theirs", "origin/theirs"]),
464
    ]
465
    for operation, command in setup_commands:
466
        row = command_row(
467
            metadata=metadata,
468
            subject=subject,
469
            kind=kind,
470
            run_index=run_index,
471
            phase="setup",
472
            operation=operation,
473
            step_kind="vcs",
474
            command=command,
475
            cwd=repo,
476
            expected_returncodes=(0,),
477
            oracle_path=oracle_path,
478
            oracle=oracle,
479
            admitted_output_chars=admitted_output_chars,
480
        )
481
        rows.append(row)
482
        if row_returncode(row) != 0:
483
            rows.append(sum_rows(rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle))
484
            return rows
485
486
    setup_rows = list(rows)
487
    workflow_rows: list[dict[str, Any]] = []
488
    expected_merge_codes = (1,) if bool(oracle["expected_conflict"]) else (0,)
489
    merge = command_row(
490
        metadata=metadata,
491
        subject=subject,
492
        kind=kind,
493
        run_index=run_index,
494
        phase="workflow",
495
        operation="conflict.merge",
496
        step_kind="vcs",
497
        command=[git, "merge", "--no-edit", "theirs"],
498
        cwd=repo,
499
        expected_returncodes=expected_merge_codes,
500
        oracle_path=oracle_path,
501
        oracle=oracle,
502
        admitted_output_chars=admitted_output_chars,
503
    )
504
    observed = observed_unmerged(repo, subject.bin)
505
    merge["conflict_files_observed"] = observed
506
    merge["conflict_file_count_observed"] = len(observed)
507
    merge["merge_conflict_observed"] = merge["process_returncode"] != 0
508
    if observed != sorted(oracle.get("conflicted_paths", [])):
509
        merge["returncode"] = 1
510
        merge["stderr"] = "observed conflict files did not match oracle"
511
    workflow_rows.append(merge)
512
    if row_returncode(merge) != 0:
513
        return [
514
            *setup_rows,
515
            sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
516
            *workflow_rows,
517
            sum_rows(workflow_rows, metadata, subject, kind, run_index, "workflow", "workflow.total", oracle_path, oracle),
518
        ]
519
520
    present = command_row(
521
        metadata=metadata,
522
        subject=subject,
523
        kind=kind,
524
        run_index=run_index,
525
        phase="workflow",
526
        operation="conflict.present",
527
        step_kind="vcs",
528
        command=[git, "diff", "--name-only", "--diff-filter=U"],
529
        cwd=repo,
530
        expected_returncodes=(0,),
531
        oracle_path=oracle_path,
532
        oracle=oracle,
533
        admitted_output_chars=admitted_output_chars,
534
    )
535
    present_files = observed_unmerged(repo, subject.bin)
536
    present["conflict_files_observed"] = present_files
537
    present["conflict_file_count_observed"] = len(present_files)
538
    workflow_rows.append(present)
539
540
    check = oracle.get("post_merge_check")
541
    if isinstance(check, dict) and trust_oracle_commands:
542
        expected_clean = int(check["expected_returncode_clean_merge"])
543
        check_row = command_row(
544
            metadata=metadata,
545
            subject=subject,
546
            kind=kind,
547
            run_index=run_index,
548
            phase="workflow",
549
            operation="conflict.clean_merge_check",
550
            step_kind="test",
551
            command=list(check["command"]),
552
            cwd=repo,
553
            expected_returncodes=(expected_clean,),
554
            oracle_path=oracle_path,
555
            oracle=oracle,
556
            admitted_output_chars=admitted_output_chars,
557
        )
558
        check_row["post_merge_check_observed_returncode_clean_merge"] = check_row["process_returncode"]
559
        workflow_rows.append(check_row)
560
        if row_returncode(check_row) != 0:
561
            return [
562
                *setup_rows,
563
                sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
564
                *workflow_rows,
565
                sum_rows(
566
                    workflow_rows,
567
                    metadata,
568
                    subject,
569
                    kind,
570
                    run_index,
571
                    "workflow",
572
                    "workflow.total",
573
                    oracle_path,
574
                    oracle,
575
                ),
576
            ]
577
578
    apply_row = command_row(
579
        metadata=metadata,
580
        subject=subject,
581
        kind=kind,
582
        run_index=run_index,
583
        phase="workflow",
584
        operation="conflict.resolve.oracle",
585
        step_kind="edit",
586
        command=[
587
            sys.executable,
588
            str(Path(__file__).resolve()),
589
            "_apply-oracle",
590
            "--repo",
591
            str(repo),
592
            "--oracle",
593
            str(oracle_path),
594
            *(["--trust-oracle-commands"] if trust_oracle_commands else []),
595
        ],
596
        cwd=repo,
597
        expected_returncodes=(0,),
598
        oracle_path=oracle_path,
599
        oracle=oracle,
600
        admitted_output_chars=admitted_output_chars,
601
    )
602
    workflow_rows.append(apply_row)
603
    if row_returncode(apply_row) != 0:
604
        return [
605
            *setup_rows,
606
            sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
607
            *workflow_rows,
608
            sum_rows(workflow_rows, metadata, subject, kind, run_index, "workflow", "workflow.total", oracle_path, oracle),
609
        ]
610
611
    commit = command_row(
612
        metadata=metadata,
613
        subject=subject,
614
        kind=kind,
615
        run_index=run_index,
616
        phase="workflow",
617
        operation="conflict.resolve.commit",
618
        step_kind="vcs",
619
        command=[git, "commit", "-q", "-m", f"resolve {kind} from oracle"],
620
        cwd=repo,
621
        expected_returncodes=(0,),
622
        oracle_path=oracle_path,
623
        oracle=oracle,
624
        admitted_output_chars=admitted_output_chars,
625
    )
626
    workflow_rows.append(commit)
627
    if row_returncode(commit) != 0:
628
        return [
629
            *setup_rows,
630
            sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
631
            *workflow_rows,
632
            sum_rows(workflow_rows, metadata, subject, kind, run_index, "workflow", "workflow.total", oracle_path, oracle),
633
        ]
634
635
    verify = command_row(
636
        metadata=metadata,
637
        subject=subject,
638
        kind=kind,
639
        run_index=run_index,
640
        phase="workflow",
641
        operation="conflict.oracle.check",
642
        step_kind="test",
643
        command=[
644
            sys.executable,
645
            str(Path(__file__).resolve()),
646
            "_check-oracle",
647
            "--repo",
648
            str(repo),
649
            "--oracle",
650
            str(oracle_path),
651
            *(["--trust-oracle-commands"] if trust_oracle_commands else []),
652
        ],
653
        cwd=repo,
654
        expected_returncodes=(0,),
655
        oracle_path=oracle_path,
656
        oracle=oracle,
657
        admitted_output_chars=admitted_output_chars,
658
    )
659
    verify["resolution_oracle_hash_verified"] = _payload_hash_verified(repo, oracle, subject.bin)
660
    workflow_rows.append(verify)
661
662
    return [
663
        *setup_rows,
664
        sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
665
        *workflow_rows,
666
        sum_rows(workflow_rows, metadata, subject, kind, run_index, "workflow", "workflow.total", oracle_path, oracle),
667
    ]
668
669
670
def sum_rows(
671
    rows: list[dict[str, Any]],
672
    metadata: dict[str, Any],
673
    subject: Subject,
674
    kind: str,
675
    run_index: int,
676
    phase: str,
677
    operation: str,
678
    oracle_path: Path | None,
679
    oracle: dict[str, Any] | None,
680
) -> dict[str, Any]:
681
    elapsed_ms = sum(float(row.get("elapsed_ms", 0.0)) for row in rows)
682
    failures = [row for row in rows if row_returncode(row) != 0]
683
    total = {
684
        **row_base(metadata, subject, kind, run_index, phase, operation, oracle_path, oracle),
685
        "step_kind": "summary",
686
        "elapsed_ms": round(elapsed_ms, 3),
687
        "returncode": 1 if failures else 0,
688
        "process_returncode": 1 if failures else 0,
689
        "command": [row["command"] for row in rows],
690
        "tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("tool_call_count")) for row in rows),
691
        "terminal_tool_call_count": sum(
692
            oakbench_tokens.int_or_zero(row.get("terminal_tool_call_count")) for row in rows
693
        ),
694
        "vcs_tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("vcs_tool_call_count")) for row in rows),
695
        "test_tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("test_tool_call_count")) for row in rows),
696
        "steps_total": len(rows),
697
        "steps_succeeded": len(rows) - len(failures),
698
        "steps_failed": len(failures),
699
        "step_failure_rate": (len(failures) / len(rows)) if rows else None,
700
        "summarized_operations": [row["operation"] for row in rows],
701
        **oakbench_tokens.summed_token_fields(
702
            rows, "sum_of_conflict_workflow_steps_command_plus_admitted_output_chars_div_4"
703
        ),
704
    }
705
    observed_sets = [row.get("conflict_files_observed") for row in rows if "conflict_files_observed" in row]
706
    if observed_sets:
707
        observed = sorted(set().union(*(set(items) for items in observed_sets if isinstance(items, list))))
708
        total["conflict_files_observed"] = observed
709
        total["conflict_file_count_observed"] = len(observed)
710
    hash_checks = [row.get("resolution_oracle_hash_verified") for row in rows if row.get("operation") == "conflict.oracle.check"]
711
    if hash_checks:
712
        total["resolution_oracle_hash_verified"] = bool(hash_checks[-1])
713
    clean_checks = [
714
        row.get("post_merge_check_observed_returncode_clean_merge")
715
        for row in rows
716
        if row.get("post_merge_check_observed_returncode_clean_merge") is not None
717
    ]
718
    if clean_checks:
719
        total["post_merge_check_observed_returncode_clean_merge"] = clean_checks[-1]
720
    return total
721
722
723
def run_subject_kind(
724
    *,
725
    subject: Subject,
726
    corpus: Path,
727
    kind: str,
728
    run_index: int,
729
    run_root: Path,
730
    metadata: dict[str, Any],
731
    admitted_output_chars: int,
732
    trust_oracle_commands: bool = False,
733
) -> list[dict[str, Any]]:
734
    oracle_path, oracle = load_oracle(corpus, kind)
735
    if subject.kind != "git":
736
        return [
737
            skip_row(
738
                metadata=metadata,
739
                subject=subject,
740
                kind=kind,
741
                run_index=run_index,
742
                reason="conflict_resolution_lane_supports_git_local_fixtures_only",
743
                oracle_path=oracle_path,
744
                oracle=oracle,
745
            )
746
        ]
747
    return run_git_kind(
748
        subject=subject,
749
        corpus=corpus,
750
        kind=kind,
751
        run_index=run_index,
752
        run_root=run_root,
753
        metadata=metadata,
754
        admitted_output_chars=admitted_output_chars,
755
        trust_oracle_commands=trust_oracle_commands,
756
    )
757
758
759
def parse_subjects(raw: str, git_bin: Path | None) -> list[Subject]:
760
    subjects: list[Subject] = []
761
    for name in [item.strip() for item in raw.split(",") if item.strip()]:
762
        if name == "git":
763
            resolved = git_bin or (Path(shutil.which("git")) if shutil.which("git") else None)
764
            subjects.append(Subject("git", "git", "Git", resolved))
765
        elif name.startswith("oak"):
766
            oak = Path(shutil.which("oak")) if shutil.which("oak") else None
767
            subjects.append(Subject(name, "oak", "Oak", oak))
768
        else:
769
            subjects.append(Subject(name, name, name, None))
770
    return subjects
771
772
773
def ensure_corpus(args: argparse.Namespace, kinds: list[str]) -> str | None:
774
    missing = [
775
        kind
776
        for kind in kinds
777
        if not (args.corpus / kind).is_dir() or load_oracle(args.corpus, kind) == (None, None)
778
    ]
779
    if not missing:
780
        return None
781
    if not args.generate_corpus:
782
        return f"conflict_corpus_missing:{args.corpus}"
783
    git = shutil.which("git")
784
    if git is None:
785
        return "git_command_unavailable_for_conflict_corpus_generation"
786
    args.corpus.parent.mkdir(parents=True, exist_ok=True)
787
    command = [
788
        sys.executable,
789
        str(GENERATOR),
790
        "--out",
791
        str(args.corpus),
792
        "--seed",
793
        args.seed,
794
        "--kinds",
795
        ",".join(missing),
796
    ]
797
    if args.quick_corpus:
798
        command.append("--quick")
799
    proc = subprocess.run(command, cwd=ROOT, env=base_env(), check=False)
800
    if proc.returncode == 3:
801
        return "git_command_unavailable_for_conflict_corpus_generation"
802
    if proc.returncode != 0:
803
        return f"conflict_corpus_generation_failed:{proc.returncode}"
804
    return None
805
806
807
def metadata(bench_id: str, track: str) -> dict[str, Any]:
808
    return {
809
        "bench_id": bench_id,
810
        "profile": PROFILE,
811
        "benchmark_track": track,
812
        "timestamp_utc": datetime.now(timezone.utc).isoformat(),
813
        "host": platform.node(),
814
        "platform": platform.platform(),
815
    }
816
817
818
def summary_text(rows: list[dict[str, Any]]) -> str:
819
    totals = [row for row in rows if row.get("operation") == "workflow.total"]
820
    skips = [row for row in rows if row.get("returncode") == SKIP_RETURNCODE]
821
    failures = [row for row in totals if row_returncode(row) != 0]
822
    lines = [
823
        "# Conflict Resolution Lane",
824
        "",
825
        f"Rows: {len(rows)}",
826
        f"Workflow totals: {len(totals)}",
827
        f"Skipped rows: {len(skips)}",
828
        f"Failed workflow totals: {len(failures)}",
829
        "",
830
        "| kind | subject | result | observed files | oracle hash |",
831
        "| --- | --- | --- | --- | --- |",
832
    ]
833
    for row in totals:
834
        result = "ok" if row_returncode(row) == 0 else "failed"
835
        lines.append(
836
            f"| `{row['conflict_kind']}` | `{row['subject']}` | {result} | "
837
            f"{row.get('conflict_file_count_observed')} | {row.get('resolution_oracle_hash_verified')} |"
838
        )
839
    for row in skips:
840
        lines.append(f"| `{row['conflict_kind']}` | `{row['subject']}` | skipped: {row['skip_reason']} | | |")
841
    return "\n".join(lines) + "\n"
842
843
844
def has_lane_failures(rows: list[dict[str, Any]]) -> bool:
845
    return any(row_returncode(row) not in (0, SKIP_RETURNCODE) for row in rows)
846
847
848
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
849
    parser = argparse.ArgumentParser(description=__doc__)
850
    parser.add_argument("--corpus", type=Path, default=DEFAULT_WORKDIR / "bench-conflict-corpus-v1")
851
    parser.add_argument("--generate-corpus", action="store_true")
852
    parser.add_argument("--quick-corpus", action="store_true")
853
    parser.add_argument("--seed", default="oakbench-conflicts-v1")
854
    parser.add_argument("--kinds", default=",".join(DEFAULT_KINDS))
855
    parser.add_argument("--subjects", default="git")
856
    parser.add_argument("--runs", type=int, default=1)
857
    parser.add_argument("--workdir", type=Path, default=DEFAULT_WORKDIR)
858
    parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
859
    parser.add_argument("--git-bin", type=Path)
860
    parser.add_argument("--keep-workdirs", action="store_true")
861
    parser.add_argument("--admitted-output-chars", type=int, default=20_000)
862
    parser.add_argument("--track", default="agent-default")
863
    parser.add_argument(
864
        "--trust-oracle-commands",
865
        action="store_true",
866
        help="execute post_merge_check.command from corpus oracle JSON; use only with trusted corpora",
867
    )
868
    parser.add_argument("_internal", nargs="*")
869
    return parser.parse_args(argv)
870
871
872
def main(argv: list[str] | None = None) -> int:
873
    if argv and argv[0] == "_apply-oracle":
874
        parser = argparse.ArgumentParser()
875
        parser.add_argument("_cmd")
876
        parser.add_argument("--repo", type=Path, required=True)
877
        parser.add_argument("--oracle", type=Path, required=True)
878
        parser.add_argument("--trust-oracle-commands", action="store_true")
879
        args = parser.parse_args(argv)
880
        return apply_resolution(
881
            args.repo,
882
            args.oracle,
883
            trust_oracle_commands=args.trust_oracle_commands,
884
        )
885
    if argv and argv[0] == "_check-oracle":
886
        parser = argparse.ArgumentParser()
887
        parser.add_argument("_cmd")
888
        parser.add_argument("--repo", type=Path, required=True)
889
        parser.add_argument("--oracle", type=Path, required=True)
890
        parser.add_argument("--trust-oracle-commands", action="store_true")
891
        args = parser.parse_args(argv)
892
        return check_oracle(
893
            args.repo,
894
            args.oracle,
895
            trust_oracle_commands=args.trust_oracle_commands,
896
        )
897
898
    args = parse_args(argv)
899
    kinds = [kind.strip() for kind in args.kinds.split(",") if kind.strip()]
900
    if not kinds:
901
        raise SystemExit("--kinds must name at least one conflict kind")
902
    subjects = parse_subjects(args.subjects, args.git_bin)
903
    if not subjects:
904
        raise SystemExit("--subjects must name at least one subject")
905
906
    bench_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
907
    meta = metadata(bench_id, args.track)
908
    rows: list[dict[str, Any]] = []
909
910
    corpus_problem = ensure_corpus(args, kinds)
911
    for run_index in range(args.runs):
912
        for kind in kinds:
913
            for subject in subjects:
914
                if corpus_problem is not None:
915
                    rows.append(
916
                        skip_row(
917
                            metadata=meta,
918
                            subject=subject,
919
                            kind=kind,
920
                            run_index=run_index,
921
                            reason=corpus_problem,
922
                        )
923
                    )
924
                else:
925
                    rows.extend(
926
                        run_subject_kind(
927
                            subject=subject,
928
                            corpus=args.corpus,
929
                            kind=kind,
930
                            run_index=run_index,
931
                            run_root=args.workdir / "runs",
932
                            metadata=meta,
933
                            admitted_output_chars=args.admitted_output_chars,
934
                            trust_oracle_commands=args.trust_oracle_commands,
935
                        )
936
                    )
937
938
    store = ResultsStore(args.results, lane="workflow", filename_suffix="conflict-resolution")
939
    raw_path, summary_path = store.write(bench_id, rows, summary_text(rows))
940
    print(f"Wrote {len(rows)} rows to {raw_path}")
941
    if summary_path:
942
        print(f"Wrote summary to {summary_path}")
943
    if not args.keep_workdirs:
944
        shutil.rmtree(args.workdir / "runs", ignore_errors=True)
945
    return 1 if has_lane_failures(rows) else 0
946
947
948
if __name__ == "__main__":
949
    raise SystemExit(main(sys.argv[1:]))