88e86b3faccd
Rebuild benchmarks as a clean single-root reposi
2 months ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Run the smallest useful conflict-resolution workflow lane.
|
| 3 |
|
| 4 | The lane consumes the synthetic bench-conflict-corpus-v1 repositories and their
|
| 5 | *.RESOLUTION.json sidecar oracles. Text oracle content uses UTF-8 bytes exactly
|
| 6 | as stored in the JSON string; newline normalization is not performed. For each
|
| 7 | kind it clones the local fixture, attempts the merge from `ours` to `theirs`,
|
| 8 | presents the unmerged paths, applies the oracle resolution, commits it, and
|
| 9 | verifies the final HEAD payload hash plus any post-resolution check.
|
| 10 |
|
| 11 | Rows use the existing workflow lane contract. Unsupported subjects and missing
|
| 12 | commands/corpus data produce explicit skip rows (returncode 77), not success.
|
| 13 | """
|
| 14 |
|
| 15 | from __future__ import annotations
|
| 16 |
|
| 17 | import argparse
|
| 18 | import base64
|
| 19 | import hashlib
|
| 20 | import json
|
| 21 | import platform
|
| 22 | import shutil
|
| 23 | import subprocess
|
| 24 | import sys
|
| 25 | import tempfile
|
| 26 | from dataclasses import dataclass
|
| 27 | from datetime import datetime, timezone
|
| 28 | from pathlib import Path
|
| 29 | from typing import Any
|
| 30 |
|
| 31 | from oakbench import environment as oakbench_environment
|
| 32 | from oakbench import tokens as oakbench_tokens
|
| 33 | from oakbench.execution import PEAK_RSS_SOURCE, run_timed
|
| 34 | from oakbench.results import ResultsStore
|
| 35 | from oakbench.rows import SKIP_RETURNCODE, row_returncode
|
| 36 |
|
| 37 | ROOT = Path(__file__).resolve().parents[1]
|
| 38 | DEFAULT_WORKDIR = Path(tempfile.gettempdir()) / "oak-conflict-resolution"
|
| 39 | DEFAULT_RESULTS = ROOT / "results" / "conflict-resolution"
|
| 40 | DEFAULT_KINDS = ("text-overlap-small", "adjacent-lines", "lockfile")
|
| 41 | ORACLE_SUFFIX = ".RESOLUTION.json"
|
| 42 | WORKFLOW = "conflict_resolution"
|
| 43 | WORKFLOW_DESCRIPTION = "Merge a curated branch-pair conflict and apply the resolution oracle."
|
| 44 | PROFILE = "conflict-resolution"
|
| 45 | MEASUREMENT_SOURCE = "direct_cli_timed_subprocess"
|
| 46 | GENERATOR = ROOT / "scripts" / "make_conflict_corpus.py"
|
| 47 |
|
| 48 |
|
| 49 | @dataclass(frozen=True)
|
| 50 | class Subject:
|
| 51 | name: str
|
| 52 | kind: str
|
| 53 | label: str
|
| 54 | bin: Path | None
|
| 55 |
|
| 56 |
|
| 57 | def base_env() -> dict[str, str]:
|
| 58 | return oakbench_environment.base_env(
|
| 59 | author_name="Oak Conflict Bench",
|
| 60 | author_email="[email protected]",
|
| 61 | oak_author="oak-conflict-bench",
|
| 62 | )
|
| 63 |
|
| 64 |
|
| 65 | def sha256_bytes(data: bytes) -> str:
|
| 66 | return hashlib.sha256(data).hexdigest()
|
| 67 |
|
| 68 |
|
| 69 | def file_sha256(path: Path) -> str:
|
| 70 | return sha256_bytes(path.read_bytes())
|
| 71 |
|
| 72 |
|
| 73 | def resolution_bytes(resolution: dict[str, Any]) -> bytes:
|
| 74 | if "content" in resolution:
|
| 75 | return str(resolution["content"]).encode("utf-8")
|
| 76 | return base64.b64decode(str(resolution["content_b64"]))
|
| 77 |
|
| 78 |
|
| 79 | def load_oracle(corpus: Path, kind: str) -> tuple[Path | None, dict[str, Any] | None]:
|
| 80 | candidates = [
|
| 81 | corpus / f"{kind}{ORACLE_SUFFIX}",
|
| 82 | corpus / kind / "RESOLUTION.json",
|
| 83 | ]
|
| 84 | for path in candidates:
|
| 85 | if path.is_file():
|
| 86 | return path, json.loads(path.read_text())
|
| 87 | return None, None
|
| 88 |
|
| 89 |
|
| 90 | def validate_oracle_schema(oracle: dict[str, Any]) -> str | None:
|
| 91 | if not isinstance(oracle.get("expected_conflict"), bool):
|
| 92 | return "missing_or_invalid_expected_conflict"
|
| 93 | paths = oracle.get("conflicted_paths")
|
| 94 | if not isinstance(paths, list) or any(not isinstance(path, str) for path in paths):
|
| 95 | return "missing_or_invalid_conflicted_paths"
|
| 96 | resolution = oracle.get("resolution")
|
| 97 | if not isinstance(resolution, dict):
|
| 98 | return "missing_or_invalid_resolution"
|
| 99 | if not isinstance(resolution.get("path"), str) or not resolution["path"]:
|
| 100 | return "missing_or_invalid_resolution_path"
|
| 101 | if not isinstance(resolution.get("content_sha256"), str) or not resolution["content_sha256"]:
|
| 102 | return "missing_or_invalid_resolution_content_sha256"
|
| 103 | has_text = isinstance(resolution.get("content"), str)
|
| 104 | has_b64 = isinstance(resolution.get("content_b64"), str)
|
| 105 | if has_text == has_b64:
|
| 106 | return "resolution_requires_exactly_one_of_content_or_content_b64"
|
| 107 | if has_b64:
|
| 108 | try:
|
| 109 | base64.b64decode(str(resolution["content_b64"]), validate=True)
|
| 110 | except (ValueError, TypeError):
|
| 111 | return "invalid_resolution_content_b64"
|
| 112 | check = oracle.get("post_merge_check")
|
| 113 | if check is not None:
|
| 114 | if not isinstance(check, dict):
|
| 115 | return "invalid_post_merge_check"
|
| 116 | command = check.get("command")
|
| 117 | if not isinstance(command, list) or any(not isinstance(part, str) for part in command):
|
| 118 | return "invalid_post_merge_check_command"
|
| 119 | if not isinstance(check.get("expected_returncode_clean_merge"), int):
|
| 120 | return "invalid_post_merge_check_expected_returncode_clean_merge"
|
| 121 | return None
|
| 122 |
|
| 123 |
|
| 124 | def oracle_common_fields(kind: str, oracle_path: Path | None, oracle: dict[str, Any] | None) -> dict[str, Any]:
|
| 125 | resolution = oracle.get("resolution") if oracle else None
|
| 126 | expected_files = list(oracle.get("conflicted_paths", [])) if oracle else []
|
| 127 | check = oracle.get("post_merge_check") if oracle else None
|
| 128 | return {
|
| 129 | "conflict_kind": kind,
|
| 130 | "conflict_files_expected": expected_files,
|
| 131 | "conflict_file_count_expected": len(expected_files),
|
| 132 | "resolution_oracle_path": str(oracle_path) if oracle_path else None,
|
| 133 | "resolution_oracle_sidecar_sha256": file_sha256(oracle_path) if oracle_path else None,
|
| 134 | "resolution_oracle_payload_path": resolution.get("path") if isinstance(resolution, dict) else None,
|
| 135 | "resolution_oracle_content_sha256": (
|
| 136 | resolution.get("content_sha256") if isinstance(resolution, dict) else None
|
| 137 | ),
|
| 138 | "resolution_oracle_hash_verified": None,
|
| 139 | "post_merge_check_command": check.get("command") if isinstance(check, dict) else None,
|
| 140 | "post_merge_check_expected_returncode_clean_merge": (
|
| 141 | check.get("expected_returncode_clean_merge") if isinstance(check, dict) else None
|
| 142 | ),
|
| 143 | "post_merge_check_observed_returncode_clean_merge": None,
|
| 144 | }
|
| 145 |
|
| 146 |
|
| 147 | def row_base(
|
| 148 | metadata: dict[str, Any],
|
| 149 | subject: Subject,
|
| 150 | kind: str,
|
| 151 | run_index: int,
|
| 152 | phase: str,
|
| 153 | operation: str,
|
| 154 | oracle_path: Path | None,
|
| 155 | oracle: dict[str, Any] | None,
|
| 156 | ) -> dict[str, Any]:
|
| 157 | return {
|
| 158 | **metadata,
|
| 159 | "subject": subject.name,
|
| 160 | "subject_kind": subject.kind,
|
| 161 | "subject_label": subject.label,
|
| 162 | "scenario": WORKFLOW,
|
| 163 | "workflow": WORKFLOW,
|
| 164 | "workflow_description": WORKFLOW_DESCRIPTION,
|
| 165 | "run": run_index,
|
| 166 | "phase": phase,
|
| 167 | "operation": operation,
|
| 168 | **oracle_common_fields(kind, oracle_path, oracle),
|
| 169 | }
|
| 170 |
|
| 171 |
|
| 172 | def interaction_fields(command: list[str], capture: Any, *, tool_kind: str) -> dict[str, Any]:
|
| 173 | command_text = oakbench_environment.command_display(command)
|
| 174 | token_fields = oakbench_tokens.interaction_token_fields(
|
| 175 | command_text,
|
| 176 | capture.stdout_text,
|
| 177 | capture.stderr_text,
|
| 178 | capture.stdout_bytes,
|
| 179 | capture.stderr_bytes,
|
| 180 | capture.stdout_truncated,
|
| 181 | capture.stderr_truncated,
|
| 182 | tool_call_count=1,
|
| 183 | )
|
| 184 | is_vcs = tool_kind == "vcs"
|
| 185 | is_test = tool_kind == "test"
|
| 186 | return {
|
| 187 | "measurement_source": MEASUREMENT_SOURCE,
|
| 188 | "tool_call_count": 1,
|
| 189 | "terminal_tool_call_count": 1,
|
| 190 | "vcs_tool_call_count": 1 if is_vcs else 0,
|
| 191 | "test_tool_call_count": 1 if is_test else 0,
|
| 192 | "peak_rss_bytes": capture.peak_rss_bytes,
|
| 193 | "peak_rss_source": PEAK_RSS_SOURCE,
|
| 194 | "tool_calls": {
|
| 195 | "total": 1,
|
| 196 | "terminal": 1,
|
| 197 | "vcs": 1 if is_vcs else 0,
|
| 198 | "test": 1 if is_test else 0,
|
| 199 | tool_kind: 1,
|
| 200 | },
|
| 201 | **token_fields,
|
| 202 | }
|
| 203 |
|
| 204 |
|
| 205 | def command_row(
|
| 206 | *,
|
| 207 | metadata: dict[str, Any],
|
| 208 | subject: Subject,
|
| 209 | kind: str,
|
| 210 | run_index: int,
|
| 211 | phase: str,
|
| 212 | operation: str,
|
| 213 | step_kind: str,
|
| 214 | command: list[str],
|
| 215 | cwd: Path,
|
| 216 | expected_returncodes: tuple[int, ...],
|
| 217 | oracle_path: Path | None,
|
| 218 | oracle: dict[str, Any] | None,
|
| 219 | admitted_output_chars: int,
|
| 220 | ) -> dict[str, Any]:
|
| 221 | capture = run_timed(command, cwd, base_env(), admitted_output_chars)
|
| 222 | expected = capture.returncode in expected_returncodes
|
| 223 | row = {
|
| 224 | **row_base(metadata, subject, kind, run_index, phase, operation, oracle_path, oracle),
|
| 225 | "step_kind": step_kind,
|
| 226 | "elapsed_ms": round(capture.elapsed_ms, 3),
|
| 227 | "returncode": 0 if expected else (capture.returncode or 1),
|
| 228 | "process_returncode": capture.returncode,
|
| 229 | "expected_returncodes": list(expected_returncodes),
|
| 230 | "command": command,
|
| 231 | **interaction_fields(command, capture, tool_kind=step_kind),
|
| 232 | }
|
| 233 | if not expected:
|
| 234 | row["stderr"] = capture.stderr_text[-4000:]
|
| 235 | return row
|
| 236 |
|
| 237 |
|
| 238 | def skip_row(
|
| 239 | *,
|
| 240 | metadata: dict[str, Any],
|
| 241 | subject: Subject,
|
| 242 | kind: str,
|
| 243 | run_index: int,
|
| 244 | reason: str,
|
| 245 | oracle_path: Path | None = None,
|
| 246 | oracle: dict[str, Any] | None = None,
|
| 247 | ) -> dict[str, Any]:
|
| 248 | return {
|
| 249 | **row_base(metadata, subject, kind, run_index, "workflow", "workflow.skipped", oracle_path, oracle),
|
| 250 | "step_kind": "summary",
|
| 251 | "elapsed_ms": 0.0,
|
| 252 | "returncode": SKIP_RETURNCODE,
|
| 253 | "process_returncode": SKIP_RETURNCODE,
|
| 254 | "command": [],
|
| 255 | "skipped": True,
|
| 256 | "skip_reason": reason,
|
| 257 | "tool_call_count": 0,
|
| 258 | "terminal_tool_call_count": 0,
|
| 259 | "vcs_tool_call_count": 0,
|
| 260 | "test_tool_call_count": 0,
|
| 261 | }
|
| 262 |
|
| 263 |
|
| 264 | def observed_unmerged(repo: Path, git_bin: Path) -> list[str]:
|
| 265 | proc = subprocess.run(
|
| 266 | [str(git_bin), "-C", str(repo), "diff", "--name-only", "--diff-filter=U"],
|
| 267 | env=base_env(),
|
| 268 | capture_output=True,
|
| 269 | text=True,
|
| 270 | check=False,
|
| 271 | )
|
| 272 | if proc.returncode != 0:
|
| 273 | return []
|
| 274 | return sorted(path for path in proc.stdout.splitlines() if path)
|
| 275 |
|
| 276 |
|
| 277 | def apply_resolution(repo: Path, oracle_path: Path, *, trust_oracle_commands: bool = False) -> int:
|
| 278 | oracle = json.loads(oracle_path.read_text())
|
| 279 | schema_error = validate_oracle_schema(oracle)
|
| 280 | if schema_error is not None:
|
| 281 | print(f"invalid oracle schema: {schema_error}", file=sys.stderr)
|
| 282 | return 77
|
| 283 | resolution = oracle["resolution"]
|
| 284 | resolved_path = Path(str(resolution["path"]))
|
| 285 | if resolved_path.is_absolute() or ".." in resolved_path.parts:
|
| 286 | print(f"unsafe resolution path: {resolved_path}", file=sys.stderr)
|
| 287 | return 1
|
| 288 | target = repo / resolved_path
|
| 289 | target.parent.mkdir(parents=True, exist_ok=True)
|
| 290 | target.write_bytes(resolution_bytes(resolution))
|
| 291 |
|
| 292 | # Rename/rename or delete/edit conflicts can leave alternate unmerged paths.
|
| 293 | # The oracle declares the canonical surviving path; remove the rest.
|
| 294 | for rel in oracle.get("conflicted_paths", []):
|
| 295 | candidate = Path(str(rel))
|
| 296 | if candidate == resolved_path or candidate.is_absolute() or ".." in candidate.parts:
|
| 297 | continue
|
| 298 | path = repo / candidate
|
| 299 | if path.exists() and path.is_file():
|
| 300 | path.unlink()
|
| 301 |
|
| 302 | git = shutil.which("git")
|
| 303 | if git is None:
|
| 304 | print("git not available for staging oracle resolution", file=sys.stderr)
|
| 305 | return 3
|
| 306 | stage = subprocess.run([git, "-C", str(repo), "add", "-A"], env=base_env(), check=False)
|
| 307 | return stage.returncode
|
| 308 |
|
| 309 |
|
| 310 | def check_oracle(repo: Path, oracle_path: Path, *, trust_oracle_commands: bool = False) -> int:
|
| 311 | oracle = json.loads(oracle_path.read_text())
|
| 312 | schema_error = validate_oracle_schema(oracle)
|
| 313 | if schema_error is not None:
|
| 314 | print(f"invalid oracle schema: {schema_error}", file=sys.stderr)
|
| 315 | return 77
|
| 316 | resolution = oracle["resolution"]
|
| 317 | rel = str(resolution["path"])
|
| 318 | expected = str(resolution["content_sha256"])
|
| 319 | git = shutil.which("git")
|
| 320 | if git is None:
|
| 321 | print("git not available for HEAD payload verification", file=sys.stderr)
|
| 322 | return 3
|
| 323 | payload = subprocess.run(
|
| 324 | [git, "-C", str(repo), "show", f"HEAD:{rel}"],
|
| 325 | env=base_env(),
|
| 326 | stdout=subprocess.PIPE,
|
| 327 | stderr=subprocess.PIPE,
|
| 328 | check=False,
|
| 329 | )
|
| 330 | if payload.returncode != 0:
|
| 331 | sys.stderr.write(payload.stderr.decode("utf-8", "replace"))
|
| 332 | return 1
|
| 333 | actual = sha256_bytes(payload.stdout)
|
| 334 | if actual != expected:
|
| 335 | print(f"sha256 mismatch for {rel}: expected {expected}, got {actual}", file=sys.stderr)
|
| 336 | return 1
|
| 337 |
|
| 338 | check = oracle.get("post_merge_check")
|
| 339 | post_resolution_returncode = None
|
| 340 | if check is not None and trust_oracle_commands:
|
| 341 | proc = subprocess.run(check["command"], cwd=repo, env=base_env(), check=False)
|
| 342 | post_resolution_returncode = proc.returncode
|
| 343 | if proc.returncode != 0:
|
| 344 | print(f"post-resolution check returned {proc.returncode}, expected 0", file=sys.stderr)
|
| 345 | return 1
|
| 346 | print(
|
| 347 | json.dumps(
|
| 348 | {
|
| 349 | "resolution_oracle_hash_verified": True,
|
| 350 | "resolution_oracle_payload_path": rel,
|
| 351 | "resolution_oracle_content_sha256": expected,
|
| 352 | "post_resolution_check_returncode": post_resolution_returncode,
|
| 353 | },
|
| 354 | sort_keys=True,
|
| 355 | )
|
| 356 | )
|
| 357 | return 0
|
| 358 |
|
| 359 |
|
| 360 | def _payload_hash_verified(repo: Path, oracle: dict[str, Any], git_bin: Path) -> bool:
|
| 361 | rel = str(oracle["resolution"]["path"])
|
| 362 | expected = str(oracle["resolution"]["content_sha256"])
|
| 363 | proc = subprocess.run(
|
| 364 | [str(git_bin), "-C", str(repo), "show", f"HEAD:{rel}"],
|
| 365 | env=base_env(),
|
| 366 | stdout=subprocess.PIPE,
|
| 367 | stderr=subprocess.DEVNULL,
|
| 368 | check=False,
|
| 369 | )
|
| 370 | return proc.returncode == 0 and sha256_bytes(proc.stdout) == expected
|
| 371 |
|
| 372 |
|
| 373 | def run_git_kind(
|
| 374 | *,
|
| 375 | subject: Subject,
|
| 376 | corpus: Path,
|
| 377 | kind: str,
|
| 378 | run_index: int,
|
| 379 | run_root: Path,
|
| 380 | metadata: dict[str, Any],
|
| 381 | admitted_output_chars: int,
|
| 382 | trust_oracle_commands: bool = False,
|
| 383 | ) -> list[dict[str, Any]]:
|
| 384 | oracle_path, oracle = load_oracle(corpus, kind)
|
| 385 | if oracle_path is None or oracle is None:
|
| 386 | return [
|
| 387 | skip_row(
|
| 388 | metadata=metadata,
|
| 389 | subject=subject,
|
| 390 | kind=kind,
|
| 391 | run_index=run_index,
|
| 392 | reason=f"conflict_oracle_missing:{corpus / f'{kind}{ORACLE_SUFFIX}'}",
|
| 393 | )
|
| 394 | ]
|
| 395 | schema_error = validate_oracle_schema(oracle)
|
| 396 | if schema_error is not None:
|
| 397 | return [
|
| 398 | skip_row(
|
| 399 | metadata=metadata,
|
| 400 | subject=subject,
|
| 401 | kind=kind,
|
| 402 | run_index=run_index,
|
| 403 | reason=f"conflict_oracle_malformed:{schema_error}",
|
| 404 | oracle_path=oracle_path,
|
| 405 | oracle=oracle,
|
| 406 | )
|
| 407 | ]
|
| 408 | repo_src = corpus / kind
|
| 409 | if not repo_src.is_dir():
|
| 410 | return [
|
| 411 | skip_row(
|
| 412 | metadata=metadata,
|
| 413 | subject=subject,
|
| 414 | kind=kind,
|
| 415 | run_index=run_index,
|
| 416 | reason=f"conflict_fixture_repo_missing:{repo_src}",
|
| 417 | oracle_path=oracle_path,
|
| 418 | oracle=oracle,
|
| 419 | )
|
| 420 | ]
|
| 421 | if subject.bin is None or not subject.bin.exists():
|
| 422 | return [
|
| 423 | skip_row(
|
| 424 | metadata=metadata,
|
| 425 | subject=subject,
|
| 426 | kind=kind,
|
| 427 | run_index=run_index,
|
| 428 | reason="git_command_unavailable",
|
| 429 | oracle_path=oracle_path,
|
| 430 | oracle=oracle,
|
| 431 | )
|
| 432 | ]
|
| 433 |
|
| 434 | repo = run_root / WORKFLOW / subject.name / kind / f"run-{run_index}"
|
| 435 | if repo.exists():
|
| 436 | shutil.rmtree(repo)
|
| 437 | repo.parent.mkdir(parents=True, exist_ok=True)
|
| 438 |
|
| 439 | rows: list[dict[str, Any]] = []
|
| 440 | git = str(subject.bin)
|
| 441 | clone = command_row(
|
| 442 | metadata=metadata,
|
| 443 | subject=subject,
|
| 444 | kind=kind,
|
| 445 | run_index=run_index,
|
| 446 | phase="setup",
|
| 447 | operation="setup.clone",
|
| 448 | step_kind="vcs",
|
| 449 | command=[git, "clone", "--quiet", str(repo_src), str(repo)],
|
| 450 | cwd=repo.parent,
|
| 451 | expected_returncodes=(0,),
|
| 452 | oracle_path=oracle_path,
|
| 453 | oracle=oracle,
|
| 454 | admitted_output_chars=admitted_output_chars,
|
| 455 | )
|
| 456 | rows.append(clone)
|
| 457 | if row_returncode(clone) != 0:
|
| 458 | rows.append(sum_rows(rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle))
|
| 459 | return rows
|
| 460 |
|
| 461 | setup_commands = [
|
| 462 | ("setup.checkout_ours", [git, "checkout", "-q", "ours"]),
|
| 463 | ("setup.theirs_branch", [git, "branch", "-f", "theirs", "origin/theirs"]),
|
| 464 | ]
|
| 465 | for operation, command in setup_commands:
|
| 466 | row = command_row(
|
| 467 | metadata=metadata,
|
| 468 | subject=subject,
|
| 469 | kind=kind,
|
| 470 | run_index=run_index,
|
| 471 | phase="setup",
|
| 472 | operation=operation,
|
| 473 | step_kind="vcs",
|
| 474 | command=command,
|
| 475 | cwd=repo,
|
| 476 | expected_returncodes=(0,),
|
| 477 | oracle_path=oracle_path,
|
| 478 | oracle=oracle,
|
| 479 | admitted_output_chars=admitted_output_chars,
|
| 480 | )
|
| 481 | rows.append(row)
|
| 482 | if row_returncode(row) != 0:
|
| 483 | rows.append(sum_rows(rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle))
|
| 484 | return rows
|
| 485 |
|
| 486 | setup_rows = list(rows)
|
| 487 | workflow_rows: list[dict[str, Any]] = []
|
| 488 | expected_merge_codes = (1,) if bool(oracle["expected_conflict"]) else (0,)
|
| 489 | merge = command_row(
|
| 490 | metadata=metadata,
|
| 491 | subject=subject,
|
| 492 | kind=kind,
|
| 493 | run_index=run_index,
|
| 494 | phase="workflow",
|
| 495 | operation="conflict.merge",
|
| 496 | step_kind="vcs",
|
| 497 | command=[git, "merge", "--no-edit", "theirs"],
|
| 498 | cwd=repo,
|
| 499 | expected_returncodes=expected_merge_codes,
|
| 500 | oracle_path=oracle_path,
|
| 501 | oracle=oracle,
|
| 502 | admitted_output_chars=admitted_output_chars,
|
| 503 | )
|
| 504 | observed = observed_unmerged(repo, subject.bin)
|
| 505 | merge["conflict_files_observed"] = observed
|
| 506 | merge["conflict_file_count_observed"] = len(observed)
|
| 507 | merge["merge_conflict_observed"] = merge["process_returncode"] != 0
|
| 508 | if observed != sorted(oracle.get("conflicted_paths", [])):
|
| 509 | merge["returncode"] = 1
|
| 510 | merge["stderr"] = "observed conflict files did not match oracle"
|
| 511 | workflow_rows.append(merge)
|
| 512 | if row_returncode(merge) != 0:
|
| 513 | return [
|
| 514 | *setup_rows,
|
| 515 | sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
|
| 516 | *workflow_rows,
|
| 517 | sum_rows(workflow_rows, metadata, subject, kind, run_index, "workflow", "workflow.total", oracle_path, oracle),
|
| 518 | ]
|
| 519 |
|
| 520 | present = command_row(
|
| 521 | metadata=metadata,
|
| 522 | subject=subject,
|
| 523 | kind=kind,
|
| 524 | run_index=run_index,
|
| 525 | phase="workflow",
|
| 526 | operation="conflict.present",
|
| 527 | step_kind="vcs",
|
| 528 | command=[git, "diff", "--name-only", "--diff-filter=U"],
|
| 529 | cwd=repo,
|
| 530 | expected_returncodes=(0,),
|
| 531 | oracle_path=oracle_path,
|
| 532 | oracle=oracle,
|
| 533 | admitted_output_chars=admitted_output_chars,
|
| 534 | )
|
| 535 | present_files = observed_unmerged(repo, subject.bin)
|
| 536 | present["conflict_files_observed"] = present_files
|
| 537 | present["conflict_file_count_observed"] = len(present_files)
|
| 538 | workflow_rows.append(present)
|
| 539 |
|
| 540 | check = oracle.get("post_merge_check")
|
| 541 | if isinstance(check, dict) and trust_oracle_commands:
|
| 542 | expected_clean = int(check["expected_returncode_clean_merge"])
|
| 543 | check_row = command_row(
|
| 544 | metadata=metadata,
|
| 545 | subject=subject,
|
| 546 | kind=kind,
|
| 547 | run_index=run_index,
|
| 548 | phase="workflow",
|
| 549 | operation="conflict.clean_merge_check",
|
| 550 | step_kind="test",
|
| 551 | command=list(check["command"]),
|
| 552 | cwd=repo,
|
| 553 | expected_returncodes=(expected_clean,),
|
| 554 | oracle_path=oracle_path,
|
| 555 | oracle=oracle,
|
| 556 | admitted_output_chars=admitted_output_chars,
|
| 557 | )
|
| 558 | check_row["post_merge_check_observed_returncode_clean_merge"] = check_row["process_returncode"]
|
| 559 | workflow_rows.append(check_row)
|
| 560 | if row_returncode(check_row) != 0:
|
| 561 | return [
|
| 562 | *setup_rows,
|
| 563 | sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
|
| 564 | *workflow_rows,
|
| 565 | sum_rows(
|
| 566 | workflow_rows,
|
| 567 | metadata,
|
| 568 | subject,
|
| 569 | kind,
|
| 570 | run_index,
|
| 571 | "workflow",
|
| 572 | "workflow.total",
|
| 573 | oracle_path,
|
| 574 | oracle,
|
| 575 | ),
|
| 576 | ]
|
| 577 |
|
| 578 | apply_row = command_row(
|
| 579 | metadata=metadata,
|
| 580 | subject=subject,
|
| 581 | kind=kind,
|
| 582 | run_index=run_index,
|
| 583 | phase="workflow",
|
| 584 | operation="conflict.resolve.oracle",
|
| 585 | step_kind="edit",
|
| 586 | command=[
|
| 587 | sys.executable,
|
| 588 | str(Path(__file__).resolve()),
|
| 589 | "_apply-oracle",
|
| 590 | "--repo",
|
| 591 | str(repo),
|
| 592 | "--oracle",
|
| 593 | str(oracle_path),
|
| 594 | *(["--trust-oracle-commands"] if trust_oracle_commands else []),
|
| 595 | ],
|
| 596 | cwd=repo,
|
| 597 | expected_returncodes=(0,),
|
| 598 | oracle_path=oracle_path,
|
| 599 | oracle=oracle,
|
| 600 | admitted_output_chars=admitted_output_chars,
|
| 601 | )
|
| 602 | workflow_rows.append(apply_row)
|
| 603 | if row_returncode(apply_row) != 0:
|
| 604 | return [
|
| 605 | *setup_rows,
|
| 606 | sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
|
| 607 | *workflow_rows,
|
| 608 | sum_rows(workflow_rows, metadata, subject, kind, run_index, "workflow", "workflow.total", oracle_path, oracle),
|
| 609 | ]
|
| 610 |
|
| 611 | commit = command_row(
|
| 612 | metadata=metadata,
|
| 613 | subject=subject,
|
| 614 | kind=kind,
|
| 615 | run_index=run_index,
|
| 616 | phase="workflow",
|
| 617 | operation="conflict.resolve.commit",
|
| 618 | step_kind="vcs",
|
| 619 | command=[git, "commit", "-q", "-m", f"resolve {kind} from oracle"],
|
| 620 | cwd=repo,
|
| 621 | expected_returncodes=(0,),
|
| 622 | oracle_path=oracle_path,
|
| 623 | oracle=oracle,
|
| 624 | admitted_output_chars=admitted_output_chars,
|
| 625 | )
|
| 626 | workflow_rows.append(commit)
|
| 627 | if row_returncode(commit) != 0:
|
| 628 | return [
|
| 629 | *setup_rows,
|
| 630 | sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
|
| 631 | *workflow_rows,
|
| 632 | sum_rows(workflow_rows, metadata, subject, kind, run_index, "workflow", "workflow.total", oracle_path, oracle),
|
| 633 | ]
|
| 634 |
|
| 635 | verify = command_row(
|
| 636 | metadata=metadata,
|
| 637 | subject=subject,
|
| 638 | kind=kind,
|
| 639 | run_index=run_index,
|
| 640 | phase="workflow",
|
| 641 | operation="conflict.oracle.check",
|
| 642 | step_kind="test",
|
| 643 | command=[
|
| 644 | sys.executable,
|
| 645 | str(Path(__file__).resolve()),
|
| 646 | "_check-oracle",
|
| 647 | "--repo",
|
| 648 | str(repo),
|
| 649 | "--oracle",
|
| 650 | str(oracle_path),
|
| 651 | *(["--trust-oracle-commands"] if trust_oracle_commands else []),
|
| 652 | ],
|
| 653 | cwd=repo,
|
| 654 | expected_returncodes=(0,),
|
| 655 | oracle_path=oracle_path,
|
| 656 | oracle=oracle,
|
| 657 | admitted_output_chars=admitted_output_chars,
|
| 658 | )
|
| 659 | verify["resolution_oracle_hash_verified"] = _payload_hash_verified(repo, oracle, subject.bin)
|
| 660 | workflow_rows.append(verify)
|
| 661 |
|
| 662 | return [
|
| 663 | *setup_rows,
|
| 664 | sum_rows(setup_rows, metadata, subject, kind, run_index, "setup", "setup.total", oracle_path, oracle),
|
| 665 | *workflow_rows,
|
| 666 | sum_rows(workflow_rows, metadata, subject, kind, run_index, "workflow", "workflow.total", oracle_path, oracle),
|
| 667 | ]
|
| 668 |
|
| 669 |
|
| 670 | def sum_rows(
|
| 671 | rows: list[dict[str, Any]],
|
| 672 | metadata: dict[str, Any],
|
| 673 | subject: Subject,
|
| 674 | kind: str,
|
| 675 | run_index: int,
|
| 676 | phase: str,
|
| 677 | operation: str,
|
| 678 | oracle_path: Path | None,
|
| 679 | oracle: dict[str, Any] | None,
|
| 680 | ) -> dict[str, Any]:
|
| 681 | elapsed_ms = sum(float(row.get("elapsed_ms", 0.0)) for row in rows)
|
| 682 | failures = [row for row in rows if row_returncode(row) != 0]
|
| 683 | total = {
|
| 684 | **row_base(metadata, subject, kind, run_index, phase, operation, oracle_path, oracle),
|
| 685 | "step_kind": "summary",
|
| 686 | "elapsed_ms": round(elapsed_ms, 3),
|
| 687 | "returncode": 1 if failures else 0,
|
| 688 | "process_returncode": 1 if failures else 0,
|
| 689 | "command": [row["command"] for row in rows],
|
| 690 | "tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("tool_call_count")) for row in rows),
|
| 691 | "terminal_tool_call_count": sum(
|
| 692 | oakbench_tokens.int_or_zero(row.get("terminal_tool_call_count")) for row in rows
|
| 693 | ),
|
| 694 | "vcs_tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("vcs_tool_call_count")) for row in rows),
|
| 695 | "test_tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("test_tool_call_count")) for row in rows),
|
| 696 | "steps_total": len(rows),
|
| 697 | "steps_succeeded": len(rows) - len(failures),
|
| 698 | "steps_failed": len(failures),
|
| 699 | "step_failure_rate": (len(failures) / len(rows)) if rows else None,
|
| 700 | "summarized_operations": [row["operation"] for row in rows],
|
| 701 | **oakbench_tokens.summed_token_fields(
|
| 702 | rows, "sum_of_conflict_workflow_steps_command_plus_admitted_output_chars_div_4"
|
| 703 | ),
|
| 704 | }
|
| 705 | observed_sets = [row.get("conflict_files_observed") for row in rows if "conflict_files_observed" in row]
|
| 706 | if observed_sets:
|
| 707 | observed = sorted(set().union(*(set(items) for items in observed_sets if isinstance(items, list))))
|
| 708 | total["conflict_files_observed"] = observed
|
| 709 | total["conflict_file_count_observed"] = len(observed)
|
| 710 | hash_checks = [row.get("resolution_oracle_hash_verified") for row in rows if row.get("operation") == "conflict.oracle.check"]
|
| 711 | if hash_checks:
|
| 712 | total["resolution_oracle_hash_verified"] = bool(hash_checks[-1])
|
| 713 | clean_checks = [
|
| 714 | row.get("post_merge_check_observed_returncode_clean_merge")
|
| 715 | for row in rows
|
| 716 | if row.get("post_merge_check_observed_returncode_clean_merge") is not None
|
| 717 | ]
|
| 718 | if clean_checks:
|
| 719 | total["post_merge_check_observed_returncode_clean_merge"] = clean_checks[-1]
|
| 720 | return total
|
| 721 |
|
| 722 |
|
| 723 | def run_subject_kind(
|
| 724 | *,
|
| 725 | subject: Subject,
|
| 726 | corpus: Path,
|
| 727 | kind: str,
|
| 728 | run_index: int,
|
| 729 | run_root: Path,
|
| 730 | metadata: dict[str, Any],
|
| 731 | admitted_output_chars: int,
|
| 732 | trust_oracle_commands: bool = False,
|
| 733 | ) -> list[dict[str, Any]]:
|
| 734 | oracle_path, oracle = load_oracle(corpus, kind)
|
| 735 | if subject.kind != "git":
|
| 736 | return [
|
| 737 | skip_row(
|
| 738 | metadata=metadata,
|
| 739 | subject=subject,
|
| 740 | kind=kind,
|
| 741 | run_index=run_index,
|
| 742 | reason="conflict_resolution_lane_supports_git_local_fixtures_only",
|
| 743 | oracle_path=oracle_path,
|
| 744 | oracle=oracle,
|
| 745 | )
|
| 746 | ]
|
| 747 | return run_git_kind(
|
| 748 | subject=subject,
|
| 749 | corpus=corpus,
|
| 750 | kind=kind,
|
| 751 | run_index=run_index,
|
| 752 | run_root=run_root,
|
| 753 | metadata=metadata,
|
| 754 | admitted_output_chars=admitted_output_chars,
|
| 755 | trust_oracle_commands=trust_oracle_commands,
|
| 756 | )
|
| 757 |
|
| 758 |
|
| 759 | def parse_subjects(raw: str, git_bin: Path | None) -> list[Subject]:
|
| 760 | subjects: list[Subject] = []
|
| 761 | for name in [item.strip() for item in raw.split(",") if item.strip()]:
|
| 762 | if name == "git":
|
| 763 | resolved = git_bin or (Path(shutil.which("git")) if shutil.which("git") else None)
|
| 764 | subjects.append(Subject("git", "git", "Git", resolved))
|
| 765 | elif name.startswith("oak"):
|
| 766 | oak = Path(shutil.which("oak")) if shutil.which("oak") else None
|
| 767 | subjects.append(Subject(name, "oak", "Oak", oak))
|
| 768 | else:
|
| 769 | subjects.append(Subject(name, name, name, None))
|
| 770 | return subjects
|
| 771 |
|
| 772 |
|
| 773 | def ensure_corpus(args: argparse.Namespace, kinds: list[str]) -> str | None:
|
| 774 | missing = [
|
| 775 | kind
|
| 776 | for kind in kinds
|
| 777 | if not (args.corpus / kind).is_dir() or load_oracle(args.corpus, kind) == (None, None)
|
| 778 | ]
|
| 779 | if not missing:
|
| 780 | return None
|
| 781 | if not args.generate_corpus:
|
| 782 | return f"conflict_corpus_missing:{args.corpus}"
|
| 783 | git = shutil.which("git")
|
| 784 | if git is None:
|
| 785 | return "git_command_unavailable_for_conflict_corpus_generation"
|
| 786 | args.corpus.parent.mkdir(parents=True, exist_ok=True)
|
| 787 | command = [
|
| 788 | sys.executable,
|
| 789 | str(GENERATOR),
|
| 790 | "--out",
|
| 791 | str(args.corpus),
|
| 792 | "--seed",
|
| 793 | args.seed,
|
| 794 | "--kinds",
|
| 795 | ",".join(missing),
|
| 796 | ]
|
| 797 | if args.quick_corpus:
|
| 798 | command.append("--quick")
|
| 799 | proc = subprocess.run(command, cwd=ROOT, env=base_env(), check=False)
|
| 800 | if proc.returncode == 3:
|
| 801 | return "git_command_unavailable_for_conflict_corpus_generation"
|
| 802 | if proc.returncode != 0:
|
| 803 | return f"conflict_corpus_generation_failed:{proc.returncode}"
|
| 804 | return None
|
| 805 |
|
| 806 |
|
| 807 | def metadata(bench_id: str, track: str) -> dict[str, Any]:
|
| 808 | return {
|
| 809 | "bench_id": bench_id,
|
| 810 | "profile": PROFILE,
|
| 811 | "benchmark_track": track,
|
| 812 | "timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
| 813 | "host": platform.node(),
|
| 814 | "platform": platform.platform(),
|
| 815 | }
|
| 816 |
|
| 817 |
|
| 818 | def summary_text(rows: list[dict[str, Any]]) -> str:
|
| 819 | totals = [row for row in rows if row.get("operation") == "workflow.total"]
|
| 820 | skips = [row for row in rows if row.get("returncode") == SKIP_RETURNCODE]
|
| 821 | failures = [row for row in totals if row_returncode(row) != 0]
|
| 822 | lines = [
|
| 823 | "# Conflict Resolution Lane",
|
| 824 | "",
|
| 825 | f"Rows: {len(rows)}",
|
| 826 | f"Workflow totals: {len(totals)}",
|
| 827 | f"Skipped rows: {len(skips)}",
|
| 828 | f"Failed workflow totals: {len(failures)}",
|
| 829 | "",
|
| 830 | "| kind | subject | result | observed files | oracle hash |",
|
| 831 | "| --- | --- | --- | --- | --- |",
|
| 832 | ]
|
| 833 | for row in totals:
|
| 834 | result = "ok" if row_returncode(row) == 0 else "failed"
|
| 835 | lines.append(
|
| 836 | f"| `{row['conflict_kind']}` | `{row['subject']}` | {result} | "
|
| 837 | f"{row.get('conflict_file_count_observed')} | {row.get('resolution_oracle_hash_verified')} |"
|
| 838 | )
|
| 839 | for row in skips:
|
| 840 | lines.append(f"| `{row['conflict_kind']}` | `{row['subject']}` | skipped: {row['skip_reason']} | | |")
|
| 841 | return "\n".join(lines) + "\n"
|
| 842 |
|
| 843 |
|
| 844 | def has_lane_failures(rows: list[dict[str, Any]]) -> bool:
|
| 845 | return any(row_returncode(row) not in (0, SKIP_RETURNCODE) for row in rows)
|
| 846 |
|
| 847 |
|
| 848 | def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
| 849 | parser = argparse.ArgumentParser(description=__doc__)
|
| 850 | parser.add_argument("--corpus", type=Path, default=DEFAULT_WORKDIR / "bench-conflict-corpus-v1")
|
| 851 | parser.add_argument("--generate-corpus", action="store_true")
|
| 852 | parser.add_argument("--quick-corpus", action="store_true")
|
| 853 | parser.add_argument("--seed", default="oakbench-conflicts-v1")
|
| 854 | parser.add_argument("--kinds", default=",".join(DEFAULT_KINDS))
|
| 855 | parser.add_argument("--subjects", default="git")
|
| 856 | parser.add_argument("--runs", type=int, default=1)
|
| 857 | parser.add_argument("--workdir", type=Path, default=DEFAULT_WORKDIR)
|
| 858 | parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
|
| 859 | parser.add_argument("--git-bin", type=Path)
|
| 860 | parser.add_argument("--keep-workdirs", action="store_true")
|
| 861 | parser.add_argument("--admitted-output-chars", type=int, default=20_000)
|
| 862 | parser.add_argument("--track", default="agent-default")
|
| 863 | parser.add_argument(
|
| 864 | "--trust-oracle-commands",
|
| 865 | action="store_true",
|
| 866 | help="execute post_merge_check.command from corpus oracle JSON; use only with trusted corpora",
|
| 867 | )
|
| 868 | parser.add_argument("_internal", nargs="*")
|
| 869 | return parser.parse_args(argv)
|
| 870 |
|
| 871 |
|
| 872 | def main(argv: list[str] | None = None) -> int:
|
| 873 | if argv and argv[0] == "_apply-oracle":
|
| 874 | parser = argparse.ArgumentParser()
|
| 875 | parser.add_argument("_cmd")
|
| 876 | parser.add_argument("--repo", type=Path, required=True)
|
| 877 | parser.add_argument("--oracle", type=Path, required=True)
|
| 878 | parser.add_argument("--trust-oracle-commands", action="store_true")
|
| 879 | args = parser.parse_args(argv)
|
| 880 | return apply_resolution(
|
| 881 | args.repo,
|
| 882 | args.oracle,
|
| 883 | trust_oracle_commands=args.trust_oracle_commands,
|
| 884 | )
|
| 885 | if argv and argv[0] == "_check-oracle":
|
| 886 | parser = argparse.ArgumentParser()
|
| 887 | parser.add_argument("_cmd")
|
| 888 | parser.add_argument("--repo", type=Path, required=True)
|
| 889 | parser.add_argument("--oracle", type=Path, required=True)
|
| 890 | parser.add_argument("--trust-oracle-commands", action="store_true")
|
| 891 | args = parser.parse_args(argv)
|
| 892 | return check_oracle(
|
| 893 | args.repo,
|
| 894 | args.oracle,
|
| 895 | trust_oracle_commands=args.trust_oracle_commands,
|
| 896 | )
|
| 897 |
|
| 898 | args = parse_args(argv)
|
| 899 | kinds = [kind.strip() for kind in args.kinds.split(",") if kind.strip()]
|
| 900 | if not kinds:
|
| 901 | raise SystemExit("--kinds must name at least one conflict kind")
|
| 902 | subjects = parse_subjects(args.subjects, args.git_bin)
|
| 903 | if not subjects:
|
| 904 | raise SystemExit("--subjects must name at least one subject")
|
| 905 |
|
| 906 | bench_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
| 907 | meta = metadata(bench_id, args.track)
|
| 908 | rows: list[dict[str, Any]] = []
|
| 909 |
|
| 910 | corpus_problem = ensure_corpus(args, kinds)
|
| 911 | for run_index in range(args.runs):
|
| 912 | for kind in kinds:
|
| 913 | for subject in subjects:
|
| 914 | if corpus_problem is not None:
|
| 915 | rows.append(
|
| 916 | skip_row(
|
| 917 | metadata=meta,
|
| 918 | subject=subject,
|
| 919 | kind=kind,
|
| 920 | run_index=run_index,
|
| 921 | reason=corpus_problem,
|
| 922 | )
|
| 923 | )
|
| 924 | else:
|
| 925 | rows.extend(
|
| 926 | run_subject_kind(
|
| 927 | subject=subject,
|
| 928 | corpus=args.corpus,
|
| 929 | kind=kind,
|
| 930 | run_index=run_index,
|
| 931 | run_root=args.workdir / "runs",
|
| 932 | metadata=meta,
|
| 933 | admitted_output_chars=args.admitted_output_chars,
|
| 934 | trust_oracle_commands=args.trust_oracle_commands,
|
| 935 | )
|
| 936 | )
|
| 937 |
|
| 938 | store = ResultsStore(args.results, lane="workflow", filename_suffix="conflict-resolution")
|
| 939 | raw_path, summary_path = store.write(bench_id, rows, summary_text(rows))
|
| 940 | print(f"Wrote {len(rows)} rows to {raw_path}")
|
| 941 | if summary_path:
|
| 942 | print(f"Wrote summary to {summary_path}")
|
| 943 | if not args.keep_workdirs:
|
| 944 | shutil.rmtree(args.workdir / "runs", ignore_errors=True)
|
| 945 | return 1 if has_lane_failures(rows) else 0
|
| 946 |
|
| 947 |
|
| 948 | if __name__ == "__main__":
|
| 949 | raise SystemExit(main(sys.argv[1:]))
|