| 1 | """Run scenarios: git merge-ort oracle vs Oak's merge engine, classify, apply merge laws."""
|
| 2 | from __future__ import annotations
|
| 3 |
|
| 4 | import concurrent.futures
|
| 5 | import hashlib
|
| 6 | import itertools
|
| 7 | import json
|
| 8 | from pathlib import Path
|
| 9 | import queue
|
| 10 | import shutil
|
| 11 | import subprocess
|
| 12 | import time
|
| 13 | import traceback
|
| 14 | from typing import Callable, Dict, List, Optional
|
| 15 |
|
| 16 | from . import classify as cl
|
| 17 | from .gitops import Git, git_conflicts, prepare_git_home
|
| 18 | from .oak_adapter import (LocalOakServer, PATH_DESCRIPTION, PATH_EXERCISED, oak_env,
|
| 19 | run_pull_sync)
|
| 20 |
|
| 21 | SCHEMA_VERSION = 1
|
| 22 |
|
| 23 |
|
| 24 | def file_sha256(path: str) -> Optional[str]:
|
| 25 | try:
|
| 26 | return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
| 27 | except OSError:
|
| 28 | return None
|
| 29 |
|
| 30 |
|
| 31 | def tree_digest(tree: Optional[dict]) -> Optional[str]:
|
| 32 | if tree is None:
|
| 33 | return None
|
| 34 | return hashlib.sha256(json.dumps(sorted(tree.items())).encode()).hexdigest()
|
| 35 |
|
| 36 |
|
| 37 | def tool_version(binary: str, *args: str) -> Optional[str]:
|
| 38 | try:
|
| 39 | return subprocess.run([binary, *args], capture_output=True, text=True, timeout=30).stdout.strip() or None
|
| 40 | except OSError:
|
| 41 | return None
|
| 42 |
|
| 43 |
|
| 44 | def git_oracle(git: Git, direction: dict) -> dict:
|
| 45 | merge = git.merge_tree(direction["oracle"]["ours"], direction["oracle"]["theirs"])
|
| 46 | return {"clean": merge["clean"], "tree": merge["tree"], "conflicts": git_conflicts(merge),
|
| 47 | "messages": [m for m in merge["messages"] if m["type"].startswith("CONFLICT")][:20]}
|
| 48 |
|
| 49 |
|
| 50 | def text_evidence(git: Git, trees: dict, oak_tree: dict, oak_conf: set, case_dir: Path,
|
| 51 | base: dict, ours: dict, theirs: dict) -> Dict[str, dict]:
|
| 52 | """Bytes for paths both sides changed that Oak text-merged (needed by the no-lost-change law)."""
|
| 53 | out = {}
|
| 54 | for path, value in oak_tree.items():
|
| 55 | if path in oak_conf:
|
| 56 | continue
|
| 57 | b, o, t = base.get(path), ours.get(path), theirs.get(path)
|
| 58 | if o is None or t is None or o == t or o == b or t == b or value in (b, o, t):
|
| 59 | continue
|
| 60 |
|
| 61 | def blob(commit):
|
| 62 | p = git.run("cat-file", "-p", f"{commit}:{path}", check=False)
|
| 63 | return p.stdout if p.returncode == 0 else b""
|
| 64 | merged_path = case_dir / "ours" / path
|
| 65 | out[path] = {"base": blob(trees["base"]) if b else b"", "ours": blob(trees["ours"]),
|
| 66 | "theirs": blob(trees["theirs"]),
|
| 67 | "merged": merged_path.read_bytes() if merged_path.is_file() else b""}
|
| 68 | return out
|
| 69 |
|
| 70 |
|
| 71 | class Suite:
|
| 72 | def __init__(self, workdir: Path, oak_bin: str, git_bin: str, jobs: int = 4,
|
| 73 | keep_equal: bool = False, run_id: Optional[str] = None):
|
| 74 | self.workdir, self.oak_bin, self.git_bin = workdir, oak_bin, git_bin
|
| 75 | self.jobs, self.keep_equal = max(1, jobs), keep_equal
|
| 76 | self.run_id = run_id or time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
|
| 77 | self.git_home = workdir / "git-home"
|
| 78 | prepare_git_home(self.git_home, git_bin)
|
| 79 | self.homes: "queue.Queue[Path]" = queue.Queue()
|
| 80 | for i in range(self.jobs):
|
| 81 | home = workdir / "oak-homes" / f"h{i}"
|
| 82 | home.mkdir(parents=True, exist_ok=True)
|
| 83 | prepare_git_home(home, git_bin)
|
| 84 | self.homes.put(home)
|
| 85 | self.meta = {
|
| 86 | "oak": {"bin": oak_bin, "sha256": file_sha256(shutil.which(oak_bin) or oak_bin),
|
| 87 | "version": tool_version(oak_bin, "--version"),
|
| 88 | "path_exercised": PATH_EXERCISED, "path_description": PATH_DESCRIPTION},
|
| 89 | "git": {"bin": git_bin, "sha256": file_sha256(git_bin), "version": tool_version(git_bin, "--version"),
|
| 90 | "role": "reference oracle (merge-ort via merge-tree --write-tree) and fixture builder"},
|
| 91 | }
|
| 92 | self._keys = itertools.count(1)
|
| 93 |
|
| 94 | def run(self, scenarios: List[dict], emit: Callable[[dict], None]) -> List[dict]:
|
| 95 | server_env = oak_env(self.workdir / "oak-homes" / "server", self.git_bin)
|
| 96 | (self.workdir / "oak-homes" / "server").mkdir(parents=True, exist_ok=True)
|
| 97 | server = LocalOakServer(self.workdir, self.oak_bin, server_env)
|
| 98 | rows: List[dict] = []
|
| 99 | try:
|
| 100 | with concurrent.futures.ThreadPoolExecutor(self.jobs) as pool:
|
| 101 | futures = [pool.submit(self._scenario, s, server) for s in scenarios]
|
| 102 | for fut in concurrent.futures.as_completed(futures):
|
| 103 | for row in fut.result():
|
| 104 | rows.append(row)
|
| 105 | emit(row)
|
| 106 | finally:
|
| 107 | server.close()
|
| 108 | return rows
|
| 109 |
|
| 110 | def _next_key(self) -> str:
|
| 111 | return f"suite/m{next(self._keys):05d}"
|
| 112 |
|
| 113 | def _scenario(self, scenario: dict, server: LocalOakServer) -> List[dict]:
|
| 114 | home = self.homes.get()
|
| 115 | try:
|
| 116 | return self._scenario_rows(scenario, server, home)
|
| 117 | finally:
|
| 118 | self.homes.put(home)
|
| 119 |
|
| 120 | def _scenario_rows(self, scenario: dict, server: LocalOakServer, home: Path) -> List[dict]:
|
| 121 | git = Git(scenario["repo"], self.git_bin, self.git_home)
|
| 122 | safe = scenario["id"].replace("/", "__")
|
| 123 | scen_dir = self.workdir / "cases" / safe
|
| 124 | rows, results = [], {}
|
| 125 | bare = scen_dir / "src.git"
|
| 126 | for direction in scenario["directions"]:
|
| 127 | start = time.monotonic()
|
| 128 | case_dir = scen_dir / direction["name"]
|
| 129 | row = self._base_row(scenario, direction)
|
| 130 | oak_result, git_result, harness = None, None, None
|
| 131 | base, ours, theirs = None, {}, {}
|
| 132 | try:
|
| 133 | if scenario.get("unsupported") is None:
|
| 134 | git_result = git_oracle(git, direction)
|
| 135 | trees = direction["trees"]
|
| 136 | unique_base = trees.get("base") and len(direction.get("merge_bases") or [1]) == 1
|
| 137 | base = git.tree(trees["base"]) if unique_base else None
|
| 138 | ours, theirs = git.tree(trees["ours"]), git.tree(trees["theirs"])
|
| 139 | if not bare.exists():
|
| 140 | self._make_bare(git, bare, scenario)
|
| 141 | if case_dir.exists():
|
| 142 | shutil.rmtree(case_dir)
|
| 143 | case_dir.mkdir(parents=True)
|
| 144 | env = oak_env(home, self.git_bin)
|
| 145 | oak_result = run_pull_sync(server, self.oak_bin, env, Git(bare, self.git_bin, self.git_home),
|
| 146 | bare, case_dir, self._next_key(), direction["plan"])
|
| 147 | except Exception as exc: # instrument failure, never an Oak verdict
|
| 148 | harness = f"{type(exc).__name__}: {exc}"
|
| 149 | row["traceback"] = traceback.format_exc()[-2000:]
|
| 150 | texts = None
|
| 151 | if oak_result and oak_result.get("tree") is not None and git_result:
|
| 152 | texts = text_evidence(git, direction["trees"], oak_result["tree"],
|
| 153 | set(oak_result.get("conflicts") or {}), case_dir, base or {}, ours, theirs)
|
| 154 | verdict = cl.classify(git_result, oak_result, base, ours, theirs,
|
| 155 | unsupported=scenario.get("unsupported"), harness_error=harness,
|
| 156 | expect=direction.get("expect"), texts=texts)
|
| 157 | row.update(self._result_fields(git_result, oak_result, verdict))
|
| 158 | if direction.get("intent_note"):
|
| 159 | row["intent_note"] = direction["intent_note"]
|
| 160 | row["elapsed_ms"] = round((time.monotonic() - start) * 1000)
|
| 161 | row["case_dir"] = str(case_dir) if case_dir.exists() else None
|
| 162 | if verdict["class"] == "equal-to-git" and not self.keep_equal and case_dir.exists():
|
| 163 | shutil.rmtree(case_dir, ignore_errors=True)
|
| 164 | row["case_dir"] = None
|
| 165 | results[direction["name"]] = oak_result
|
| 166 | rows.append(row)
|
| 167 | if len(rows) == 2:
|
| 168 | sym = cl.symmetry(*(results[d["name"]] for d in scenario["directions"]))
|
| 169 | for row in rows:
|
| 170 | row["laws"]["symmetric"] = None if sym is None else sym["holds"]
|
| 171 | if sym and not sym["holds"]:
|
| 172 | row.setdefault("evidence", {})["symmetry"] = sym
|
| 173 | if not self.keep_equal and all(r["class"] == "equal-to-git" for r in rows):
|
| 174 | shutil.rmtree(scen_dir, ignore_errors=True)
|
| 175 | return rows
|
| 176 |
|
| 177 | def _make_bare(self, git: Git, bare: Path, scenario: dict) -> None:
|
| 178 | """Bare mirror holding every plan commit under a ref (builder commits are unreferenced;
|
| 179 | `oak clone <x.git>` imports from the mirror's HEAD, which the adapter points per step)."""
|
| 180 | wanted = set()
|
| 181 | for d in scenario["directions"]:
|
| 182 | plan = d["plan"]
|
| 183 | wanted.update(x for x in (plan.get("main"), plan.get("fork"), plan.get("final_stage")) if x)
|
| 184 | for step in plan["steps"]:
|
| 185 | wanted.update(x for x in (step.get("tree"), step.get("stage")) if x)
|
| 186 | ns = "refs/mergesuite/" + hashlib.sha256(scenario["id"].encode()).hexdigest()[:16]
|
| 187 | for i, commit in enumerate(sorted(wanted)):
|
| 188 | git.run("update-ref", f"{ns}/keep{i}", commit)
|
| 189 | bare.parent.mkdir(parents=True, exist_ok=True)
|
| 190 | Git(bare.parent, self.git_bin, self.git_home).run("init", "-q", "--bare", str(bare))
|
| 191 | Git(bare, self.git_bin, self.git_home).run("fetch", "-q", str(scenario["repo"]),
|
| 192 | f"+{ns}/*:refs/mergesuite/*")
|
| 193 |
|
| 194 | def _base_row(self, scenario: dict, direction: dict) -> dict:
|
| 195 | return {
|
| 196 | "merge_suite_schema_version": SCHEMA_VERSION,
|
| 197 | "evidence_class": "diagnostic_merge_correctness",
|
| 198 | "run_id": self.run_id,
|
| 199 | "scenario": scenario["id"], "scenario_version": scenario["version"],
|
| 200 | "direction": direction["name"], "origin": scenario["origin"], "upstream": scenario["upstream"],
|
| 201 | "notes": scenario.get("notes") or None,
|
| 202 | "oak": self.meta["oak"], "git": self.meta["git"],
|
| 203 | "laws": {"no_lost_change": None, "idempotent": None, "symmetric": None,
|
| 204 | "false_conflict": None, "missed_conflict": None, "prediction_agrees": None},
|
| 205 | "repro": ("python3 suites/merge/run_merge_suite.py --workdir <dir> --only "
|
| 206 | f"'{scenario['id']}'"),
|
| 207 | }
|
| 208 |
|
| 209 | def _result_fields(self, git_result: Optional[dict], oak_result: Optional[dict], verdict: dict) -> dict:
|
| 210 | out = {"class": verdict["class"], "reasons": verdict.get("reasons", [])}
|
| 211 | if verdict.get("notes"):
|
| 212 | out["class_notes"] = verdict["notes"]
|
| 213 | evidence = dict(verdict.get("evidence") or {})
|
| 214 | laws = {"no_lost_change": None, "idempotent": None, "symmetric": None,
|
| 215 | "false_conflict": None, "missed_conflict": None, "prediction_agrees": None}
|
| 216 | if git_result:
|
| 217 | out["git_result"] = {"clean": git_result["clean"], "conflicts": git_result["conflicts"],
|
| 218 | "tree_digest": tree_digest(git_result["tree"]),
|
| 219 | "conflict_messages": git_result["messages"]}
|
| 220 | if oak_result:
|
| 221 | out["oak_result"] = {
|
| 222 | "path_exercised": oak_result.get("path_exercised"),
|
| 223 | "clean": oak_result.get("clean"), "conflicts": oak_result.get("conflicts"),
|
| 224 | "tree_digest": tree_digest(oak_result.get("tree")),
|
| 225 | "pull_returncode": oak_result.get("pull_returncode"),
|
| 226 | "pull_receipt": oak_result.get("pull_receipt"),
|
| 227 | "prediction": oak_result.get("prediction"),
|
| 228 | "idempotence": oak_result.get("idempotence"),
|
| 229 | "worktree_matches_head": oak_result.get("worktree_matches_head"),
|
| 230 | "result_source": oak_result.get("result_source"),
|
| 231 | "conflict_worktree_stale_paths": oak_result.get("conflict_worktree_stale_paths") or None,
|
| 232 | "error": oak_result.get("error"), "error_kind": oak_result.get("error_kind"),
|
| 233 | "shim_log": oak_result.get("shim_log") or None,
|
| 234 | }
|
| 235 | if verdict["class"] not in ("equal-to-git",):
|
| 236 | out["oak_steps"] = [{k: s[k] for k in ("label", "argv", "returncode", "ms")}
|
| 237 | | {"stderr_tail": s["stderr"][-300:]} for s in oak_result.get("steps", [])]
|
| 238 | if oak_result.get("clean") is not None and git_result:
|
| 239 | law = evidence.get("law_no_lost_change")
|
| 240 | laws["no_lost_change"] = None if law is None else not (law.get("lost") or law.get("fabricated"))
|
| 241 | laws["idempotent"] = cl.idempotence_holds(oak_result)
|
| 242 | oak_conf = set(oak_result.get("conflicts") or {})
|
| 243 | git_conf = set(git_result["conflicts"])
|
| 244 | laws["false_conflict"] = bool(oak_conf - git_conf)
|
| 245 | laws["missed_conflict"] = bool(git_conf - oak_conf)
|
| 246 | pred = oak_result.get("prediction") or {}
|
| 247 | if pred.get("available"):
|
| 248 | laws["prediction_agrees"] = (bool(pred.get("clean")) == bool(oak_result["clean"])
|
| 249 | and set(pred.get("conflict_files") or []) == oak_conf)
|
| 250 | out["laws"] = laws
|
| 251 | if evidence:
|
| 252 | out["evidence"] = evidence
|
| 253 | return out
|
| 254 |
|
| 255 |
|
| 256 | def summarize(rows: List[dict]) -> dict:
|
| 257 | counts = {c: 0 for c in cl.CLASSES}
|
| 258 | by_origin: Dict[str, Dict[str, int]] = {}
|
| 259 | for r in rows:
|
| 260 | counts[r["class"]] += 1
|
| 261 | by_origin.setdefault(r["origin"], {c: 0 for c in cl.CLASSES})[r["class"]] += 1
|
| 262 |
|
| 263 | def rate(key):
|
| 264 | vals = [r["laws"][key] for r in rows if r["laws"].get(key) is not None]
|
| 265 | return {"true": sum(1 for v in vals if v), "measured": len(vals)}
|
| 266 | return {"cases": len(rows), "scenarios": len({r["scenario"] for r in rows}), "counts": counts,
|
| 267 | "by_origin": by_origin,
|
| 268 | "laws": {k: rate(k) for k in ("no_lost_change", "idempotent", "symmetric", "false_conflict",
|
| 269 | "missed_conflict", "prediction_agrees")}}
|