Viewing files at commit 46ad54da0178Commit detailsView latest

suites/merge/mergesuite/runner.py 269 lines · 13.7 KB · python
1
"""Run scenarios: git merge-ort oracle vs Oak's merge engine, classify, apply merge laws."""
2
from __future__ import annotations
3
4
import concurrent.futures
5
import hashlib
6
import itertools
7
import json
8
from pathlib import Path
9
import queue
10
import shutil
11
import subprocess
12
import time
13
import traceback
14
from typing import Callable, Dict, List, Optional
15
16
from . import classify as cl
17
from .gitops import Git, git_conflicts, prepare_git_home
18
from .oak_adapter import (LocalOakServer, PATH_DESCRIPTION, PATH_EXERCISED, oak_env,
19
                          run_pull_sync)
20
21
SCHEMA_VERSION = 1
22
23
24
def file_sha256(path: str) -> Optional[str]:
25
    try:
26
        return hashlib.sha256(Path(path).read_bytes()).hexdigest()
27
    except OSError:
28
        return None
29
30
31
def tree_digest(tree: Optional[dict]) -> Optional[str]:
32
    if tree is None:
33
        return None
34
    return hashlib.sha256(json.dumps(sorted(tree.items())).encode()).hexdigest()
35
36
37
def tool_version(binary: str, *args: str) -> Optional[str]:
38
    try:
39
        return subprocess.run([binary, *args], capture_output=True, text=True, timeout=30).stdout.strip() or None
40
    except OSError:
41
        return None
42
43
44
def git_oracle(git: Git, direction: dict) -> dict:
45
    merge = git.merge_tree(direction["oracle"]["ours"], direction["oracle"]["theirs"])
46
    return {"clean": merge["clean"], "tree": merge["tree"], "conflicts": git_conflicts(merge),
47
            "messages": [m for m in merge["messages"] if m["type"].startswith("CONFLICT")][:20]}
48
49
50
def text_evidence(git: Git, trees: dict, oak_tree: dict, oak_conf: set, case_dir: Path,
51
                  base: dict, ours: dict, theirs: dict) -> Dict[str, dict]:
52
    """Bytes for paths both sides changed that Oak text-merged (needed by the no-lost-change law)."""
53
    out = {}
54
    for path, value in oak_tree.items():
55
        if path in oak_conf:
56
            continue
57
        b, o, t = base.get(path), ours.get(path), theirs.get(path)
58
        if o is None or t is None or o == t or o == b or t == b or value in (b, o, t):
59
            continue
60
61
        def blob(commit):
62
            p = git.run("cat-file", "-p", f"{commit}:{path}", check=False)
63
            return p.stdout if p.returncode == 0 else b""
64
        merged_path = case_dir / "ours" / path
65
        out[path] = {"base": blob(trees["base"]) if b else b"", "ours": blob(trees["ours"]),
66
                     "theirs": blob(trees["theirs"]),
67
                     "merged": merged_path.read_bytes() if merged_path.is_file() else b""}
68
    return out
69
70
71
class Suite:
72
    def __init__(self, workdir: Path, oak_bin: str, git_bin: str, jobs: int = 4,
73
                 keep_equal: bool = False, run_id: Optional[str] = None):
74
        self.workdir, self.oak_bin, self.git_bin = workdir, oak_bin, git_bin
75
        self.jobs, self.keep_equal = max(1, jobs), keep_equal
76
        self.run_id = run_id or time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
77
        self.git_home = workdir / "git-home"
78
        prepare_git_home(self.git_home, git_bin)
79
        self.homes: "queue.Queue[Path]" = queue.Queue()
80
        for i in range(self.jobs):
81
            home = workdir / "oak-homes" / f"h{i}"
82
            home.mkdir(parents=True, exist_ok=True)
83
            prepare_git_home(home, git_bin)
84
            self.homes.put(home)
85
        self.meta = {
86
            "oak": {"bin": oak_bin, "sha256": file_sha256(shutil.which(oak_bin) or oak_bin),
87
                    "version": tool_version(oak_bin, "--version"),
88
                    "path_exercised": PATH_EXERCISED, "path_description": PATH_DESCRIPTION},
89
            "git": {"bin": git_bin, "sha256": file_sha256(git_bin), "version": tool_version(git_bin, "--version"),
90
                    "role": "reference oracle (merge-ort via merge-tree --write-tree) and fixture builder"},
91
        }
92
        self._keys = itertools.count(1)
93
94
    def run(self, scenarios: List[dict], emit: Callable[[dict], None]) -> List[dict]:
95
        server_env = oak_env(self.workdir / "oak-homes" / "server", self.git_bin)
96
        (self.workdir / "oak-homes" / "server").mkdir(parents=True, exist_ok=True)
97
        server = LocalOakServer(self.workdir, self.oak_bin, server_env)
98
        rows: List[dict] = []
99
        try:
100
            with concurrent.futures.ThreadPoolExecutor(self.jobs) as pool:
101
                futures = [pool.submit(self._scenario, s, server) for s in scenarios]
102
                for fut in concurrent.futures.as_completed(futures):
103
                    for row in fut.result():
104
                        rows.append(row)
105
                        emit(row)
106
        finally:
107
            server.close()
108
        return rows
109
110
    def _next_key(self) -> str:
111
        return f"suite/m{next(self._keys):05d}"
112
113
    def _scenario(self, scenario: dict, server: LocalOakServer) -> List[dict]:
114
        home = self.homes.get()
115
        try:
116
            return self._scenario_rows(scenario, server, home)
117
        finally:
118
            self.homes.put(home)
119
120
    def _scenario_rows(self, scenario: dict, server: LocalOakServer, home: Path) -> List[dict]:
121
        git = Git(scenario["repo"], self.git_bin, self.git_home)
122
        safe = scenario["id"].replace("/", "__")
123
        scen_dir = self.workdir / "cases" / safe
124
        rows, results = [], {}
125
        bare = scen_dir / "src.git"
126
        for direction in scenario["directions"]:
127
            start = time.monotonic()
128
            case_dir = scen_dir / direction["name"]
129
            row = self._base_row(scenario, direction)
130
            oak_result, git_result, harness = None, None, None
131
            base, ours, theirs = None, {}, {}
132
            try:
133
                if scenario.get("unsupported") is None:
134
                    git_result = git_oracle(git, direction)
135
                    trees = direction["trees"]
136
                    unique_base = trees.get("base") and len(direction.get("merge_bases") or [1]) == 1
137
                    base = git.tree(trees["base"]) if unique_base else None
138
                    ours, theirs = git.tree(trees["ours"]), git.tree(trees["theirs"])
139
                    if not bare.exists():
140
                        self._make_bare(git, bare, scenario)
141
                    if case_dir.exists():
142
                        shutil.rmtree(case_dir)
143
                    case_dir.mkdir(parents=True)
144
                    env = oak_env(home, self.git_bin)
145
                    oak_result = run_pull_sync(server, self.oak_bin, env, Git(bare, self.git_bin, self.git_home),
146
                                               bare, case_dir, self._next_key(), direction["plan"])
147
            except Exception as exc:  # instrument failure, never an Oak verdict
148
                harness = f"{type(exc).__name__}: {exc}"
149
                row["traceback"] = traceback.format_exc()[-2000:]
150
            texts = None
151
            if oak_result and oak_result.get("tree") is not None and git_result:
152
                texts = text_evidence(git, direction["trees"], oak_result["tree"],
153
                                      set(oak_result.get("conflicts") or {}), case_dir, base or {}, ours, theirs)
154
            verdict = cl.classify(git_result, oak_result, base, ours, theirs,
155
                                  unsupported=scenario.get("unsupported"), harness_error=harness,
156
                                  expect=direction.get("expect"), texts=texts)
157
            row.update(self._result_fields(git_result, oak_result, verdict))
158
            if direction.get("intent_note"):
159
                row["intent_note"] = direction["intent_note"]
160
            row["elapsed_ms"] = round((time.monotonic() - start) * 1000)
161
            row["case_dir"] = str(case_dir) if case_dir.exists() else None
162
            if verdict["class"] == "equal-to-git" and not self.keep_equal and case_dir.exists():
163
                shutil.rmtree(case_dir, ignore_errors=True)
164
                row["case_dir"] = None
165
            results[direction["name"]] = oak_result
166
            rows.append(row)
167
        if len(rows) == 2:
168
            sym = cl.symmetry(*(results[d["name"]] for d in scenario["directions"]))
169
            for row in rows:
170
                row["laws"]["symmetric"] = None if sym is None else sym["holds"]
171
                if sym and not sym["holds"]:
172
                    row.setdefault("evidence", {})["symmetry"] = sym
173
        if not self.keep_equal and all(r["class"] == "equal-to-git" for r in rows):
174
            shutil.rmtree(scen_dir, ignore_errors=True)
175
        return rows
176
177
    def _make_bare(self, git: Git, bare: Path, scenario: dict) -> None:
178
        """Bare mirror holding every plan commit under a ref (builder commits are unreferenced;
179
        `oak clone <x.git>` imports from the mirror's HEAD, which the adapter points per step)."""
180
        wanted = set()
181
        for d in scenario["directions"]:
182
            plan = d["plan"]
183
            wanted.update(x for x in (plan.get("main"), plan.get("fork"), plan.get("final_stage")) if x)
184
            for step in plan["steps"]:
185
                wanted.update(x for x in (step.get("tree"), step.get("stage")) if x)
186
        ns = "refs/mergesuite/" + hashlib.sha256(scenario["id"].encode()).hexdigest()[:16]
187
        for i, commit in enumerate(sorted(wanted)):
188
            git.run("update-ref", f"{ns}/keep{i}", commit)
189
        bare.parent.mkdir(parents=True, exist_ok=True)
190
        Git(bare.parent, self.git_bin, self.git_home).run("init", "-q", "--bare", str(bare))
191
        Git(bare, self.git_bin, self.git_home).run("fetch", "-q", str(scenario["repo"]),
192
                                                   f"+{ns}/*:refs/mergesuite/*")
193
194
    def _base_row(self, scenario: dict, direction: dict) -> dict:
195
        return {
196
            "merge_suite_schema_version": SCHEMA_VERSION,
197
            "evidence_class": "diagnostic_merge_correctness",
198
            "run_id": self.run_id,
199
            "scenario": scenario["id"], "scenario_version": scenario["version"],
200
            "direction": direction["name"], "origin": scenario["origin"], "upstream": scenario["upstream"],
201
            "notes": scenario.get("notes") or None,
202
            "oak": self.meta["oak"], "git": self.meta["git"],
203
            "laws": {"no_lost_change": None, "idempotent": None, "symmetric": None,
204
                     "false_conflict": None, "missed_conflict": None, "prediction_agrees": None},
205
            "repro": ("python3 suites/merge/run_merge_suite.py --workdir <dir> --only "
206
                      f"'{scenario['id']}'"),
207
        }
208
209
    def _result_fields(self, git_result: Optional[dict], oak_result: Optional[dict], verdict: dict) -> dict:
210
        out = {"class": verdict["class"], "reasons": verdict.get("reasons", [])}
211
        if verdict.get("notes"):
212
            out["class_notes"] = verdict["notes"]
213
        evidence = dict(verdict.get("evidence") or {})
214
        laws = {"no_lost_change": None, "idempotent": None, "symmetric": None,
215
                "false_conflict": None, "missed_conflict": None, "prediction_agrees": None}
216
        if git_result:
217
            out["git_result"] = {"clean": git_result["clean"], "conflicts": git_result["conflicts"],
218
                                 "tree_digest": tree_digest(git_result["tree"]),
219
                                 "conflict_messages": git_result["messages"]}
220
        if oak_result:
221
            out["oak_result"] = {
222
                "path_exercised": oak_result.get("path_exercised"),
223
                "clean": oak_result.get("clean"), "conflicts": oak_result.get("conflicts"),
224
                "tree_digest": tree_digest(oak_result.get("tree")),
225
                "pull_returncode": oak_result.get("pull_returncode"),
226
                "pull_receipt": oak_result.get("pull_receipt"),
227
                "prediction": oak_result.get("prediction"),
228
                "idempotence": oak_result.get("idempotence"),
229
                "worktree_matches_head": oak_result.get("worktree_matches_head"),
230
                "result_source": oak_result.get("result_source"),
231
                "conflict_worktree_stale_paths": oak_result.get("conflict_worktree_stale_paths") or None,
232
                "error": oak_result.get("error"), "error_kind": oak_result.get("error_kind"),
233
                "shim_log": oak_result.get("shim_log") or None,
234
            }
235
            if verdict["class"] not in ("equal-to-git",):
236
                out["oak_steps"] = [{k: s[k] for k in ("label", "argv", "returncode", "ms")}
237
                                    | {"stderr_tail": s["stderr"][-300:]} for s in oak_result.get("steps", [])]
238
            if oak_result.get("clean") is not None and git_result:
239
                law = evidence.get("law_no_lost_change")
240
                laws["no_lost_change"] = None if law is None else not (law.get("lost") or law.get("fabricated"))
241
                laws["idempotent"] = cl.idempotence_holds(oak_result)
242
                oak_conf = set(oak_result.get("conflicts") or {})
243
                git_conf = set(git_result["conflicts"])
244
                laws["false_conflict"] = bool(oak_conf - git_conf)
245
                laws["missed_conflict"] = bool(git_conf - oak_conf)
246
                pred = oak_result.get("prediction") or {}
247
                if pred.get("available"):
248
                    laws["prediction_agrees"] = (bool(pred.get("clean")) == bool(oak_result["clean"])
249
                                                 and set(pred.get("conflict_files") or []) == oak_conf)
250
        out["laws"] = laws
251
        if evidence:
252
            out["evidence"] = evidence
253
        return out
254
255
256
def summarize(rows: List[dict]) -> dict:
257
    counts = {c: 0 for c in cl.CLASSES}
258
    by_origin: Dict[str, Dict[str, int]] = {}
259
    for r in rows:
260
        counts[r["class"]] += 1
261
        by_origin.setdefault(r["origin"], {c: 0 for c in cl.CLASSES})[r["class"]] += 1
262
263
    def rate(key):
264
        vals = [r["laws"][key] for r in rows if r["laws"].get(key) is not None]
265
        return {"true": sum(1 for v in vals if v), "measured": len(vals)}
266
    return {"cases": len(rows), "scenarios": len({r["scenario"] for r in rows}), "counts": counts,
267
            "by_origin": by_origin,
268
            "laws": {k: rate(k) for k in ("no_lost_change", "idempotent", "symmetric", "false_conflict",
269
                                          "missed_conflict", "prediction_agrees")}}