Viewing files at commit 2f5107c20e85Commit detailsView latest

Browse files
tests/test_merge_suite.py 365 lines · 19.6 KB · python
1
"""Instrument tests for suites/merge (classification, laws, oracle parsing, vendoring).
2
3
Hermetic: no Oak binary and no network. Git-backed tests skip when git is absent.
4
"""
5
import hashlib
6
import json
7
from pathlib import Path
8
import shutil
9
import sys
10
import tempfile
11
import unittest
12
13
ROOT = Path(__file__).resolve().parents[1]
14
sys.path.insert(0, str(ROOT / "suites" / "merge"))
15
sys.path.insert(0, str(ROOT / "scripts"))
16
17
from mergesuite import classify as cl  # noqa: E402
18
from mergesuite.gitops import (Git, git_conflicts, normalize_git_conflict_type,  # noqa: E402
19
                               parse_merge_tree_z, prepare_git_home, worktree_manifest)
20
from mergesuite.oak_adapter import parse_conflict_show, summarize_prediction  # noqa: E402
21
22
VENDOR = ROOT / "suites" / "merge" / "vendor" / "gitoxide"
23
24
25
def v(tag):
26
    return ("file", hashlib.sha256(tag.encode()).hexdigest())
27
28
29
def oak(tree, conflicts=(), clean=None):
30
    return {"tree": tree, "conflicts": {p: [] for p in conflicts},
31
            "clean": (not conflicts) if clean is None else clean, "error": None, "error_kind": None}
32
33
34
def gitres(tree, conflicts=()):
35
    return {"tree": tree, "conflicts": {p: ["content"] for p in conflicts}, "clean": not conflicts}
36
37
38
class ClassifyTests(unittest.TestCase):
39
    base = {"a": v("a0"), "b": v("b0")}
40
    ours = {"a": v("a1"), "b": v("b0")}
41
    theirs = {"a": v("a0"), "b": v("b1")}
42
    merged = {"a": v("a1"), "b": v("b1")}
43
44
    def test_equal(self):
45
        r = cl.classify(gitres(self.merged), oak(self.merged), self.base, self.ours, self.theirs)
46
        self.assertEqual(r["class"], "equal-to-git")
47
48
    def test_lost_change_is_oak_wrong_even_if_git_agrees(self):
49
        # Oak drops theirs' change to b.
50
        r = cl.classify(gitres(self.merged), oak(self.ours), self.base, self.ours, self.theirs)
51
        self.assertEqual(r["class"], "oak-wrong")
52
        self.assertIn("lost_change: b", r["reasons"][0])
53
54
    def test_lost_deletion(self):
55
        theirs = {"a": v("a0")}  # theirs deletes b
56
        r = cl.classify(gitres({"a": v("a1")}), oak({"a": v("a1"), "b": v("b0")}),
57
                        self.base, self.ours, theirs)
58
        self.assertEqual(r["class"], "oak-wrong")
59
60
    def test_fabricated_content(self):
61
        r = cl.classify(gitres(self.merged), oak({"a": v("a1"), "b": v("zzz")}),
62
                        self.base, self.ours, self.theirs)
63
        self.assertEqual(r["class"], "oak-wrong")
64
        self.assertIn("fabricated", r["reasons"][0])
65
66
    def test_false_conflict_is_differs_but_valid(self):
67
        r = cl.classify(gitres(self.merged), oak(self.merged, conflicts=["a"]),
68
                        self.base, self.ours, self.theirs)
69
        self.assertEqual(r["class"], "differs-but-valid")
70
        self.assertTrue(any("oak_conflicts_where_git_clean" in x for x in r["reasons"]))
71
72
    def test_rename_not_detected_keeps_both_sides(self):
73
        # ours renames a->c; theirs edits b. git moves; Oak path-level result matches git here,
74
        # but when git conflicts and Oak keeps everything, it is differs-but-valid.
75
        base = {"a": v("a0")}
76
        ours = {"b": v("a0")}
77
        theirs = {"c": v("a0")}
78
        r = cl.classify(gitres({"b": v("a0"), "c": v("a0")}, conflicts=["a", "b", "c"]),
79
                        oak({"b": v("a0"), "c": v("a0")}), base, ours, theirs)
80
        self.assertEqual(r["class"], "differs-but-valid")
81
        self.assertTrue(any("oak_clean_where_git_conflicts" in x for x in r["reasons"]))
82
83
    def test_expectation_violation_wins(self):
84
        r = cl.classify(gitres(self.merged), oak(self.merged), self.base, self.ours, self.theirs,
85
                        expect={"paths": {"a": v("a0")[1]}})
86
        self.assertEqual(r["class"], "oak-wrong")
87
        self.assertIn("expectation_violated", r["reasons"][0])
88
89
    def test_expect_absent_and_clean(self):
90
        r = cl.classify(gitres(self.merged), oak(self.merged), self.base, self.ours, self.theirs,
91
                        expect={"clean": False})
92
        self.assertEqual(r["class"], "oak-wrong")
93
        r = cl.classify(gitres(self.merged), oak(self.merged), self.base, self.ours, self.theirs,
94
                        expect={"paths": {"gone": None}, "clean": True})
95
        self.assertEqual(r["class"], "equal-to-git")
96
97
    def test_unsupported_and_harness_and_oak_error(self):
98
        self.assertEqual(cl.classify(None, None, {}, {}, {}, unsupported="gitlink")["class"], "oak-unsupported")
99
        self.assertEqual(cl.classify(None, None, {}, {}, {}, harness_error="x")["class"], "harness-error")
100
        self.assertEqual(cl.classify(gitres({}), {"error_kind": "harness", "error": "shim"}, {}, {}, {})["class"],
101
                         "harness-error")
102
        r = cl.classify(gitres({}), {"error_kind": "oak", "error": "boom", "clean": None}, {}, {}, {})
103
        self.assertEqual(r["class"], "oak-wrong")
104
        self.assertEqual(cl.classify(gitres({}), None, {}, {}, {})["class"], "harness-error")
105
106
    def test_no_unique_base_leaves_law_unmeasured(self):
107
        r = cl.classify(gitres(self.merged), oak(self.merged), None, self.ours, self.theirs)
108
        self.assertEqual(r["class"], "equal-to-git")
109
        self.assertIsNone(r["evidence"]["law_no_lost_change"])
110
111
    def test_text_merge_law(self):
112
        base = b"1\n2\n3\n4\n5\n6\n"
113
        o = b"1 ours\n2\n3\n4\n5\n6\n"
114
        t = b"1\n2\n3\n4\n5\n6 theirs\n"
115
        good = b"1 ours\n2\n3\n4\n5\n6 theirs\n"
116
        self.assertTrue(cl.text_preserves_sides(base, o, t, good))
117
        self.assertFalse(cl.text_preserves_sides(base, o, t, o))
118
        self.assertFalse(cl.text_preserves_sides(base, o, t, b"<<<<<<< a\n" + good))
119
        trees = ({"f": v("b")}, {"f": v("o")}, {"f": v("t")})
120
        texts = {"f": {"base": base, "ours": o, "theirs": t, "merged": good}}
121
        law = cl.no_lost_change(*trees, {"f": v("m")}, set(), texts)
122
        self.assertEqual(law["text_merged"], ["f"])
123
        texts["f"]["merged"] = o
124
        self.assertEqual(cl.no_lost_change(*trees, {"f": v("m")}, set(), texts)["lost"], ["f"])
125
126
    def test_one_side_taken_wholesale_is_lost(self):
127
        # base exec+empty; ours: mode-only change; theirs: content change. Oak keeping theirs'
128
        # entry (content AND exec bit) drops ours' mode change.
129
        base = {"w": ("exec", "empty")}
130
        ours = {"w": ("file", "empty")}
131
        theirs = {"w": ("exec", "123")}
132
        law = cl.no_lost_change(base, ours, theirs, {"w": ("exec", "123")}, set())
133
        self.assertEqual(law["mode_lost"], ["w"])
134
        # a content change dropped wholesale stays a plain lost change
135
        law = cl.no_lost_change({"f": v("0")}, {"f": v("1")}, {"f": v("2")}, {"f": v("1")}, set())
136
        self.assertEqual(law["lost"], ["f"])
137
        law = cl.no_lost_change(base, ours, theirs, {"w": ("file", "123")}, set(),
138
                                {"w": {"base": b"", "ours": b"", "theirs": b"1\n2\n3\n", "merged": b"1\n2\n3\n"}})
139
        self.assertEqual(law["text_merged"], ["w"])
140
141
    def test_conflicted_paths_are_exempt_from_law(self):
142
        law = cl.no_lost_change(self.base, self.ours, self.theirs, {}, {"a", "b"})
143
        self.assertEqual(law, {"lost": [], "mode_lost": [], "markers": [], "markers_symlink": [], "fabricated": [], "text_merged": []})
144
145
    def test_text_merged_but_mode_dropped_is_oak_wrong(self):
146
        # QA shape generated/s20260928-022: ours edits lines, theirs edits other lines AND chmods.
147
        base, ours, theirs = {"f": ("file", "b")}, {"f": ("file", "o")}, {"f": ("exec", "t")}
148
        merged = b"1 ours\n2\n3\n4\n5\n6 theirs\n"
149
        texts = {"f": {"base": b"1\n2\n3\n4\n5\n6\n", "ours": b"1 ours\n2\n3\n4\n5\n6\n",
150
                       "theirs": b"1\n2\n3\n4\n5\n6 theirs\n", "merged": merged}}
151
        bad = cl.classify(gitres({"f": ("exec", "m")}), oak({"f": ("file", "m")}), base, ours, theirs, texts=texts)
152
        self.assertEqual(bad["class"], "oak-wrong")
153
        self.assertTrue(bad["reasons"][0].startswith("lost_mode_change"))
154
        good = cl.classify(gitres({"f": ("exec", "m")}), oak({"f": ("exec", "m")}), base, ours, theirs, texts=texts)
155
        self.assertEqual(good["class"], "equal-to-git")
156
157
    def test_same_content_different_modes_taken_wholesale_is_mode_loss(self):
158
        # both sides rename to the same path with different modes; Oak keeps one side silently
159
        law = cl.no_lost_change({}, {"t": ("file", "x")}, {"t": ("exec", "x")}, {"t": ("exec", "x")}, set())
160
        self.assertEqual(law["mode_lost"], ["t"])
161
162
    def test_symlink_markers_are_distinguished(self):
163
        ev = {"base": b"", "ours": b"a", "theirs": b"b", "merged": b"<<<<<<< x\na=======\nb>>>>>>> main\n"}
164
        law = cl.no_lost_change({}, {"l": ("symlink", "a")}, {"l": ("symlink", "b")}, {"l": ("symlink", "m")},
165
                                set(), {"l": ev})
166
        self.assertEqual(law["markers_symlink"], ["l"])
167
168
    def test_committed_markers_have_their_own_reason(self):
169
        base, ours, theirs = {"f": v("b")}, {"f": v("o")}, {"f": v("t")}
170
        texts = {"f": {"base": b"a\nx", "ours": b"a\ny", "theirs": b"a\nz",
171
                       "merged": b"a\n<<<<<<< br\ny||||||| original\nx=======\nz>>>>>>> main\n"}}
172
        r = cl.classify(gitres({"f": v("g")}, conflicts=["f"]), oak({"f": v("m")}), base, ours, theirs, texts=texts)
173
        self.assertEqual(r["class"], "oak-wrong")
174
        self.assertTrue(r["reasons"][0].startswith("committed_conflict_markers: f"))
175
176
    def test_failed_pull_hazard_reason(self):
177
        res = {"error_kind": "oak", "error": "IO error", "clean": None,
178
               "post_failure": {"hazard_head_moved_worktree_stale": True, "head_before": "a" * 64,
179
                                "head_after": "b" * 64, "worktree_changes_vs_head": 2,
180
                                "repull_parent_sync": "up_to_date"}}
181
        r = cl.classify(gitres({}), res, {}, {}, {})
182
        self.assertEqual(r["class"], "oak-wrong")
183
        self.assertTrue(r["reasons"][1].startswith("failed_pull_moved_head_worktree_stale"))
184
185
    def test_unknown_class_is_rejected(self):
186
        self.assertEqual(cl.validate_class("equal-to-git"), "equal-to-git")
187
        with self.assertRaises(ValueError):
188
            cl.validate_class("pass")
189
190
    def test_symmetry(self):
191
        f = oak(self.merged, conflicts=["a"])
192
        r = oak({"a": v("other-side-markers"), "b": v("b1")}, conflicts=["a"])
193
        self.assertTrue(cl.symmetry(f, r)["holds"])
194
        self.assertFalse(cl.symmetry(f, oak(self.merged))["holds"])
195
        self.assertIsNone(cl.symmetry(f, {"clean": None}))
196
197
    def test_idempotence(self):
198
        self.assertIsNone(cl.idempotence_holds({}))
199
        good = {"idempotence": {"returncode": 0, "head_unchanged": True, "tree_unchanged": True}}
200
        self.assertTrue(cl.idempotence_holds(good))
201
        good["idempotence"]["tree_unchanged"] = False
202
        self.assertFalse(cl.idempotence_holds(good))
203
204
    def test_oak_conflict_type(self):
205
        base = {"a": ("file", "1"), "m": ("file", "1")}
206
        self.assertEqual(cl.oak_conflict_type("a", base, {"a": ("file", "2")}, {}), "modify/delete")
207
        self.assertEqual(cl.oak_conflict_type("n", base, {"n": ("file", "2")}, {"n": ("file", "3")}), "add/add")
208
        self.assertEqual(cl.oak_conflict_type("m", base, {"m": ("exec", "1")}, {"m": ("symlink", "1")}), "mode")
209
210
211
class RunnerAccountingTests(unittest.TestCase):
212
    def test_failed_pull_counts_as_prediction_disagreement(self):
213
        from mergesuite.runner import Suite
214
        oak_res = {"error_kind": "oak", "error": "x", "clean": None, "tree": {},
215
                   "prediction": {"available": True, "clean": True, "conflict_files": []},
216
                   "post_failure": {"hazard_head_moved_worktree_stale": True}}
217
        verdict = cl.classify(gitres({}), oak_res, {}, {}, {})
218
        out = Suite._result_fields(None, gitres({}), oak_res, verdict)
219
        self.assertIs(out["laws"]["prediction_agrees"], False)
220
        self.assertTrue(out["post_failure_hazard"])
221
222
    def test_known_bug_mapping_and_summary(self):
223
        from mergesuite.runner import load_known_bugs, match_known_bug, summarize
224
        bugs = load_known_bugs()
225
        self.assertEqual({b["id"] for b in bugs}, {"fb-537", "fb-538", "fb-539", "fb-540", "fb-541"})
226
        laws = {k: None for k in ("no_lost_change", "idempotent", "symmetric", "false_conflict",
227
                                  "missed_conflict", "prediction_agrees")}
228
        rows = [
229
            {"class": "oak-wrong", "scenario": "gix-tree/symlink-addition/A-B", "origin": "o",
230
             "reasons": ["committed_conflict_markers_symlink: link"], "laws": dict(laws)},
231
            {"class": "oak-wrong", "scenario": "gix-text/complex/conflict-without-LF", "origin": "o",
232
             "reasons": ["committed_conflict_markers: file"], "laws": dict(laws)},
233
            {"class": "oak-wrong", "scenario": "gix-cartesian/matrix/keep--file-to-dir", "origin": "o",
234
             "reasons": ["oak_error: IO error"], "laws": dict(laws)},
235
            {"class": "oak-wrong", "scenario": "x", "origin": "o", "reasons": ["lost_change: q"], "laws": dict(laws)},
236
            {"class": "equal-to-git", "scenario": "y", "origin": "o", "reasons": [], "laws": dict(laws)},
237
        ]
238
        self.assertEqual([match_known_bug(r, bugs) for r in rows], ["fb-538", "fb-537", "fb-540", None, None])
239
        summary = summarize(rows, bugs)
240
        self.assertEqual(summary["oak_wrong_unmapped"], 1)
241
        self.assertEqual(summary["known_bugs"]["fb-541"]["status"], "NOT reproduced (fixed?)")
242
243
244
class OracleParsingTests(unittest.TestCase):
245
    def test_parse_merge_tree_z(self):
246
        out = (b"tree123\0"
247
               b"100644 aaa 1\ta\x00100644 bbb 2\ta\x00100644 ccc 3\ta\x00"
248
               b"\0"
249
               b"1\0a\0Auto-merging\0Auto-merging a\n\0"
250
               b"1\0a\0CONFLICT (contents)\0CONFLICT (content): Merge conflict in a\n\0")
251
        parsed = parse_merge_tree_z(out)
252
        self.assertEqual(parsed["tree_oid"], "tree123")
253
        self.assertEqual(parsed["conflict_stages"], {"a": [1, 2, 3]})
254
        self.assertEqual(git_conflicts(parsed), {"a": ["content"]})
255
256
    def test_parse_clean(self):
257
        parsed = parse_merge_tree_z(b"t\0\0")
258
        self.assertEqual(parsed["conflict_stages"], {})
259
        self.assertEqual(parsed["messages"], [])
260
261
    def test_type_vocabulary(self):
262
        self.assertIsNone(normalize_git_conflict_type("Auto-merging"))
263
        self.assertEqual(normalize_git_conflict_type("CONFLICT (modify/delete)"), "modify/delete")
264
        self.assertEqual(normalize_git_conflict_type("CONFLICT (rename/delete)"), "rename/delete")
265
        self.assertEqual(normalize_git_conflict_type("CONFLICT (directory rename split)"), "directory-rename")
266
267
    def test_conflict_show_and_prediction_parsing(self):
268
        doc = {"conflicts": [{"path": "f", "has_conflict_markers": False, "exists": True}]}
269
        self.assertEqual(parse_conflict_show(doc, {}), {"f": ["no-markers"]})
270
        self.assertEqual(parse_conflict_show(None, {"conflict_paths": ["x"]}), {"x": []})
271
        self.assertEqual(summarize_prediction(None, 6), {"available": False, "returncode": 6})
272
        pred = summarize_prediction({"prediction_available": True, "clean": False,
273
                                     "conflict_files": [{"path": "b"}, "a"]}, 0)
274
        self.assertEqual(pred["conflict_files"], ["a", "b"])
275
276
277
@unittest.skipUnless(shutil.which("git"), "git not installed")
278
class GitBackedTests(unittest.TestCase):
279
    def setUp(self):
280
        self.tmp = Path(tempfile.mkdtemp(prefix="mergesuite-test-"))
281
        self.home = self.tmp / "home"
282
        from oakbench.subjects import which_measured_git
283
        self.git_bin = which_measured_git() or shutil.which("git")
284
        prepare_git_home(self.home, self.git_bin)
285
286
    def tearDown(self):
287
        shutil.rmtree(self.tmp, ignore_errors=True)
288
289
    def test_builder_oracle_and_materialize_roundtrip(self):
290
        from mergesuite.builder import RepoBuilder, lines
291
        b = RepoBuilder(self.tmp / "r", self.git_bin, self.home)
292
        base = b.commit({"f": lines("1", "2", "3"), "x": ("exec", b"#!/bin/sh\n"), "l": ("link", "f")})
293
        a = b.commit({"f": lines("1 a", "2", "3"), "x": ("exec", b"#!/bin/sh\n"), "l": ("link", "f")}, [base])
294
        c = b.commit({"f": lines("1", "2", "3 c"), "l": ("link", "f")}, [base], message="unique")
295
        self.assertEqual(b.merge_bases(a, c), [base])
296
        merge = b.merge_tree(a, c)
297
        self.assertTrue(merge["clean"])
298
        self.assertEqual(sorted(merge["tree"]), ["f", "l"])
299
        self.assertEqual(merge["tree"]["l"][0], "symlink")
300
        out = self.tmp / "mat"
301
        out.mkdir()
302
        b.materialize(base, out)
303
        self.assertEqual(worktree_manifest(out), b.tree(base))
304
        # first parent order is preserved
305
        m = b.commit({"f": lines("z")}, [a, c])
306
        self.assertEqual(b.out("rev-parse", m + "^1"), a)
307
308
    def test_oracle_reports_conflict_types(self):
309
        from mergesuite.builder import RepoBuilder, lines
310
        b = RepoBuilder(self.tmp / "r2", self.git_bin, self.home)
311
        base = b.commit({"f": lines("1", "2"), "g": lines("g")})
312
        a = b.commit({"f": lines("1", "A")}, [base])
313
        c = b.commit({"f": lines("1", "C"), "g": lines("g2")}, [base])
314
        merge = b.merge_tree(a, c)
315
        self.assertFalse(merge["clean"])
316
        self.assertEqual(git_conflicts(merge), {"f": ["content"], "g": ["modify/delete"]})
317
318
    def test_sources_build_without_oak(self):
319
        from mergesuite import sources
320
        scen = sources.git_t64xx(self.tmp / "t64", self.git_bin, self.home)
321
        ids = [s["id"] for s in scen]
322
        self.assertIn("git-t6423-1a", ids)
323
        self.assertTrue(all(len(s["directions"]) == 2 and s["unsupported"] is None for s in scen))
324
        native = sources.native_incidents(self.tmp / "nat", self.git_bin, self.home)
325
        by_id = {s["id"]: s for s in native}
326
        cc = by_id["incident/criss-cross-tie-break-side-newer"]["directions"][0]
327
        self.assertEqual(len(cc["merge_bases"]), 2)  # two best common ancestors by construction
328
        self.assertIsNone(cc["trees"]["base"])
329
        skew = by_id["incident/clock-skew-merge-base-skewed"]["directions"][0]
330
        self.assertEqual(len(skew["merge_bases"]), 1)
331
        # self-merges produce a single direction (no duplicate rows sharing a case dir)
332
        rev = by_id["incident/revert-after-client-sync"]
333
        tip = rev["directions"][0]["oracle"]["theirs"]
334
        one = sources.symmetric("self", "o", {}, Git(rev["repo"], self.git_bin, self.home), tip, tip)
335
        self.assertEqual(len(one["directions"]), 1)
336
        gen = sources.generated(self.tmp / "gen", self.git_bin, self.home, 3, 7)
337
        gen2 = sources.generated(self.tmp / "gen2", self.git_bin, self.home, 3, 7)
338
        self.assertEqual([s["notes"] for s in gen], [s["notes"] for s in gen2])  # seeded
339
340
341
class VendoringTests(unittest.TestCase):
342
    def test_vendored_scripts_are_byte_identical_to_pinned_upstream(self):
343
        meta = json.loads((VENDOR / "UPSTREAM.json").read_text())
344
        self.assertEqual(meta["license"], "MIT OR Apache-2.0")
345
        for name, info in meta["files"].items():
346
            path = VENDOR / name
347
            self.assertTrue(path.is_file(), name)
348
            if "sha256" in info:
349
                self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), info["sha256"], name)
350
        notice = (VENDOR / "NOTICE").read_text()
351
        self.assertIn(meta["commit"], notice)
352
        self.assertIn("Permission is hereby granted", (VENDOR / "LICENSE-MIT").read_text())
353
        self.assertIn("Apache License", (VENDOR / "LICENSE-APACHE").read_text())
354
355
    def test_no_git_test_suite_text_is_vendored(self):
356
        merge_root = ROOT / "suites" / "merge"
357
        for path in merge_root.rglob("*"):
358
            if path.is_file() and path.suffix in (".py", ".sh", ".md", ".json"):
359
                text = path.read_text(errors="replace")
360
                self.assertNotIn("test_expect_success", text, path)
361
                self.assertNotIn(". ./test-lib.sh", text, path)
362
363
364
if __name__ == "__main__":
365
    unittest.main()