Viewing files at commit 54b9920be8dfCommit detailsView latest

Browse files
tests/test_make_conflict_corpus.py 182 lines · 8.0 KB · python
1
from __future__ import annotations
2
3
import base64
4
import hashlib
5
import json
6
import shutil
7
import subprocess
8
import sys
9
import tempfile
10
import unittest
11
from pathlib import Path
12
13
ROOT = Path(__file__).resolve().parents[1]
14
GENERATOR = ROOT / "scripts" / "make_conflict_corpus.py"
15
16
sys.path.insert(0, str(ROOT / "scripts"))
17
18
from oakbench.fixture_registry import load_fixture_registry  # noqa: E402
19
20
# A fast, representative subset: one text size, the clean-merge/broken-build
21
# case, an unmergeable deletion, and the most common real agent conflict.
22
TEST_KINDS = ("text-overlap-small", "adjacent-lines", "delete-edit", "lockfile")
23
24
25
def run_generator(out: Path, seed: str) -> subprocess.CompletedProcess:
26
    return subprocess.run(
27
        [sys.executable, str(GENERATOR), "--out", str(out), "--seed", seed,
28
         "--quick", "--kinds", ",".join(TEST_KINDS)],
29
        capture_output=True, text=True,
30
    )
31
32
33
def git(repo: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess:
34
    return subprocess.run(["git", "-C", str(repo), *args], check=check,
35
                          capture_output=True, text=True)
36
37
38
def resolution_bytes(resolution: dict) -> bytes:
39
    if "content" in resolution:
40
        return resolution["content"].encode()
41
    return base64.b64decode(resolution["content_b64"])
42
43
44
class MakeConflictCorpusTest(unittest.TestCase):
45
    """Generate once, then merge each kind in a scratch clone and check the oracle."""
46
47
    @classmethod
48
    def setUpClass(cls) -> None:
49
        if shutil.which("git") is None:
50
            raise unittest.SkipTest("git not available")
51
        cls.tmp = tempfile.TemporaryDirectory()
52
        cls.addClassCleanup(cls.tmp.cleanup)
53
        base = Path(cls.tmp.name)
54
        cls.corpus_a = base / "a"
55
        cls.corpus_b = base / "b"
56
        for out in (cls.corpus_a, cls.corpus_b):
57
            proc = run_generator(out, "test")
58
            if proc.returncode != 0:
59
                raise AssertionError(
60
                    f"generator failed for {out} (exit {proc.returncode}):\n"
61
                    f"{proc.stdout}\n{proc.stderr}")
62
        cls.merges = {kind: cls._merge_in_clone(kind) for kind in TEST_KINDS}
63
64
    @classmethod
65
    def _merge_in_clone(cls, kind: str) -> dict:
66
        clone = Path(cls.tmp.name) / "merges" / kind
67
        clone.parent.mkdir(parents=True, exist_ok=True)
68
        subprocess.run(["git", "clone", "--quiet", str(cls.corpus_a / kind), str(clone)],
69
                       check=True, capture_output=True)
70
        git(clone, "branch", "theirs", "origin/theirs")
71
        merge = git(clone, "-c", "user.name=Test", "-c", "[email protected]",
72
                    "merge", "--no-edit", "theirs", check=False)
73
        unmerged = git(clone, "diff", "--name-only", "--diff-filter=U")
74
        return {
75
            "clone": clone,
76
            "returncode": merge.returncode,
77
            "unmerged": sorted(p for p in unmerged.stdout.splitlines() if p),
78
        }
79
80
    def oracle(self, kind: str, corpus: Path | None = None) -> dict:
81
        corpus = corpus or self.corpus_a
82
        return json.loads((corpus / f"{kind}.RESOLUTION.json").read_text())
83
84
    def test_oracle_schema(self) -> None:
85
        for kind in TEST_KINDS:
86
            oracle = self.oracle(kind)
87
            self.assertEqual(oracle["schema_version"], 1)
88
            self.assertEqual(oracle["kind"], kind)
89
            self.assertEqual(oracle["seed"], "test")
90
            self.assertIn("expected_conflict", oracle)
91
            self.assertIn("conflicted_paths", oracle)
92
            self.assertIn("path", oracle["resolution"])
93
            self.assertIn("content_sha256", oracle["resolution"])
94
95
    def test_merge_outcome_matches_oracle(self) -> None:
96
        for kind in TEST_KINDS:
97
            oracle = self.oracle(kind)
98
            merge = self.merges[kind]
99
            conflicted = merge["returncode"] != 0
100
            self.assertEqual(conflicted, oracle["expected_conflict"],
101
                             f"{kind}: merge outcome disagrees with oracle")
102
            self.assertEqual(merge["unmerged"], oracle["conflicted_paths"],
103
                             f"{kind}: unmerged paths disagree with oracle")
104
105
    def test_adjacent_lines_clean_merge_breaks_build(self) -> None:
106
        oracle = self.oracle("adjacent-lines")
107
        merge = self.merges["adjacent-lines"]
108
        self.assertEqual(merge["returncode"], 0, "adjacent-lines must merge clean")
109
        check = oracle["post_merge_check"]
110
        self.assertIsNotNone(check)
111
        proc = subprocess.run(check["command"], cwd=merge["clone"],
112
                              capture_output=True, text=True)
113
        self.assertNotEqual(proc.returncode, 0, "build must break after the clean merge")
114
        self.assertEqual(proc.returncode, check["expected_returncode_clean_merge"])
115
116
    def test_adjacent_lines_check_passes_on_each_branch(self) -> None:
117
        oracle = self.oracle("adjacent-lines")
118
        repo = self.corpus_a / "adjacent-lines"
119
        for branch in ("theirs", "main", "ours"):
120
            git(repo, "checkout", "-q", branch)
121
            proc = subprocess.run(oracle["post_merge_check"]["command"], cwd=repo,
122
                                  capture_output=True, text=True)
123
            self.assertEqual(proc.returncode, 0, f"check must pass on branch {branch}")
124
125
    def test_text_overlap_resolution_applies(self) -> None:
126
        oracle = self.oracle("text-overlap-small")
127
        merge = self.merges["text-overlap-small"]
128
        self.assertNotEqual(merge["returncode"], 0)
129
        path = oracle["resolution"]["path"]
130
        self.assertIn(path, oracle["conflicted_paths"])
131
        content = resolution_bytes(oracle["resolution"])
132
        self.assertNotIn(b"<<<<<<<", content)
133
        # Apply the oracle: write the resolved content over the conflicted file.
134
        target = merge["clone"] / path
135
        target.write_bytes(content)
136
        self.assertEqual(hashlib.sha256(target.read_bytes()).hexdigest(),
137
                         oracle["resolution"]["content_sha256"])
138
        # The resolved file completes the merge cleanly.
139
        git(merge["clone"], "add", path)
140
        git(merge["clone"], "-c", "user.name=Test", "-c", "[email protected]",
141
            "commit", "-q", "-m", "resolve")
142
        status = git(merge["clone"], "status", "--porcelain")
143
        self.assertEqual(status.stdout.strip(), "")
144
145
    def test_resolution_sha_matches_content_all_kinds(self) -> None:
146
        for kind in TEST_KINDS:
147
            resolution = self.oracle(kind)["resolution"]
148
            self.assertEqual(hashlib.sha256(resolution_bytes(resolution)).hexdigest(),
149
                             resolution["content_sha256"], kind)
150
151
    def test_same_seed_yields_identical_oracles(self) -> None:
152
        for kind in TEST_KINDS:
153
            a = (self.corpus_a / f"{kind}.RESOLUTION.json").read_bytes()
154
            b = (self.corpus_b / f"{kind}.RESOLUTION.json").read_bytes()
155
            self.assertEqual(a, b, f"{kind}: oracle differs between identical-seed runs")
156
157
    def test_same_seed_yields_identical_history(self) -> None:
158
        for kind in TEST_KINDS:
159
            for ref in ("main", "ours", "theirs"):
160
                sha_a = git(self.corpus_a / kind, "rev-parse", ref).stdout
161
                sha_b = git(self.corpus_b / kind, "rev-parse", ref).stdout
162
                self.assertEqual(sha_a, sha_b, f"{kind}/{ref}: history differs")
163
164
165
class ConflictCorpusRegistryTest(unittest.TestCase):
166
    def test_registry_entry(self) -> None:
167
        registry = load_fixture_registry()
168
        self.assertIn("bench-conflict-corpus-v1", registry)
169
        spec = registry["bench-conflict-corpus-v1"]
170
        self.assertEqual(spec.generator, "scripts/make_conflict_corpus.py")
171
        self.assertEqual(spec.seed, "oakbench-conflicts-v1")
172
        self.assertEqual(spec.license, "synthetic")
173
        self.assertIsNone(spec.manifest)
174
        self.assertEqual(spec.hosting, ("local",))
175
        for kind in ("text-overlap-small", "text-overlap-medium", "text-overlap-large",
176
                     "adjacent-lines", "rename-rename", "rename-edit", "delete-edit",
177
                     "binary-binary", "lockfile"):
178
            self.assertIn(kind, spec.description)
179
180
181
if __name__ == "__main__":
182
    unittest.main()