Viewing files at commit c467b16393a4Commit detailsView latest

Browse files
tests/test_conformance_matrix.py 313 lines · 13.6 KB · python
1
"""Conformance matrix oracles: every behaviour passes a correct VCS and fails injected faults.
2
3
``CopyVCS`` is a directory-copy "VCS" with no bugs; each fault flag injects
4
one class of defect at one point of the roundtrip (commit, source worktree,
5
clone, pull, status). Every behaviour must pass the faultless model and fail
6
each applicable fault at the named leg, so no behaviour has a vacuous oracle.
7
"""
8
import os
9
from pathlib import Path
10
import re
11
import shutil
12
import sys
13
import tempfile
14
import unittest
15
16
ROOT = Path(__file__).resolve().parents[1]
17
sys.path.insert(0, str(ROOT / "scripts"))
18
from oakbench import conformance_gate  # noqa: E402
19
from oakbench import conformance_matrix as matrix  # noqa: E402
20
from oakbench.conformance import SCENARIOS  # noqa: E402
21
22
GIT_TEST_IDS = {"t0010", "t0020", "t0021", "t0027", "t0050", "t2003", "t2007", "t2021", "t3700", "t3902", "t3910",
23
                "t4011", "t4012", "t4102", "t4114", "t4129", "t7001"}
24
25
26
def copy_tree(source: Path, destination: Path, skip=(".fake",)):
27
    destination.mkdir(parents=True, exist_ok=True)
28
    for entry in sorted(os.listdir(source)):
29
        if entry in skip:
30
            continue
31
        src, dst = source / entry, destination / entry
32
        if src.is_symlink():
33
            os.symlink(os.readlink(src), dst)
34
        elif src.is_dir():
35
            copy_tree(src, dst, skip=())
36
        else:
37
            shutil.copy2(src, dst)
38
39
40
def clear(directory: Path):
41
    for entry in os.listdir(directory):
42
        path = directory / entry
43
        if entry == ".fake":
44
            continue
45
        if path.is_symlink() or path.is_file():
46
            path.unlink()
47
        else:
48
            shutil.rmtree(path)
49
50
51
class CopyVCS:
52
    """A faultless copy-based model of commit/push/clone/pull, with injectable faults."""
53
    kind = "fake"
54
55
    def __init__(self, base: Path, faults=()):
56
        self.base, self.faults = base, set(faults)
57
        self.snapshots = {}
58
        self.counter = 0
59
60
    def _snapshot(self, source: Path) -> Path:
61
        self.counter += 1
62
        target = self.base / f"snap{self.counter}"
63
        copy_tree(source, target)
64
        return target
65
66
    def init(self, repo):
67
        (repo / ".fake").mkdir()
68
69
    def commit(self, repo, message):
70
        snap = self._snapshot(repo)
71
        if "commit_inject" in self.faults:
72
            (snap / "zz-injected-by-commit").write_bytes(b"x")
73
        if "mutate_source" in self.faults:
74
            (repo / "zz-written-by-commit").write_bytes(b"x")
75
        self.snapshots[repo] = snap
76
77
    def push(self, repo):
78
        self.remote = self.snapshots[repo]
79
80
    def clone(self, repo, destination):
81
        copy_tree(self.remote, destination)
82
        (destination / ".fake").mkdir()
83
        self.snapshots[destination] = self.remote
84
        if "clone_inject" in self.faults:
85
            (destination / "zz-injected-by-clone").write_bytes(b"x")
86
87
    def pull(self, repo):
88
        if "pull_stale" in self.faults:
89
            (repo / "zz-stale-after-pull").write_bytes(b"x")
90
            return
91
        clear(repo)
92
        copy_tree(self.remote, repo)
93
        self.snapshots[repo] = self.remote
94
95
    def status_clean(self, repo):
96
        if "dirty" in self.faults:
97
            raise matrix.LegFailure("status reports changes")
98
99
    def committed(self, repo):
100
        return matrix.fs_manifest(self.snapshots[repo]), {"head": str(self.snapshots[repo])}
101
102
103
class BehaviourOracleTests(unittest.TestCase):
104
    @classmethod
105
    def setUpClass(cls):
106
        cls.caps = matrix.probe_capabilities(Path(tempfile.gettempdir()) / "oakbench-caps-probe")
107
108
    def runnable(self):
109
        for behaviour in matrix.BEHAVIOURS:
110
            if behaviour.gated or any(not self.caps.get(cap) for cap in behaviour.requires):
111
                continue
112
            yield behaviour
113
114
    def run_fake(self, behaviour, faults=()):
115
        with tempfile.TemporaryDirectory() as tmp:
116
            base = Path(tmp)
117
            (base / "work").mkdir()
118
            vcs = CopyVCS(base, faults)
119
            legs, committed = matrix.run_behaviour(vcs, behaviour, base / "work", {})
120
            return legs
121
122
    def test_every_behaviour_passes_a_correct_vcs(self):
123
        ran = 0
124
        for behaviour in self.runnable():
125
            with self.subTest(behaviour=behaviour.id):
126
                legs = self.run_fake(behaviour)
127
                self.assertTrue(legs)
128
                self.assertEqual({name: leg for name, leg in legs.items() if leg["status"] != "pass"}, {})
129
                if len(behaviour.steps) > 1:
130
                    self.assertEqual(legs["pull_fs"]["status"], "pass")
131
                ran += 1
132
        self.assertGreater(ran, 40)
133
134
    def test_every_behaviour_detects_each_fault_at_its_leg(self):
135
        for behaviour in self.runnable():
136
            if behaviour.id in ("large_file_chunked_v1", "many_small_files_v1"):
137
                continue  # same oracle path as the small behaviours; their copies are slow (covered by the pass test)
138
            multi = len(behaviour.steps) > 1
139
            cases = {"commit_inject": "commit_read@1" if multi else "commit_read",
140
                     "mutate_source": "source_preserved@1" if multi else "source_preserved", "clone_inject": "clone_fs",
141
                     "dirty": "clone_status_clean"}
142
            if multi:
143
                cases["pull_stale"] = "pull_fs"
144
            for fault, leg in cases.items():
145
                with self.subTest(behaviour=behaviour.id, fault=fault):
146
                    legs = self.run_fake(behaviour, {fault})
147
                    self.assertEqual(legs[leg]["status"], "fail", legs)
148
149
    def test_clone_fault_is_not_mistaken_for_committed_state(self):
150
        legs = self.run_fake(matrix.BY_ID["eol_crlf_v1"], {"clone_inject"})
151
        self.assertEqual(legs["clone_fs"]["status"], "fail")
152
        self.assertEqual(legs["clone_read"]["status"], "pass")
153
        self.assertEqual(legs["commit_read"]["status"], "pass")
154
155
156
class ManifestOracleTests(unittest.TestCase):
157
    def test_each_loss_class_is_detected(self):
158
        with tempfile.TemporaryDirectory() as tmp:
159
            root = Path(tmp)
160
            (root / "f").write_bytes(b"payload")
161
            (root / "l").symlink_to("f")
162
            expected = matrix.fs_manifest(root)
163
            mutations = [
164
                lambda: (root / "f").write_bytes(b"payloae"),
165
                lambda: (root / "f").chmod(0o755),
166
                lambda: ((root / "l").unlink(), (root / "l").symlink_to("g")),
167
                lambda: ((root / "l").unlink(), (root / "l").write_bytes(b"f")),
168
                lambda: (root / "extra").write_bytes(b""),
169
                lambda: (root / "f").rename(root / "F2"),
170
            ]
171
            for mutate in mutations:
172
                mutate()
173
                self.assertIsNotNone(matrix.diff_manifests(expected, matrix.fs_manifest(root)))
174
                shutil.rmtree(root)
175
                root.mkdir()
176
                (root / "f").write_bytes(b"payload")
177
                (root / "f").chmod(0o644)
178
                (root / "l").symlink_to("f")
179
                self.assertIsNone(matrix.diff_manifests(expected, matrix.fs_manifest(root)))
180
181
    def test_empty_directories_are_observations_not_content(self):
182
        with tempfile.TemporaryDirectory() as tmp:
183
            root = Path(tmp)
184
            (root / "void" / "deeper").mkdir(parents=True)
185
            dirs = []
186
            self.assertEqual(matrix.fs_manifest(root, dirs), {})
187
            self.assertEqual(dirs, ["void", "void/deeper"])
188
189
    def test_inplace_edit_keeps_size_and_mtime(self):
190
        with tempfile.TemporaryDirectory() as tmp:
191
            root = Path(tmp)
192
            matrix.apply_op(root, ("write", "r", b"aaaa", False))
193
            before = (root / "r").stat()
194
            matrix.apply_op(root, ("inplace", "r", b"bbbb"))
195
            after = (root / "r").stat()
196
            self.assertEqual((before.st_size, before.st_mtime_ns), (after.st_size, after.st_mtime_ns))
197
            self.assertEqual((root / "r").read_bytes(), b"bbbb")
198
199
    def test_sparse_op_writes_declared_size_and_tail(self):
200
        with tempfile.TemporaryDirectory() as tmp:
201
            root = Path(tmp)
202
            matrix.apply_op(root, ("sparse", "s", 1 << 20, b"tail\n"))
203
            data = (root / "s").read_bytes()
204
            self.assertEqual(len(data), 1 << 20)
205
            self.assertTrue(data.endswith(b"tail\n"))
206
207
208
class LoopbackServerTests(unittest.TestCase):
209
    @unittest.skipUnless(os.name == "posix", "process groups required")
210
    def test_stop_reaches_grandchild_behind_a_wrapper_script(self):
211
        with tempfile.TemporaryDirectory() as tmp:
212
            root = Path(tmp)
213
            pidfile = root / "grandchild.pid"
214
            listener = root / "listen.py"
215
            listener.write_text(
216
                "import socket, sys, time, os\n"
217
                "port = int(sys.argv[sys.argv.index('--port') + 1])\n"
218
                f"open({str(pidfile)!r}, 'w').write(str(os.getpid()))\n"
219
                "s = socket.socket(); s.bind(('127.0.0.1', port)); s.listen()\n"
220
                "time.sleep(300)\n")
221
            wrapper = root / "fake-oak"
222
            # No exec: the listener is a grandchild of the process Popen started.
223
            wrapper.write_text(f"#!/bin/sh\n{sys.executable} {listener} \"$@\" &\nwait\n")
224
            wrapper.chmod(0o755)
225
            with matrix.LoopbackOakServer(str(wrapper), root / "data", dict(os.environ)):
226
                grandchild = int(pidfile.read_text())
227
                os.kill(grandchild, 0)
228
            for _ in range(50):
229
                try:
230
                    os.kill(grandchild, 0)
231
                except ProcessLookupError:
232
                    break
233
                import time
234
                time.sleep(0.1)
235
            else:
236
                self.fail("grandchild server survived LoopbackOakServer exit")
237
238
239
class CatalogTests(unittest.TestCase):
240
    def test_ids_are_versioned_unique_and_disjoint_from_legacy(self):
241
        ids = [b.id for b in matrix.BEHAVIOURS]
242
        self.assertEqual(len(ids), len(set(ids)))
243
        self.assertTrue(all(re.fullmatch(r"[a-z0-9_]+_v\d+", i) for i in ids))
244
        self.assertFalse(set(ids) & set(SCENARIOS))
245
        self.assertGreaterEqual(len(ids), 50)
246
247
    def test_required_coverage_groups(self):
248
        groups = {b.group for b in matrix.BEHAVIOURS}
249
        self.assertEqual(groups, {"paths", "modes", "symlinks", "file_dir", "empty", "content", "history"})
250
        for needed in ("path_tab_v1", "path_unicode_nfd_v1", "path_case_only_rename_v1", "mode_exec_remove_v1",
251
                       "symlink_absolute_v1", "file_to_symlink_v1", "dir_to_file_v1", "empty_dir_semantics_v1",
252
                       "huge_file_sparse_v1", "binary_detection_v1", "eol_mixed_v1", "delete_readd_v1",
253
                       "many_small_files_v1"):
254
            self.assertIn(needed, matrix.BY_ID)
255
        self.assertTrue(matrix.BY_ID["huge_file_sparse_v1"].gated)
256
257
    def test_prior_art_cites_known_git_test_ids_only(self):
258
        for behaviour in matrix.BEHAVIOURS:
259
            for citation in behaviour.prior_art:
260
                match = re.match(r"git (t\d{4})-", citation)
261
                if match:
262
                    self.assertIn(match.group(1), GIT_TEST_IDS, citation)
263
264
    def test_expected_failures_name_real_behaviours_legs_and_tickets(self):
265
        expected = conformance_gate.load_expected()
266
        known = set(matrix.BY_ID) | set(SCENARIOS)
267
        legs = set(matrix.LEGS) | {"commit_push", "behaviour"}
268
        for behaviour, entry in expected["entries"].items():
269
            self.assertIn(behaviour, known)
270
            for leg in entry["failed_legs"]:
271
                self.assertIn(leg.split("@")[0], legs)
272
        tickets = {b: e["ticket"] for b, e in expected["entries"].items()}
273
        self.assertEqual(tickets["path_tab_v1"], "fb-531")
274
        self.assertEqual(tickets["path_roundtrip_v1"], "fb-531")
275
        self.assertEqual(tickets["symlink_to_dir_real_v1"], "fb-542")
276
        self.assertEqual(expected["entries"]["path_tab_v1"]["outcome"], "refused")
277
        for skip in expected["coverage"]["darwin-ci-ni"]["expected_skips"]:
278
            self.assertIn(skip, matrix.BY_ID)
279
280
281
@unittest.skipUnless(shutil.which("git"), "Git required")
282
class RealGitMatrixTests(unittest.TestCase):
283
    def test_git_passes_a_representative_subset(self):
284
        subset = ["path_tab_v1", "mode_exec_add_v1", "file_to_dir_v1", "eol_crlf_v1", "delete_all_v1"]
285
        with tempfile.TemporaryDirectory() as tmp:
286
            rows = matrix.run_matrix(Path(tmp), {"git": "git"}, subset)
287
        for row in rows:
288
            with self.subTest(behaviour=row["scenario"]):
289
                if row["returncode"] == 77:
290
                    continue
291
                self.assertEqual(row["returncode"], 0, row.get("failure"))
292
                multi = len(matrix.BY_ID[row["scenario"]].steps) > 1
293
                self.assertEqual(row["legs"]["commit_read@1" if multi else "commit_read"]["status"], "pass")
294
295
296
@unittest.skipUnless(shutil.which("oak") and shutil.which("git"), "Oak and Git required")
297
class RealOakMatrixTests(unittest.TestCase):
298
    def test_oak_rows_carry_independent_reader_and_git_differential(self):
299
        with tempfile.TemporaryDirectory() as tmp:
300
            rows = matrix.run_matrix(Path(tmp), {"git": "git", "oak": "oak"}, ["eol_mixed_v1", "path_tab_v1"])
301
        oak = {row["scenario"]: row for row in rows if row["subject"] == "oak"}
302
        mixed = oak["eol_mixed_v1"]
303
        self.assertEqual(mixed["returncode"], 0, mixed.get("failure"))
304
        self.assertEqual(mixed["observations"]["committed_reader"]["reader"], "independent_oak_sqlite_reader")
305
        self.assertEqual(mixed["git_differential"], "identical")
306
        # fb-531: rejected at commit; if Oak fixes it this flips and the gate reports FIXED.
307
        self.assertIn(oak["path_tab_v1"]["returncode"], (0, 1))
308
        if oak["path_tab_v1"]["returncode"] == 1:
309
            self.assertEqual(oak["path_tab_v1"]["failed_legs"], ["commit_push"])
310
311
312
if __name__ == "__main__":
313
    unittest.main()