| 1 | """Conformance matrix oracles: every behaviour passes a correct VCS and fails injected faults.
|
| 2 |
|
| 3 | ``CopyVCS`` is a directory-copy "VCS" with no bugs; each fault flag injects
|
| 4 | one class of defect at one point of the roundtrip (commit, source worktree,
|
| 5 | clone, pull, status). Every behaviour must pass the faultless model and fail
|
| 6 | each applicable fault at the named leg, so no behaviour has a vacuous oracle.
|
| 7 | """
|
| 8 | import os
|
| 9 | from pathlib import Path
|
| 10 | import re
|
| 11 | import shutil
|
| 12 | import sys
|
| 13 | import tempfile
|
| 14 | import unittest
|
| 15 |
|
| 16 | ROOT = Path(__file__).resolve().parents[1]
|
| 17 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 18 | from oakbench import conformance_gate # noqa: E402
|
| 19 | from oakbench import conformance_matrix as matrix # noqa: E402
|
| 20 | from oakbench.conformance import SCENARIOS # noqa: E402
|
| 21 |
|
| 22 | GIT_TEST_IDS = {"t0010", "t0020", "t0021", "t0027", "t0050", "t2003", "t2007", "t2021", "t3700", "t3902", "t3910",
|
| 23 | "t4011", "t4012", "t4102", "t4114", "t4129", "t7001"}
|
| 24 |
|
| 25 |
|
| 26 | def copy_tree(source: Path, destination: Path, skip=(".fake",)):
|
| 27 | destination.mkdir(parents=True, exist_ok=True)
|
| 28 | for entry in sorted(os.listdir(source)):
|
| 29 | if entry in skip:
|
| 30 | continue
|
| 31 | src, dst = source / entry, destination / entry
|
| 32 | if src.is_symlink():
|
| 33 | os.symlink(os.readlink(src), dst)
|
| 34 | elif src.is_dir():
|
| 35 | copy_tree(src, dst, skip=())
|
| 36 | else:
|
| 37 | shutil.copy2(src, dst)
|
| 38 |
|
| 39 |
|
| 40 | def clear(directory: Path):
|
| 41 | for entry in os.listdir(directory):
|
| 42 | path = directory / entry
|
| 43 | if entry == ".fake":
|
| 44 | continue
|
| 45 | if path.is_symlink() or path.is_file():
|
| 46 | path.unlink()
|
| 47 | else:
|
| 48 | shutil.rmtree(path)
|
| 49 |
|
| 50 |
|
| 51 | class CopyVCS:
|
| 52 | """A faultless copy-based model of commit/push/clone/pull, with injectable faults."""
|
| 53 | kind = "fake"
|
| 54 |
|
| 55 | def __init__(self, base: Path, faults=()):
|
| 56 | self.base, self.faults = base, set(faults)
|
| 57 | self.snapshots = {}
|
| 58 | self.counter = 0
|
| 59 |
|
| 60 | def _snapshot(self, source: Path) -> Path:
|
| 61 | self.counter += 1
|
| 62 | target = self.base / f"snap{self.counter}"
|
| 63 | copy_tree(source, target)
|
| 64 | return target
|
| 65 |
|
| 66 | def init(self, repo):
|
| 67 | (repo / ".fake").mkdir()
|
| 68 |
|
| 69 | def commit(self, repo, message):
|
| 70 | snap = self._snapshot(repo)
|
| 71 | if "commit_inject" in self.faults:
|
| 72 | (snap / "zz-injected-by-commit").write_bytes(b"x")
|
| 73 | if "mutate_source" in self.faults:
|
| 74 | (repo / "zz-written-by-commit").write_bytes(b"x")
|
| 75 | self.snapshots[repo] = snap
|
| 76 |
|
| 77 | def push(self, repo):
|
| 78 | self.remote = self.snapshots[repo]
|
| 79 |
|
| 80 | def clone(self, repo, destination):
|
| 81 | copy_tree(self.remote, destination)
|
| 82 | (destination / ".fake").mkdir()
|
| 83 | self.snapshots[destination] = self.remote
|
| 84 | if "clone_inject" in self.faults:
|
| 85 | (destination / "zz-injected-by-clone").write_bytes(b"x")
|
| 86 |
|
| 87 | def pull(self, repo):
|
| 88 | if "pull_stale" in self.faults:
|
| 89 | (repo / "zz-stale-after-pull").write_bytes(b"x")
|
| 90 | return
|
| 91 | clear(repo)
|
| 92 | copy_tree(self.remote, repo)
|
| 93 | self.snapshots[repo] = self.remote
|
| 94 |
|
| 95 | def status_clean(self, repo):
|
| 96 | if "dirty" in self.faults:
|
| 97 | raise matrix.LegFailure("status reports changes")
|
| 98 |
|
| 99 | def committed(self, repo):
|
| 100 | return matrix.fs_manifest(self.snapshots[repo]), {"head": str(self.snapshots[repo])}
|
| 101 |
|
| 102 |
|
| 103 | class BehaviourOracleTests(unittest.TestCase):
|
| 104 | @classmethod
|
| 105 | def setUpClass(cls):
|
| 106 | cls.caps = matrix.probe_capabilities(Path(tempfile.gettempdir()) / "oakbench-caps-probe")
|
| 107 |
|
| 108 | def runnable(self):
|
| 109 | for behaviour in matrix.BEHAVIOURS:
|
| 110 | if behaviour.gated or any(not self.caps.get(cap) for cap in behaviour.requires):
|
| 111 | continue
|
| 112 | yield behaviour
|
| 113 |
|
| 114 | def run_fake(self, behaviour, faults=()):
|
| 115 | with tempfile.TemporaryDirectory() as tmp:
|
| 116 | base = Path(tmp)
|
| 117 | (base / "work").mkdir()
|
| 118 | vcs = CopyVCS(base, faults)
|
| 119 | legs, committed = matrix.run_behaviour(vcs, behaviour, base / "work", {})
|
| 120 | return legs
|
| 121 |
|
| 122 | def test_every_behaviour_passes_a_correct_vcs(self):
|
| 123 | ran = 0
|
| 124 | for behaviour in self.runnable():
|
| 125 | with self.subTest(behaviour=behaviour.id):
|
| 126 | legs = self.run_fake(behaviour)
|
| 127 | self.assertTrue(legs)
|
| 128 | self.assertEqual({name: leg for name, leg in legs.items() if leg["status"] != "pass"}, {})
|
| 129 | if len(behaviour.steps) > 1:
|
| 130 | self.assertEqual(legs["pull_fs"]["status"], "pass")
|
| 131 | ran += 1
|
| 132 | self.assertGreater(ran, 40)
|
| 133 |
|
| 134 | def test_every_behaviour_detects_each_fault_at_its_leg(self):
|
| 135 | for behaviour in self.runnable():
|
| 136 | if behaviour.id in ("large_file_chunked_v1", "many_small_files_v1"):
|
| 137 | continue # same oracle path as the small behaviours; their copies are slow (covered by the pass test)
|
| 138 | multi = len(behaviour.steps) > 1
|
| 139 | cases = {"commit_inject": "commit_read@1" if multi else "commit_read",
|
| 140 | "mutate_source": "source_preserved@1" if multi else "source_preserved", "clone_inject": "clone_fs",
|
| 141 | "dirty": "clone_status_clean"}
|
| 142 | if multi:
|
| 143 | cases["pull_stale"] = "pull_fs"
|
| 144 | for fault, leg in cases.items():
|
| 145 | with self.subTest(behaviour=behaviour.id, fault=fault):
|
| 146 | legs = self.run_fake(behaviour, {fault})
|
| 147 | self.assertEqual(legs[leg]["status"], "fail", legs)
|
| 148 |
|
| 149 | def test_clone_fault_is_not_mistaken_for_committed_state(self):
|
| 150 | legs = self.run_fake(matrix.BY_ID["eol_crlf_v1"], {"clone_inject"})
|
| 151 | self.assertEqual(legs["clone_fs"]["status"], "fail")
|
| 152 | self.assertEqual(legs["clone_read"]["status"], "pass")
|
| 153 | self.assertEqual(legs["commit_read"]["status"], "pass")
|
| 154 |
|
| 155 |
|
| 156 | class ManifestOracleTests(unittest.TestCase):
|
| 157 | def test_each_loss_class_is_detected(self):
|
| 158 | with tempfile.TemporaryDirectory() as tmp:
|
| 159 | root = Path(tmp)
|
| 160 | (root / "f").write_bytes(b"payload")
|
| 161 | (root / "l").symlink_to("f")
|
| 162 | expected = matrix.fs_manifest(root)
|
| 163 | mutations = [
|
| 164 | lambda: (root / "f").write_bytes(b"payloae"),
|
| 165 | lambda: (root / "f").chmod(0o755),
|
| 166 | lambda: ((root / "l").unlink(), (root / "l").symlink_to("g")),
|
| 167 | lambda: ((root / "l").unlink(), (root / "l").write_bytes(b"f")),
|
| 168 | lambda: (root / "extra").write_bytes(b""),
|
| 169 | lambda: (root / "f").rename(root / "F2"),
|
| 170 | ]
|
| 171 | for mutate in mutations:
|
| 172 | mutate()
|
| 173 | self.assertIsNotNone(matrix.diff_manifests(expected, matrix.fs_manifest(root)))
|
| 174 | shutil.rmtree(root)
|
| 175 | root.mkdir()
|
| 176 | (root / "f").write_bytes(b"payload")
|
| 177 | (root / "f").chmod(0o644)
|
| 178 | (root / "l").symlink_to("f")
|
| 179 | self.assertIsNone(matrix.diff_manifests(expected, matrix.fs_manifest(root)))
|
| 180 |
|
| 181 | def test_empty_directories_are_observations_not_content(self):
|
| 182 | with tempfile.TemporaryDirectory() as tmp:
|
| 183 | root = Path(tmp)
|
| 184 | (root / "void" / "deeper").mkdir(parents=True)
|
| 185 | dirs = []
|
| 186 | self.assertEqual(matrix.fs_manifest(root, dirs), {})
|
| 187 | self.assertEqual(dirs, ["void", "void/deeper"])
|
| 188 |
|
| 189 | def test_inplace_edit_keeps_size_and_mtime(self):
|
| 190 | with tempfile.TemporaryDirectory() as tmp:
|
| 191 | root = Path(tmp)
|
| 192 | matrix.apply_op(root, ("write", "r", b"aaaa", False))
|
| 193 | before = (root / "r").stat()
|
| 194 | matrix.apply_op(root, ("inplace", "r", b"bbbb"))
|
| 195 | after = (root / "r").stat()
|
| 196 | self.assertEqual((before.st_size, before.st_mtime_ns), (after.st_size, after.st_mtime_ns))
|
| 197 | self.assertEqual((root / "r").read_bytes(), b"bbbb")
|
| 198 |
|
| 199 | def test_sparse_op_writes_declared_size_and_tail(self):
|
| 200 | with tempfile.TemporaryDirectory() as tmp:
|
| 201 | root = Path(tmp)
|
| 202 | matrix.apply_op(root, ("sparse", "s", 1 << 20, b"tail\n"))
|
| 203 | data = (root / "s").read_bytes()
|
| 204 | self.assertEqual(len(data), 1 << 20)
|
| 205 | self.assertTrue(data.endswith(b"tail\n"))
|
| 206 |
|
| 207 |
|
| 208 | class LoopbackServerTests(unittest.TestCase):
|
| 209 | @unittest.skipUnless(os.name == "posix", "process groups required")
|
| 210 | def test_stop_reaches_grandchild_behind_a_wrapper_script(self):
|
| 211 | with tempfile.TemporaryDirectory() as tmp:
|
| 212 | root = Path(tmp)
|
| 213 | pidfile = root / "grandchild.pid"
|
| 214 | listener = root / "listen.py"
|
| 215 | listener.write_text(
|
| 216 | "import socket, sys, time, os\n"
|
| 217 | "port = int(sys.argv[sys.argv.index('--port') + 1])\n"
|
| 218 | f"open({str(pidfile)!r}, 'w').write(str(os.getpid()))\n"
|
| 219 | "s = socket.socket(); s.bind(('127.0.0.1', port)); s.listen()\n"
|
| 220 | "time.sleep(300)\n")
|
| 221 | wrapper = root / "fake-oak"
|
| 222 | # No exec: the listener is a grandchild of the process Popen started.
|
| 223 | wrapper.write_text(f"#!/bin/sh\n{sys.executable} {listener} \"$@\" &\nwait\n")
|
| 224 | wrapper.chmod(0o755)
|
| 225 | with matrix.LoopbackOakServer(str(wrapper), root / "data", dict(os.environ)):
|
| 226 | grandchild = int(pidfile.read_text())
|
| 227 | os.kill(grandchild, 0)
|
| 228 | for _ in range(50):
|
| 229 | try:
|
| 230 | os.kill(grandchild, 0)
|
| 231 | except ProcessLookupError:
|
| 232 | break
|
| 233 | import time
|
| 234 | time.sleep(0.1)
|
| 235 | else:
|
| 236 | self.fail("grandchild server survived LoopbackOakServer exit")
|
| 237 |
|
| 238 |
|
| 239 | class CatalogTests(unittest.TestCase):
|
| 240 | def test_ids_are_versioned_unique_and_disjoint_from_legacy(self):
|
| 241 | ids = [b.id for b in matrix.BEHAVIOURS]
|
| 242 | self.assertEqual(len(ids), len(set(ids)))
|
| 243 | self.assertTrue(all(re.fullmatch(r"[a-z0-9_]+_v\d+", i) for i in ids))
|
| 244 | self.assertFalse(set(ids) & set(SCENARIOS))
|
| 245 | self.assertGreaterEqual(len(ids), 50)
|
| 246 |
|
| 247 | def test_required_coverage_groups(self):
|
| 248 | groups = {b.group for b in matrix.BEHAVIOURS}
|
| 249 | self.assertEqual(groups, {"paths", "modes", "symlinks", "file_dir", "empty", "content", "history"})
|
| 250 | for needed in ("path_tab_v1", "path_unicode_nfd_v1", "path_case_only_rename_v1", "mode_exec_remove_v1",
|
| 251 | "symlink_absolute_v1", "file_to_symlink_v1", "dir_to_file_v1", "empty_dir_semantics_v1",
|
| 252 | "huge_file_sparse_v1", "binary_detection_v1", "eol_mixed_v1", "delete_readd_v1",
|
| 253 | "many_small_files_v1"):
|
| 254 | self.assertIn(needed, matrix.BY_ID)
|
| 255 | self.assertTrue(matrix.BY_ID["huge_file_sparse_v1"].gated)
|
| 256 |
|
| 257 | def test_prior_art_cites_known_git_test_ids_only(self):
|
| 258 | for behaviour in matrix.BEHAVIOURS:
|
| 259 | for citation in behaviour.prior_art:
|
| 260 | match = re.match(r"git (t\d{4})-", citation)
|
| 261 | if match:
|
| 262 | self.assertIn(match.group(1), GIT_TEST_IDS, citation)
|
| 263 |
|
| 264 | def test_expected_failures_name_real_behaviours_legs_and_tickets(self):
|
| 265 | expected = conformance_gate.load_expected()
|
| 266 | known = set(matrix.BY_ID) | set(SCENARIOS)
|
| 267 | legs = set(matrix.LEGS) | {"commit_push", "behaviour"}
|
| 268 | for behaviour, entry in expected["entries"].items():
|
| 269 | self.assertIn(behaviour, known)
|
| 270 | for leg in entry["failed_legs"]:
|
| 271 | self.assertIn(leg.split("@")[0], legs)
|
| 272 | tickets = {b: e["ticket"] for b, e in expected["entries"].items()}
|
| 273 | self.assertEqual(tickets["path_tab_v1"], "fb-531")
|
| 274 | self.assertEqual(tickets["path_roundtrip_v1"], "fb-531")
|
| 275 | self.assertEqual(tickets["symlink_to_dir_real_v1"], "fb-542")
|
| 276 | self.assertEqual(expected["entries"]["path_tab_v1"]["outcome"], "refused")
|
| 277 | for skip in expected["coverage"]["darwin-ci-ni"]["expected_skips"]:
|
| 278 | self.assertIn(skip, matrix.BY_ID)
|
| 279 |
|
| 280 |
|
| 281 | @unittest.skipUnless(shutil.which("git"), "Git required")
|
| 282 | class RealGitMatrixTests(unittest.TestCase):
|
| 283 | def test_git_passes_a_representative_subset(self):
|
| 284 | subset = ["path_tab_v1", "mode_exec_add_v1", "file_to_dir_v1", "eol_crlf_v1", "delete_all_v1"]
|
| 285 | with tempfile.TemporaryDirectory() as tmp:
|
| 286 | rows = matrix.run_matrix(Path(tmp), {"git": "git"}, subset)
|
| 287 | for row in rows:
|
| 288 | with self.subTest(behaviour=row["scenario"]):
|
| 289 | if row["returncode"] == 77:
|
| 290 | continue
|
| 291 | self.assertEqual(row["returncode"], 0, row.get("failure"))
|
| 292 | multi = len(matrix.BY_ID[row["scenario"]].steps) > 1
|
| 293 | self.assertEqual(row["legs"]["commit_read@1" if multi else "commit_read"]["status"], "pass")
|
| 294 |
|
| 295 |
|
| 296 | @unittest.skipUnless(shutil.which("oak") and shutil.which("git"), "Oak and Git required")
|
| 297 | class RealOakMatrixTests(unittest.TestCase):
|
| 298 | def test_oak_rows_carry_independent_reader_and_git_differential(self):
|
| 299 | with tempfile.TemporaryDirectory() as tmp:
|
| 300 | rows = matrix.run_matrix(Path(tmp), {"git": "git", "oak": "oak"}, ["eol_mixed_v1", "path_tab_v1"])
|
| 301 | oak = {row["scenario"]: row for row in rows if row["subject"] == "oak"}
|
| 302 | mixed = oak["eol_mixed_v1"]
|
| 303 | self.assertEqual(mixed["returncode"], 0, mixed.get("failure"))
|
| 304 | self.assertEqual(mixed["observations"]["committed_reader"]["reader"], "independent_oak_sqlite_reader")
|
| 305 | self.assertEqual(mixed["git_differential"], "identical")
|
| 306 | # fb-531: rejected at commit; if Oak fixes it this flips and the gate reports FIXED.
|
| 307 | self.assertIn(oak["path_tab_v1"]["returncode"], (0, 1))
|
| 308 | if oak["path_tab_v1"]["returncode"] == 1:
|
| 309 | self.assertEqual(oak["path_tab_v1"]["failed_legs"], ["commit_push"])
|
| 310 |
|
| 311 |
|
| 312 | if __name__ == "__main__":
|
| 313 | unittest.main()
|