| 1 | """Independent behavioral conformance probes, never performance evidence.
|
| 2 |
|
| 3 | See docs/vcs-conformance.md for the contract and upstream behavioral indexes.
|
| 4 | """
|
| 5 | from __future__ import annotations
|
| 6 |
|
| 7 | import hashlib
|
| 8 | import os
|
| 9 | from pathlib import Path
|
| 10 | import shutil
|
| 11 | import stat
|
| 12 | import subprocess
|
| 13 | import tempfile
|
| 14 |
|
| 15 | from oakbench.environment import base_env
|
| 16 |
|
| 17 | SCENARIOS = (
|
| 18 | "path_roundtrip_v1", "byte_roundtrip_v1", "mode_symlink_roundtrip_v1",
|
| 19 | "scoped_restore_preservation_v1", "failed_restore_preservation_v1",
|
| 20 | )
|
| 21 |
|
| 22 |
|
| 23 | class CapabilityUnavailable(Exception):
|
| 24 | pass
|
| 25 |
|
| 26 |
|
| 27 | def tree_manifest(root: Path) -> dict:
|
| 28 | """Exact path/type/content projection; never follows directory symlinks.
|
| 29 |
|
| 30 | VCS directories at the repository root are excluded. Empty directories and
|
| 31 | non-executable permission bits are outside the portable VCS contract.
|
| 32 | """
|
| 33 | result = {}
|
| 34 | def visit(directory: Path) -> None:
|
| 35 | for path in sorted(directory.iterdir()):
|
| 36 | if directory == root and path.name in {".git", ".oak"}:
|
| 37 | continue
|
| 38 | mode = path.lstat().st_mode
|
| 39 | name = path.relative_to(root).as_posix()
|
| 40 | if stat.S_ISLNK(mode):
|
| 41 | result[name] = {"kind": "symlink", "target": os.readlink(path)}
|
| 42 | elif stat.S_ISDIR(mode):
|
| 43 | visit(path)
|
| 44 | elif stat.S_ISREG(mode):
|
| 45 | data = path.read_bytes()
|
| 46 | result[name] = {"kind": "file", "size": len(data),
|
| 47 | "sha256": hashlib.sha256(data).hexdigest(),
|
| 48 | "executable": bool(mode & 0o111)}
|
| 49 | else:
|
| 50 | raise AssertionError(f"unexpected file type: {name!r}")
|
| 51 | visit(root)
|
| 52 | return result
|
| 53 |
|
| 54 |
|
| 55 | def assert_state(expected: dict, actual: dict) -> None:
|
| 56 | if expected != actual:
|
| 57 | missing = sorted(expected.keys() - actual.keys())
|
| 58 | extra = sorted(actual.keys() - expected.keys())
|
| 59 | changed = sorted(k for k in expected.keys() & actual.keys() if expected[k] != actual[k])
|
| 60 | raise AssertionError(f"state mismatch: missing={missing!r}, extra={extra!r}, changed={changed!r}")
|
| 61 |
|
| 62 |
|
| 63 | class Repository:
|
| 64 | def __init__(self, root: Path, kind: str, binary: str, home: Path):
|
| 65 | self.root, self.kind, self.binary = root, kind, binary
|
| 66 | self.env = base_env(home=home)
|
| 67 | self.commands = []
|
| 68 |
|
| 69 | def run(self, *args: str, success: bool = True) -> subprocess.CompletedProcess:
|
| 70 | process = subprocess.run([self.binary, *args], cwd=self.root, env=self.env,
|
| 71 | stdin=subprocess.DEVNULL, capture_output=True, timeout=30)
|
| 72 | self.commands.append({"argv": list(args), "returncode": process.returncode,
|
| 73 | "stdout": process.stdout.decode("utf-8", "backslashreplace"),
|
| 74 | "stderr": process.stderr.decode("utf-8", "backslashreplace")})
|
| 75 | if success and process.returncode:
|
| 76 | raise AssertionError(f"command {args!r} failed ({process.returncode}): {process.stderr!r}")
|
| 77 | return process
|
| 78 |
|
| 79 | def initialize(self):
|
| 80 | self.run("init")
|
| 81 | if self.kind == "git":
|
| 82 | self.run("config", "core.autocrlf", "false")
|
| 83 | self.run("config", "core.filemode", "true")
|
| 84 |
|
| 85 | def commit(self):
|
| 86 | if self.kind == "git":
|
| 87 | self.run("add", "--all")
|
| 88 | self.run("commit", "--no-verify", "-m", "independent conformance fixture v1")
|
| 89 | else:
|
| 90 | self.run("commit", "--no-verify")
|
| 91 |
|
| 92 | def identity(self):
|
| 93 | if self.kind == "git":
|
| 94 | return {"head": self.run("rev-parse", "HEAD").stdout.hex(),
|
| 95 | "branch": self.run("symbolic-ref", "HEAD").stdout.hex(),
|
| 96 | "index": self.run("ls-files", "--stage", "-z").stdout.hex()}
|
| 97 | # Read-only logical identity; storage bookkeeping is not an atomicity oracle.
|
| 98 | import json
|
| 99 | status = json.loads(self.run("status", "--json").stdout)
|
| 100 | return {"head": status["head"], "branch": status["branch"]}
|
| 101 |
|
| 102 | def restore(self, paths: list[str], invalid_source: bool = False):
|
| 103 | source = "conformance-definitely-missing-source-v1" if invalid_source else "HEAD"
|
| 104 | if self.kind == "git":
|
| 105 | return self.run("restore", "--source", source, "--", *paths,
|
| 106 | success=not invalid_source)
|
| 107 | source_args = ["--source", source] if invalid_source else []
|
| 108 | return self.run("restore", "--force", *source_args, "--", *paths,
|
| 109 | success=not invalid_source)
|
| 110 |
|
| 111 |
|
| 112 | def fixture(root: Path, scenario: str) -> None:
|
| 113 | files = {"target.txt": b"baseline payload\n", "other.txt": b"separate payload\n"}
|
| 114 | if scenario == "path_roundtrip_v1":
|
| 115 | files = {name: (str(i) + "\nindependent path sentinel\n").encode()
|
| 116 | for i, name in enumerate(("space name", "tab\tname", "line\nbreak",
|
| 117 | "quote\"name", "-leading", "unicode-λ", "nested/leaf"))}
|
| 118 | elif scenario == "byte_roundtrip_v1":
|
| 119 | files = {"empty": b"", "binary": bytes(range(256)) * 17,
|
| 120 | "mixed-newlines": b"first\r\nsecond\nthird\rfinal", "no-final-lf": b"tail"}
|
| 121 | for name, payload in files.items():
|
| 122 | path = root / name
|
| 123 | path.parent.mkdir(parents=True, exist_ok=True)
|
| 124 | path.write_bytes(payload)
|
| 125 | if scenario == "mode_symlink_roundtrip_v1":
|
| 126 | if os.name != "posix":
|
| 127 | raise CapabilityUnavailable("POSIX executable-bit contract unavailable")
|
| 128 | (root / "target.txt").chmod(0o755)
|
| 129 | if not ((root / "target.txt").stat().st_mode & 0o111):
|
| 130 | raise CapabilityUnavailable("filesystem does not retain executable bit")
|
| 131 | try:
|
| 132 | (root / "link").symlink_to("target.txt")
|
| 133 | (root / "dangling").symlink_to("absent-target")
|
| 134 | except (NotImplementedError, OSError) as exc:
|
| 135 | raise CapabilityUnavailable(f"filesystem symlink creation unavailable: {exc}") from exc
|
| 136 |
|
| 137 |
|
| 138 | def exercise(repo: Repository, scenario: str) -> None:
|
| 139 | fixture(repo.root, scenario)
|
| 140 | expected = tree_manifest(repo.root)
|
| 141 | repo.commit()
|
| 142 | identity = repo.identity()
|
| 143 | if scenario.endswith("roundtrip_v1"):
|
| 144 | # Remove every fixture entry: a no-op restore cannot pass this oracle.
|
| 145 | for name in expected:
|
| 146 | (repo.root / name).unlink()
|
| 147 | repo.restore(list(expected))
|
| 148 | assert_state(expected, tree_manifest(repo.root))
|
| 149 | assert_state(identity, repo.identity())
|
| 150 | return
|
| 151 | (repo.root / "target.txt").write_bytes(b"discard only this change\n")
|
| 152 | (repo.root / "other.txt").write_bytes(b"preserve my unrelated work\x00\n")
|
| 153 | (repo.root / "untracked.txt").write_bytes(b"not yet checkpointed\n")
|
| 154 | before = tree_manifest(repo.root)
|
| 155 | identity = repo.identity()
|
| 156 | if scenario == "scoped_restore_preservation_v1":
|
| 157 | repo.restore(["target.txt"])
|
| 158 | before["target.txt"] = expected["target.txt"]
|
| 159 | else:
|
| 160 | failed = repo.restore(["target.txt"], invalid_source=True)
|
| 161 | if failed.returncode <= 0:
|
| 162 | raise AssertionError("invalid source must fail normally, not succeed or terminate by signal")
|
| 163 | assert_state(before, tree_manifest(repo.root))
|
| 164 | assert_state(identity, repo.identity())
|
| 165 |
|
| 166 |
|
| 167 | def run_scenario(workdir: Path, kind: str, binary: str, scenario: str) -> dict:
|
| 168 | if kind not in {"git", "oak"} or scenario not in SCENARIOS:
|
| 169 | raise ValueError("unsupported subject or scenario")
|
| 170 | row = {"conformance_schema_version": 1, "evidence_class": "diagnostic_conformance",
|
| 171 | "subject": kind, "scenario": scenario, "fixture_version": 1,
|
| 172 | "wall_ms": None, "tokens": None, "integrity_ok": None,
|
| 173 | "oracle": "exact_tree_and_logical_identity_v1", "commands": []}
|
| 174 | resolved = shutil.which(binary)
|
| 175 | if resolved is None:
|
| 176 | return {**row, "returncode": 77, "skip_reason": f"binary unavailable: {binary}"}
|
| 177 | resolved = str(Path(resolved).resolve())
|
| 178 | row["binary"] = resolved
|
| 179 | row["binary_sha256"] = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
|
| 180 | workdir.mkdir(parents=True, exist_ok=True)
|
| 181 | with tempfile.TemporaryDirectory(prefix="conformance-", dir=workdir) as temporary:
|
| 182 | root = Path(temporary) / "repo"
|
| 183 | root.mkdir()
|
| 184 | repo = Repository(root, kind, resolved, Path(temporary) / "home")
|
| 185 | row["commands"] = repo.commands
|
| 186 | try:
|
| 187 | row["version"] = repo.run("--version").stdout.decode("utf-8", "replace").strip()
|
| 188 | repo.initialize()
|
| 189 | exercise(repo, scenario)
|
| 190 | except CapabilityUnavailable as exc:
|
| 191 | row.update(returncode=77, skip_reason=str(exc))
|
| 192 | except (AssertionError, OSError, ValueError, KeyError, subprocess.TimeoutExpired) as exc:
|
| 193 | row.update(returncode=1, integrity_ok=False, failure=str(exc))
|
| 194 | else:
|
| 195 | row.update(returncode=0, integrity_ok=True)
|
| 196 | return row
|