Viewing files at commit ebd567552629Commit detailsView latest

scripts/oakbench/conformance.py 203 lines · 9.1 KB · python
1
"""Independent behavioral conformance probes, never performance evidence.
2
3
See docs/vcs-conformance.md for the contract and upstream behavioral indexes.
4
"""
5
from __future__ import annotations
6
7
import hashlib
8
import os
9
from pathlib import Path
10
import shutil
11
import stat
12
import subprocess
13
import tempfile
14
15
from oakbench.environment import base_env
16
17
SCENARIOS = (
18
    "path_roundtrip_v1", "byte_roundtrip_v1", "mode_symlink_roundtrip_v1",
19
    "scoped_restore_preservation_v1", "failed_restore_preservation_v1",
20
)
21
22
23
class CapabilityUnavailable(Exception):
24
    pass
25
26
27
def tree_manifest(root: Path) -> dict:
28
    """Exact path/type/content projection; never follows directory symlinks.
29
30
    VCS directories at the repository root are excluded. Empty directories and
31
    non-executable permission bits are outside the portable VCS contract.
32
    """
33
    result = {}
34
    def visit(directory: Path) -> None:
35
        for path in sorted(directory.iterdir()):
36
            if directory == root and path.name in {".git", ".oak"}:
37
                continue
38
            mode = path.lstat().st_mode
39
            name = path.relative_to(root).as_posix()
40
            if stat.S_ISLNK(mode):
41
                result[name] = {"kind": "symlink", "target": os.readlink(path)}
42
            elif stat.S_ISDIR(mode):
43
                visit(path)
44
            elif stat.S_ISREG(mode):
45
                data = path.read_bytes()
46
                result[name] = {"kind": "file", "size": len(data),
47
                                "sha256": hashlib.sha256(data).hexdigest(),
48
                                "executable": bool(mode & 0o111)}
49
            else:
50
                raise AssertionError(f"unexpected file type: {name!r}")
51
    visit(root)
52
    return result
53
54
55
def assert_state(expected: dict, actual: dict) -> None:
56
    if expected != actual:
57
        missing = sorted(expected.keys() - actual.keys())
58
        extra = sorted(actual.keys() - expected.keys())
59
        changed = sorted(k for k in expected.keys() & actual.keys() if expected[k] != actual[k])
60
        raise AssertionError(f"state mismatch: missing={missing!r}, extra={extra!r}, changed={changed!r}")
61
62
63
class Repository:
64
    def __init__(self, root: Path, kind: str, binary: str, home: Path):
65
        self.root, self.kind, self.binary = root, kind, binary
66
        self.env = base_env(home=home)
67
        # Inherited Git plumbing can silently redirect commands to another repo.
68
        for key in list(self.env):
69
            if key.startswith("GIT_") and key not in {
70
                "GIT_AUTHOR_NAME", "GIT_AUTHOR_EMAIL", "GIT_COMMITTER_NAME",
71
                "GIT_COMMITTER_EMAIL", "GIT_CONFIG_GLOBAL", "GIT_CONFIG_NOSYSTEM",
72
            }:
73
                self.env.pop(key)
74
        self.commands = []
75
76
    def run(self, *args: str, success: bool = True) -> subprocess.CompletedProcess:
77
        process = subprocess.run([self.binary, *args], cwd=self.root, env=self.env,
78
                                 stdin=subprocess.DEVNULL, capture_output=True, timeout=30)
79
        self.commands.append({"argv": list(args), "returncode": process.returncode,
80
                              "stdout": process.stdout.decode("utf-8", "backslashreplace"),
81
                              "stderr": process.stderr.decode("utf-8", "backslashreplace")})
82
        if success and process.returncode:
83
            raise AssertionError(f"command {args!r} failed ({process.returncode}): {process.stderr!r}")
84
        return process
85
86
    def initialize(self):
87
        self.run("init")
88
        if self.kind == "git":
89
            self.run("config", "core.autocrlf", "false")
90
            self.run("config", "core.filemode", "true")
91
92
    def commit(self):
93
        if self.kind == "git":
94
            self.run("add", "--all")
95
            self.run("commit", "--no-verify", "-m", "independent conformance fixture v1")
96
        else:
97
            self.run("commit", "--no-verify")
98
99
    def identity(self):
100
        if self.kind == "git":
101
            return {"head": self.run("rev-parse", "HEAD").stdout.hex(),
102
                    "branch": self.run("symbolic-ref", "HEAD").stdout.hex(),
103
                    "index": self.run("ls-files", "--stage", "-z").stdout.hex()}
104
        # Read-only logical identity; storage bookkeeping is not an atomicity oracle.
105
        import json
106
        status = json.loads(self.run("status", "--json").stdout)
107
        return {"head": status["head"], "branch": status["branch"]}
108
109
    def restore(self, paths: list[str], invalid_source: bool = False):
110
        source = "conformance-definitely-missing-source-v1" if invalid_source else "HEAD"
111
        if self.kind == "git":
112
            return self.run("restore", "--source", source, "--", *paths,
113
                            success=not invalid_source)
114
        source_args = ["--source", source] if invalid_source else []
115
        return self.run("restore", "--force", *source_args, "--", *paths,
116
                        success=not invalid_source)
117
118
119
def fixture(root: Path, scenario: str) -> None:
120
    files = {"target.txt": b"baseline payload\n", "other.txt": b"separate payload\n"}
121
    if scenario == "path_roundtrip_v1":
122
        files = {name: (str(i) + "\nindependent path sentinel\n").encode()
123
                 for i, name in enumerate(("space name", "tab\tname", "line\nbreak",
124
                                           "quote\"name", "-leading", "unicode-λ", "nested/leaf"))}
125
    elif scenario == "byte_roundtrip_v1":
126
        files = {"empty": b"", "binary": bytes(range(256)) * 17,
127
                 "mixed-newlines": b"first\r\nsecond\nthird\rfinal", "no-final-lf": b"tail"}
128
    for name, payload in files.items():
129
        path = root / name
130
        path.parent.mkdir(parents=True, exist_ok=True)
131
        path.write_bytes(payload)
132
    if scenario == "mode_symlink_roundtrip_v1":
133
        if os.name != "posix":
134
            raise CapabilityUnavailable("POSIX executable-bit contract unavailable")
135
        (root / "target.txt").chmod(0o755)
136
        if not ((root / "target.txt").stat().st_mode & 0o111):
137
            raise CapabilityUnavailable("filesystem does not retain executable bit")
138
        try:
139
            (root / "link").symlink_to("target.txt")
140
            (root / "dangling").symlink_to("absent-target")
141
        except (NotImplementedError, OSError) as exc:
142
            raise CapabilityUnavailable(f"filesystem symlink creation unavailable: {exc}") from exc
143
144
145
def exercise(repo: Repository, scenario: str) -> None:
146
    fixture(repo.root, scenario)
147
    expected = tree_manifest(repo.root)
148
    repo.commit()
149
    identity = repo.identity()
150
    if scenario.endswith("roundtrip_v1"):
151
        # Remove every fixture entry: a no-op restore cannot pass this oracle.
152
        for name in expected:
153
            (repo.root / name).unlink()
154
        repo.restore(list(expected))
155
        assert_state(expected, tree_manifest(repo.root))
156
        assert_state(identity, repo.identity())
157
        return
158
    (repo.root / "target.txt").write_bytes(b"discard only this change\n")
159
    (repo.root / "other.txt").write_bytes(b"preserve my unrelated work\x00\n")
160
    (repo.root / "untracked.txt").write_bytes(b"not yet checkpointed\n")
161
    before = tree_manifest(repo.root)
162
    identity = repo.identity()
163
    if scenario == "scoped_restore_preservation_v1":
164
        repo.restore(["target.txt"])
165
        before["target.txt"] = expected["target.txt"]
166
    else:
167
        failed = repo.restore(["target.txt"], invalid_source=True)
168
        if failed.returncode <= 0:
169
            raise AssertionError("invalid source must fail normally, not succeed or terminate by signal")
170
    assert_state(before, tree_manifest(repo.root))
171
    assert_state(identity, repo.identity())
172
173
174
def run_scenario(workdir: Path, kind: str, binary: str, scenario: str) -> dict:
175
    if kind not in {"git", "oak"} or scenario not in SCENARIOS:
176
        raise ValueError("unsupported subject or scenario")
177
    row = {"conformance_schema_version": 1, "evidence_class": "diagnostic_conformance",
178
           "subject": kind, "scenario": scenario, "fixture_version": 1,
179
           "wall_ms": None, "tokens": None, "integrity_ok": None,
180
           "oracle": "exact_tree_and_logical_identity_v1", "commands": []}
181
    resolved = shutil.which(binary)
182
    if resolved is None:
183
        return {**row, "returncode": 77, "skip_reason": f"binary unavailable: {binary}"}
184
    resolved = str(Path(resolved).resolve())
185
    row["binary"] = resolved
186
    row["binary_sha256"] = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
187
    workdir.mkdir(parents=True, exist_ok=True)
188
    with tempfile.TemporaryDirectory(prefix="conformance-", dir=workdir) as temporary:
189
        root = Path(temporary) / "repo"
190
        root.mkdir()
191
        repo = Repository(root, kind, resolved, Path(temporary) / "home")
192
        row["commands"] = repo.commands
193
        try:
194
            row["version"] = repo.run("--version").stdout.decode("utf-8", "replace").strip()
195
            repo.initialize()
196
            exercise(repo, scenario)
197
        except CapabilityUnavailable as exc:
198
            row.update(returncode=77, skip_reason=str(exc))
199
        except (AssertionError, OSError, ValueError, KeyError, subprocess.TimeoutExpired) as exc:
200
            row.update(returncode=1, integrity_ok=False, failure=str(exc))
201
        else:
202
            row.update(returncode=0, integrity_ok=True)
203
    return row