Viewing files at commit 538751fb07e6Commit detailsView latest

scripts/oakbench/conformance.py 196 lines · 8.7 KB · python
1
"""Independent behavioral conformance probes, never performance evidence.
2
3
See docs/vcs-conformance.md for the contract and upstream behavioral indexes.
4
"""
5
from __future__ import annotations
6
7
import hashlib
8
import os
9
from pathlib import Path
10
import shutil
11
import stat
12
import subprocess
13
import tempfile
14
15
from oakbench.environment import base_env
16
17
SCENARIOS = (
18
    "path_roundtrip_v1", "byte_roundtrip_v1", "mode_symlink_roundtrip_v1",
19
    "scoped_restore_preservation_v1", "failed_restore_preservation_v1",
20
)
21
22
23
class CapabilityUnavailable(Exception):
24
    pass
25
26
27
def tree_manifest(root: Path) -> dict:
28
    """Exact path/type/content projection; never follows directory symlinks.
29
30
    VCS directories at the repository root are excluded. Empty directories and
31
    non-executable permission bits are outside the portable VCS contract.
32
    """
33
    result = {}
34
    def visit(directory: Path) -> None:
35
        for path in sorted(directory.iterdir()):
36
            if directory == root and path.name in {".git", ".oak"}:
37
                continue
38
            mode = path.lstat().st_mode
39
            name = path.relative_to(root).as_posix()
40
            if stat.S_ISLNK(mode):
41
                result[name] = {"kind": "symlink", "target": os.readlink(path)}
42
            elif stat.S_ISDIR(mode):
43
                visit(path)
44
            elif stat.S_ISREG(mode):
45
                data = path.read_bytes()
46
                result[name] = {"kind": "file", "size": len(data),
47
                                "sha256": hashlib.sha256(data).hexdigest(),
48
                                "executable": bool(mode & 0o111)}
49
            else:
50
                raise AssertionError(f"unexpected file type: {name!r}")
51
    visit(root)
52
    return result
53
54
55
def assert_state(expected: dict, actual: dict) -> None:
56
    if expected != actual:
57
        missing = sorted(expected.keys() - actual.keys())
58
        extra = sorted(actual.keys() - expected.keys())
59
        changed = sorted(k for k in expected.keys() & actual.keys() if expected[k] != actual[k])
60
        raise AssertionError(f"state mismatch: missing={missing!r}, extra={extra!r}, changed={changed!r}")
61
62
63
class Repository:
64
    def __init__(self, root: Path, kind: str, binary: str, home: Path):
65
        self.root, self.kind, self.binary = root, kind, binary
66
        self.env = base_env(home=home)
67
        self.commands = []
68
69
    def run(self, *args: str, success: bool = True) -> subprocess.CompletedProcess:
70
        process = subprocess.run([self.binary, *args], cwd=self.root, env=self.env,
71
                                 stdin=subprocess.DEVNULL, capture_output=True, timeout=30)
72
        self.commands.append({"argv": list(args), "returncode": process.returncode,
73
                              "stdout": process.stdout.decode("utf-8", "backslashreplace"),
74
                              "stderr": process.stderr.decode("utf-8", "backslashreplace")})
75
        if success and process.returncode:
76
            raise AssertionError(f"command {args!r} failed ({process.returncode}): {process.stderr!r}")
77
        return process
78
79
    def initialize(self):
80
        self.run("init")
81
        if self.kind == "git":
82
            self.run("config", "core.autocrlf", "false")
83
            self.run("config", "core.filemode", "true")
84
85
    def commit(self):
86
        if self.kind == "git":
87
            self.run("add", "--all")
88
            self.run("commit", "--no-verify", "-m", "independent conformance fixture v1")
89
        else:
90
            self.run("commit", "--no-verify")
91
92
    def identity(self):
93
        if self.kind == "git":
94
            return {"head": self.run("rev-parse", "HEAD").stdout.hex(),
95
                    "branch": self.run("symbolic-ref", "HEAD").stdout.hex(),
96
                    "index": self.run("ls-files", "--stage", "-z").stdout.hex()}
97
        # Read-only logical identity; storage bookkeeping is not an atomicity oracle.
98
        import json
99
        status = json.loads(self.run("status", "--json").stdout)
100
        return {"head": status["head"], "branch": status["branch"]}
101
102
    def restore(self, paths: list[str], invalid_source: bool = False):
103
        source = "conformance-definitely-missing-source-v1" if invalid_source else "HEAD"
104
        if self.kind == "git":
105
            return self.run("restore", "--source", source, "--", *paths,
106
                            success=not invalid_source)
107
        source_args = ["--source", source] if invalid_source else []
108
        return self.run("restore", "--force", *source_args, "--", *paths,
109
                        success=not invalid_source)
110
111
112
def fixture(root: Path, scenario: str) -> None:
113
    files = {"target.txt": b"baseline payload\n", "other.txt": b"separate payload\n"}
114
    if scenario == "path_roundtrip_v1":
115
        files = {name: (str(i) + "\nindependent path sentinel\n").encode()
116
                 for i, name in enumerate(("space name", "tab\tname", "line\nbreak",
117
                                           "quote\"name", "-leading", "unicode-λ", "nested/leaf"))}
118
    elif scenario == "byte_roundtrip_v1":
119
        files = {"empty": b"", "binary": bytes(range(256)) * 17,
120
                 "mixed-newlines": b"first\r\nsecond\nthird\rfinal", "no-final-lf": b"tail"}
121
    for name, payload in files.items():
122
        path = root / name
123
        path.parent.mkdir(parents=True, exist_ok=True)
124
        path.write_bytes(payload)
125
    if scenario == "mode_symlink_roundtrip_v1":
126
        if os.name != "posix":
127
            raise CapabilityUnavailable("POSIX executable-bit contract unavailable")
128
        (root / "target.txt").chmod(0o755)
129
        if not ((root / "target.txt").stat().st_mode & 0o111):
130
            raise CapabilityUnavailable("filesystem does not retain executable bit")
131
        try:
132
            (root / "link").symlink_to("target.txt")
133
            (root / "dangling").symlink_to("absent-target")
134
        except (NotImplementedError, OSError) as exc:
135
            raise CapabilityUnavailable(f"filesystem symlink creation unavailable: {exc}") from exc
136
137
138
def exercise(repo: Repository, scenario: str) -> None:
139
    fixture(repo.root, scenario)
140
    expected = tree_manifest(repo.root)
141
    repo.commit()
142
    identity = repo.identity()
143
    if scenario.endswith("roundtrip_v1"):
144
        # Remove every fixture entry: a no-op restore cannot pass this oracle.
145
        for name in expected:
146
            (repo.root / name).unlink()
147
        repo.restore(list(expected))
148
        assert_state(expected, tree_manifest(repo.root))
149
        assert_state(identity, repo.identity())
150
        return
151
    (repo.root / "target.txt").write_bytes(b"discard only this change\n")
152
    (repo.root / "other.txt").write_bytes(b"preserve my unrelated work\x00\n")
153
    (repo.root / "untracked.txt").write_bytes(b"not yet checkpointed\n")
154
    before = tree_manifest(repo.root)
155
    identity = repo.identity()
156
    if scenario == "scoped_restore_preservation_v1":
157
        repo.restore(["target.txt"])
158
        before["target.txt"] = expected["target.txt"]
159
    else:
160
        failed = repo.restore(["target.txt"], invalid_source=True)
161
        if failed.returncode <= 0:
162
            raise AssertionError("invalid source must fail normally, not succeed or terminate by signal")
163
    assert_state(before, tree_manifest(repo.root))
164
    assert_state(identity, repo.identity())
165
166
167
def run_scenario(workdir: Path, kind: str, binary: str, scenario: str) -> dict:
168
    if kind not in {"git", "oak"} or scenario not in SCENARIOS:
169
        raise ValueError("unsupported subject or scenario")
170
    row = {"conformance_schema_version": 1, "evidence_class": "diagnostic_conformance",
171
           "subject": kind, "scenario": scenario, "fixture_version": 1,
172
           "wall_ms": None, "tokens": None, "integrity_ok": None,
173
           "oracle": "exact_tree_and_logical_identity_v1", "commands": []}
174
    resolved = shutil.which(binary)
175
    if resolved is None:
176
        return {**row, "returncode": 77, "skip_reason": f"binary unavailable: {binary}"}
177
    resolved = str(Path(resolved).resolve())
178
    row["binary"] = resolved
179
    row["binary_sha256"] = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
180
    workdir.mkdir(parents=True, exist_ok=True)
181
    with tempfile.TemporaryDirectory(prefix="conformance-", dir=workdir) as temporary:
182
        root = Path(temporary) / "repo"
183
        root.mkdir()
184
        repo = Repository(root, kind, resolved, Path(temporary) / "home")
185
        row["commands"] = repo.commands
186
        try:
187
            row["version"] = repo.run("--version").stdout.decode("utf-8", "replace").strip()
188
            repo.initialize()
189
            exercise(repo, scenario)
190
        except CapabilityUnavailable as exc:
191
            row.update(returncode=77, skip_reason=str(exc))
192
        except (AssertionError, OSError, ValueError, KeyError, subprocess.TimeoutExpired) as exc:
193
            row.update(returncode=1, integrity_ok=False, failure=str(exc))
194
        else:
195
            row.update(returncode=0, integrity_ok=True)
196
    return row