Viewing files at commit d234051be762Commit detailsView latest

suites/robustness/robustness/readers.py 241 lines · 9.6 KB · python
1
"""Independent readers: judge committed state without trusting the subject.
2
3
Oak: the local ``.oak/oak.db`` and the loopback server's ``<owner>/<repo>.oakdb``
4
are SQLite files. We read them with Python's stdlib sqlite3 on a private copy
5
(WAL checkpointed into the copy by SQLite itself, the standard recovery path;
6
the original files are never opened for write). Committed trees are decoded
7
and hash-verified by ``scripts/oakbench/oak_sqlite_reader.py`` (P1's
8
independent reader, run under a pinned reader Python with blake3+zstandard).
9
10
Git: the real Git plumbing is the oracle (for-each-ref, merge-base, fsck,
11
archive), per the program's "git fine as fixture/oracle" rule.
12
"""
13
14
from __future__ import annotations
15
16
import io
17
import os
18
import shutil
19
import sqlite3
20
import subprocess
21
import tarfile
22
import tempfile
23
from pathlib import Path
24
from typing import Iterable, Optional
25
26
from . import REPO_ROOT
27
from oakbench.conformance import tree_manifest
28
29
OAK_READER = REPO_ROOT / "scripts" / "oakbench" / "oak_sqlite_reader.py"
30
READER_PY_ENV = "OAK_BENCH_OAK_READER_PYTHON"
31
32
33
def worktree_manifest(root: Path) -> Optional[dict]:
34
    try:
35
        return tree_manifest(root)
36
    except (OSError, AssertionError):
37
        return None
38
39
40
def snapshot_sqlite(db_file: Path, dest_dir: Path) -> Path:
41
    """Copy a (possibly WAL-mode, possibly crashed) SQLite DB and fold its WAL in."""
42
    dest_dir.mkdir(parents=True, exist_ok=True)
43
    target = dest_dir / db_file.name
44
    shutil.copyfile(db_file, target)
45
    for suffix in ("-wal", "-shm", "-journal"):
46
        side = Path(str(db_file) + suffix)
47
        if side.exists():
48
            shutil.copyfile(side, Path(str(target) + suffix))
49
    conn = sqlite3.connect(str(target), timeout=5)
50
    try:
51
        conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
52
        conn.execute("PRAGMA journal_mode=DELETE")
53
    finally:
54
        conn.close()
55
    return target
56
57
58
class OakGraph:
59
    """Stdlib view of an Oak SQLite store's refs and commit graph."""
60
61
    def __init__(self, db_file: Path):
62
        self._tmp = tempfile.TemporaryDirectory(prefix="p4-oakgraph-")
63
        self.path = snapshot_sqlite(Path(db_file), Path(self._tmp.name))
64
        self.conn = sqlite3.connect(self.path.as_uri() + "?mode=ro", uri=True, timeout=5)
65
66
    def close(self) -> None:
67
        self.conn.close()
68
        self._tmp.cleanup()
69
70
    def __enter__(self) -> "OakGraph":
71
        return self
72
73
    def __exit__(self, *exc) -> None:
74
        self.close()
75
76
    def integrity(self) -> str:
77
        rows = self.conn.execute("PRAGMA integrity_check").fetchall()
78
        return "ok" if rows == [("ok",)] else "; ".join(str(r[0]) for r in rows[:5])
79
80
    def _table_exists(self, name: str) -> bool:
81
        return self.conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone() is not None
82
83
    def metadata(self, key: str) -> Optional[str]:
84
        if not self._table_exists("metadata"):
85
            return None
86
        row = self.conn.execute("SELECT value FROM metadata WHERE key=?", (key,)).fetchone()
87
        return row[0] if row else None
88
89
    def branch_heads(self) -> dict[str, str]:
90
        if not self._table_exists("branch_heads"):
91
            return {}
92
        return {name: head for name, head in self.conn.execute("SELECT branch_name, head_hash FROM branch_heads")
93
                if head}
94
95
    def ref_heads(self) -> set[str]:
96
        """Every commit a ref-like row names: branch heads, HEAD, tags."""
97
        heads = set(self.branch_heads().values())
98
        head = self.metadata("head")
99
        if head:
100
            heads.add(head)
101
        if self._table_exists("tags"):
102
            cols = {r[1] for r in self.conn.execute("PRAGMA table_info(tags)")}
103
            for col in ("commit_hash", "target_hash", "hash"):
104
                if col in cols:
105
                    heads.update(r[0] for r in self.conn.execute(f"SELECT {col} FROM tags") if r[0])
106
                    break
107
        return heads
108
109
    def commit_row(self, commit: str) -> Optional[tuple]:
110
        return self.conn.execute("SELECT parent_hash, merge_parent_hash FROM commits WHERE hash=?", (commit,)).fetchone()
111
112
    def has_commit(self, commit: str) -> bool:
113
        return self.commit_row(commit) is not None
114
115
    def ancestors(self, heads: Iterable[str]) -> set[str]:
116
        seen: set[str] = set()
117
        stack = [h for h in heads if h]
118
        while stack:
119
            current = stack.pop()
120
            if current in seen:
121
                continue
122
            seen.add(current)
123
            row = self.commit_row(current)
124
            if row:
125
                stack.extend(p for p in row if p)
126
        return seen
127
128
    def reachable(self) -> set[str]:
129
        return self.ancestors(self.ref_heads())
130
131
    def is_ancestor(self, ancestor: str, descendant: str) -> bool:
132
        return ancestor in self.ancestors([descendant])
133
134
    def missing_parents(self, heads: Iterable[str]) -> list[str]:
135
        """Commits reachable by hash links whose rows are absent (a torn graph).
136
137
        A shallow store legitimately lacks history below its boundary; callers
138
        pass only heads whose full history they created in the fixture."""
139
        missing = []
140
        for commit in self.ancestors(heads):
141
            if not self.has_commit(commit):
142
                missing.append(commit)
143
        return missing
144
145
146
def reader_python() -> Optional[str]:
147
    configured = os.environ.get(READER_PY_ENV)
148
    if configured:
149
        return configured if Path(configured).exists() else None
150
    return None
151
152
153
def oak_commit_tree(db_file: Path, commit: Optional[str] = None) -> tuple[Optional[dict], str]:
154
    """Hash-verified manifest of an Oak commit's tree via the independent reader.
155
156
    Returns (manifest, status) with status one of: ok | corrupt:<reason> |
157
    unavailable:<reason>. ``commit`` defaults to the store's HEAD; otherwise the
158
    private copy's metadata is pointed at it (the copy is ours; the subject's
159
    store is never modified).
160
    """
161
    python = reader_python()
162
    if python is None:
163
        return None, f"unavailable:set {READER_PY_ENV} to a Python with blake3+zstandard"
164
    with tempfile.TemporaryDirectory(prefix="p4-oaktree-") as temp:
165
        repo = Path(temp) / "repo"
166
        snap = snapshot_sqlite(Path(db_file), repo / ".oak")
167
        if snap.name != "oak.db":
168
            snap.rename(repo / ".oak" / "oak.db")
169
        db = repo / ".oak" / "oak.db"
170
        if commit is not None:
171
            conn = sqlite3.connect(str(db))
172
            try:
173
                row = conn.execute("SELECT branch_name FROM commits WHERE hash=?", (commit,)).fetchone()
174
                if row is None:
175
                    return None, "corrupt:commit row missing"
176
                for key, value in (("head", commit), ("current_branch", row[0] or "p4-reader")):
177
                    conn.execute("DELETE FROM metadata WHERE key=?", (key,))
178
                    conn.execute("INSERT INTO metadata(key, value) VALUES (?, ?)", (key, value))
179
                conn.commit()
180
            finally:
181
                conn.close()
182
        out = Path(temp) / "out"
183
        proc = subprocess.run([python, str(OAK_READER), str(repo), str(out)], capture_output=True, text=True,
184
                              timeout=120)
185
        if proc.returncode == 0:
186
            return tree_manifest(out), "ok"
187
        tail = (proc.stdout or proc.stderr).strip()[-300:]
188
        if proc.returncode == 1:
189
            return None, "corrupt:" + tail
190
        return None, "unavailable:" + tail
191
192
193
class GitOracle:
194
    """Real-Git plumbing used as the Git subject's committed-state oracle."""
195
196
    def __init__(self, git_bin: str, env: dict):
197
        self.git = git_bin
198
        self.env = env
199
200
    def _run(self, repo: Path, *args: str, check: bool = False) -> subprocess.CompletedProcess:
201
        return subprocess.run([self.git, *args], cwd=str(repo), env=self.env, capture_output=True, timeout=120,
202
                              check=check)
203
204
    def refs(self, repo: Path) -> dict[str, str]:
205
        proc = self._run(repo, "for-each-ref", "--format=%(objectname) %(refname)")
206
        result = {}
207
        for line in proc.stdout.decode().splitlines():
208
            obj, _, name = line.partition(" ")
209
            result[name] = obj
210
        head = self._run(repo, "rev-parse", "--verify", "-q", "HEAD")
211
        if head.returncode == 0:
212
            result["HEAD"] = head.stdout.decode().strip()
213
        return result
214
215
    def head(self, repo: Path) -> Optional[str]:
216
        return self.refs(repo).get("HEAD")
217
218
    def reachable_from_refs(self, repo: Path, commit: str) -> bool:
219
        proc = self._run(repo, "for-each-ref", "--contains", commit, "--format=%(refname)")
220
        if proc.returncode == 0 and proc.stdout.strip():
221
            return True
222
        head = self.head(repo)
223
        return bool(head) and self.is_ancestor(repo, commit, head)
224
225
    def is_ancestor(self, repo: Path, ancestor: str, descendant: str) -> bool:
226
        return self._run(repo, "merge-base", "--is-ancestor", ancestor, descendant).returncode == 0
227
228
    def fsck(self, repo: Path) -> str:
229
        proc = self._run(repo, "fsck", "--no-progress", "--no-dangling")
230
        if proc.returncode == 0:
231
            return "ok"
232
        return (proc.stderr.decode("utf-8", "replace") or proc.stdout.decode("utf-8", "replace")).strip()[-400:] or "fsck failed"
233
234
    def commit_tree(self, repo: Path, commit: str) -> tuple[Optional[dict], str]:
235
        proc = self._run(repo, "archive", "--format=tar", commit)
236
        if proc.returncode != 0:
237
            return None, "corrupt:" + proc.stderr.decode("utf-8", "replace")[-300:]
238
        with tempfile.TemporaryDirectory(prefix="p4-gittree-") as temp:
239
            with tarfile.open(fileobj=io.BytesIO(proc.stdout)) as tar:
240
                tar.extractall(temp)  # noqa: S202 - our own fixture content
241
            return tree_manifest(Path(temp)), "ok"