| 1 | """Independent readers: judge committed state without trusting the subject.
|
| 2 |
|
| 3 | Oak: the local ``.oak/oak.db`` and the loopback server's ``<owner>/<repo>.oakdb``
|
| 4 | are SQLite files. We read them with Python's stdlib sqlite3 on a private copy
|
| 5 | (WAL checkpointed into the copy by SQLite itself, the standard recovery path;
|
| 6 | the original files are never opened for write). Committed trees are decoded
|
| 7 | and hash-verified by ``scripts/oakbench/oak_sqlite_reader.py`` (P1's
|
| 8 | independent reader, run under a pinned reader Python with blake3+zstandard).
|
| 9 |
|
| 10 | Git: the real Git plumbing is the oracle (for-each-ref, merge-base, fsck,
|
| 11 | archive), per the program's "git fine as fixture/oracle" rule.
|
| 12 | """
|
| 13 |
|
| 14 | from __future__ import annotations
|
| 15 |
|
| 16 | import io
|
| 17 | import os
|
| 18 | import shutil
|
| 19 | import sqlite3
|
| 20 | import subprocess
|
| 21 | import tarfile
|
| 22 | import tempfile
|
| 23 | from pathlib import Path
|
| 24 | from typing import Iterable, Optional
|
| 25 |
|
| 26 | from . import REPO_ROOT
|
| 27 | from oakbench.conformance import tree_manifest
|
| 28 |
|
| 29 | OAK_READER = REPO_ROOT / "scripts" / "oakbench" / "oak_sqlite_reader.py"
|
| 30 | READER_PY_ENV = "OAK_BENCH_OAK_READER_PYTHON"
|
| 31 |
|
| 32 |
|
| 33 | def worktree_manifest(root: Path) -> Optional[dict]:
|
| 34 | try:
|
| 35 | return tree_manifest(root)
|
| 36 | except (OSError, AssertionError):
|
| 37 | return None
|
| 38 |
|
| 39 |
|
| 40 | def snapshot_sqlite(db_file: Path, dest_dir: Path) -> Path:
|
| 41 | """Copy a (possibly WAL-mode, possibly crashed) SQLite DB and fold its WAL in."""
|
| 42 | dest_dir.mkdir(parents=True, exist_ok=True)
|
| 43 | target = dest_dir / db_file.name
|
| 44 | shutil.copyfile(db_file, target)
|
| 45 | for suffix in ("-wal", "-shm", "-journal"):
|
| 46 | side = Path(str(db_file) + suffix)
|
| 47 | if side.exists():
|
| 48 | shutil.copyfile(side, Path(str(target) + suffix))
|
| 49 | conn = sqlite3.connect(str(target), timeout=5)
|
| 50 | try:
|
| 51 | conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
| 52 | conn.execute("PRAGMA journal_mode=DELETE")
|
| 53 | finally:
|
| 54 | conn.close()
|
| 55 | return target
|
| 56 |
|
| 57 |
|
| 58 | class OakGraph:
|
| 59 | """Stdlib view of an Oak SQLite store's refs and commit graph."""
|
| 60 |
|
| 61 | def __init__(self, db_file: Path):
|
| 62 | self._tmp = tempfile.TemporaryDirectory(prefix="p4-oakgraph-")
|
| 63 | self.path = snapshot_sqlite(Path(db_file), Path(self._tmp.name))
|
| 64 | self.conn = sqlite3.connect(self.path.as_uri() + "?mode=ro", uri=True, timeout=5)
|
| 65 |
|
| 66 | def close(self) -> None:
|
| 67 | self.conn.close()
|
| 68 | self._tmp.cleanup()
|
| 69 |
|
| 70 | def __enter__(self) -> "OakGraph":
|
| 71 | return self
|
| 72 |
|
| 73 | def __exit__(self, *exc) -> None:
|
| 74 | self.close()
|
| 75 |
|
| 76 | def integrity(self) -> str:
|
| 77 | rows = self.conn.execute("PRAGMA integrity_check").fetchall()
|
| 78 | return "ok" if rows == [("ok",)] else "; ".join(str(r[0]) for r in rows[:5])
|
| 79 |
|
| 80 | def _table_exists(self, name: str) -> bool:
|
| 81 | return self.conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone() is not None
|
| 82 |
|
| 83 | def metadata(self, key: str) -> Optional[str]:
|
| 84 | if not self._table_exists("metadata"):
|
| 85 | return None
|
| 86 | row = self.conn.execute("SELECT value FROM metadata WHERE key=?", (key,)).fetchone()
|
| 87 | return row[0] if row else None
|
| 88 |
|
| 89 | def branch_heads(self) -> dict[str, str]:
|
| 90 | if not self._table_exists("branch_heads"):
|
| 91 | return {}
|
| 92 | return {name: head for name, head in self.conn.execute("SELECT branch_name, head_hash FROM branch_heads")
|
| 93 | if head}
|
| 94 |
|
| 95 | def ref_heads(self) -> set[str]:
|
| 96 | """Every commit a ref-like row names: branch heads, HEAD, tags."""
|
| 97 | heads = set(self.branch_heads().values())
|
| 98 | head = self.metadata("head")
|
| 99 | if head:
|
| 100 | heads.add(head)
|
| 101 | if self._table_exists("tags"):
|
| 102 | cols = {r[1] for r in self.conn.execute("PRAGMA table_info(tags)")}
|
| 103 | for col in ("commit_hash", "target_hash", "hash"):
|
| 104 | if col in cols:
|
| 105 | heads.update(r[0] for r in self.conn.execute(f"SELECT {col} FROM tags") if r[0])
|
| 106 | break
|
| 107 | return heads
|
| 108 |
|
| 109 | def commit_row(self, commit: str) -> Optional[tuple]:
|
| 110 | return self.conn.execute("SELECT parent_hash, merge_parent_hash FROM commits WHERE hash=?", (commit,)).fetchone()
|
| 111 |
|
| 112 | def has_commit(self, commit: str) -> bool:
|
| 113 | return self.commit_row(commit) is not None
|
| 114 |
|
| 115 | def ancestors(self, heads: Iterable[str]) -> set[str]:
|
| 116 | seen: set[str] = set()
|
| 117 | stack = [h for h in heads if h]
|
| 118 | while stack:
|
| 119 | current = stack.pop()
|
| 120 | if current in seen:
|
| 121 | continue
|
| 122 | seen.add(current)
|
| 123 | row = self.commit_row(current)
|
| 124 | if row:
|
| 125 | stack.extend(p for p in row if p)
|
| 126 | return seen
|
| 127 |
|
| 128 | def reachable(self) -> set[str]:
|
| 129 | return self.ancestors(self.ref_heads())
|
| 130 |
|
| 131 | def is_ancestor(self, ancestor: str, descendant: str) -> bool:
|
| 132 | return ancestor in self.ancestors([descendant])
|
| 133 |
|
| 134 | def missing_parents(self, heads: Iterable[str]) -> list[str]:
|
| 135 | """Commits reachable by hash links whose rows are absent (a torn graph).
|
| 136 |
|
| 137 | A shallow store legitimately lacks history below its boundary; callers
|
| 138 | pass only heads whose full history they created in the fixture."""
|
| 139 | missing = []
|
| 140 | for commit in self.ancestors(heads):
|
| 141 | if not self.has_commit(commit):
|
| 142 | missing.append(commit)
|
| 143 | return missing
|
| 144 |
|
| 145 |
|
| 146 | def reader_python() -> Optional[str]:
|
| 147 | configured = os.environ.get(READER_PY_ENV)
|
| 148 | if configured:
|
| 149 | return configured if Path(configured).exists() else None
|
| 150 | return None
|
| 151 |
|
| 152 |
|
| 153 | def oak_commit_tree(db_file: Path, commit: Optional[str] = None) -> tuple[Optional[dict], str]:
|
| 154 | """Hash-verified manifest of an Oak commit's tree via the independent reader.
|
| 155 |
|
| 156 | Returns (manifest, status) with status one of: ok | corrupt:<reason> |
|
| 157 | unavailable:<reason>. ``commit`` defaults to the store's HEAD; otherwise the
|
| 158 | private copy's metadata is pointed at it (the copy is ours; the subject's
|
| 159 | store is never modified).
|
| 160 | """
|
| 161 | python = reader_python()
|
| 162 | if python is None:
|
| 163 | return None, f"unavailable:set {READER_PY_ENV} to a Python with blake3+zstandard"
|
| 164 | with tempfile.TemporaryDirectory(prefix="p4-oaktree-") as temp:
|
| 165 | repo = Path(temp) / "repo"
|
| 166 | snap = snapshot_sqlite(Path(db_file), repo / ".oak")
|
| 167 | if snap.name != "oak.db":
|
| 168 | snap.rename(repo / ".oak" / "oak.db")
|
| 169 | db = repo / ".oak" / "oak.db"
|
| 170 | if commit is not None:
|
| 171 | conn = sqlite3.connect(str(db))
|
| 172 | try:
|
| 173 | row = conn.execute("SELECT branch_name FROM commits WHERE hash=?", (commit,)).fetchone()
|
| 174 | if row is None:
|
| 175 | return None, "corrupt:commit row missing"
|
| 176 | for key, value in (("head", commit), ("current_branch", row[0] or "p4-reader")):
|
| 177 | conn.execute("DELETE FROM metadata WHERE key=?", (key,))
|
| 178 | conn.execute("INSERT INTO metadata(key, value) VALUES (?, ?)", (key, value))
|
| 179 | conn.commit()
|
| 180 | finally:
|
| 181 | conn.close()
|
| 182 | out = Path(temp) / "out"
|
| 183 | proc = subprocess.run([python, str(OAK_READER), str(repo), str(out)], capture_output=True, text=True,
|
| 184 | timeout=120)
|
| 185 | if proc.returncode == 0:
|
| 186 | return tree_manifest(out), "ok"
|
| 187 | tail = (proc.stdout or proc.stderr).strip()[-300:]
|
| 188 | if proc.returncode == 1:
|
| 189 | return None, "corrupt:" + tail
|
| 190 | return None, "unavailable:" + tail
|
| 191 |
|
| 192 |
|
| 193 | class GitOracle:
|
| 194 | """Real-Git plumbing used as the Git subject's committed-state oracle."""
|
| 195 |
|
| 196 | def __init__(self, git_bin: str, env: dict):
|
| 197 | self.git = git_bin
|
| 198 | self.env = env
|
| 199 |
|
| 200 | def _run(self, repo: Path, *args: str, check: bool = False) -> subprocess.CompletedProcess:
|
| 201 | return subprocess.run([self.git, *args], cwd=str(repo), env=self.env, capture_output=True, timeout=120,
|
| 202 | check=check)
|
| 203 |
|
| 204 | def refs(self, repo: Path) -> dict[str, str]:
|
| 205 | proc = self._run(repo, "for-each-ref", "--format=%(objectname) %(refname)")
|
| 206 | result = {}
|
| 207 | for line in proc.stdout.decode().splitlines():
|
| 208 | obj, _, name = line.partition(" ")
|
| 209 | result[name] = obj
|
| 210 | head = self._run(repo, "rev-parse", "--verify", "-q", "HEAD")
|
| 211 | if head.returncode == 0:
|
| 212 | result["HEAD"] = head.stdout.decode().strip()
|
| 213 | return result
|
| 214 |
|
| 215 | def head(self, repo: Path) -> Optional[str]:
|
| 216 | return self.refs(repo).get("HEAD")
|
| 217 |
|
| 218 | def reachable_from_refs(self, repo: Path, commit: str) -> bool:
|
| 219 | proc = self._run(repo, "for-each-ref", "--contains", commit, "--format=%(refname)")
|
| 220 | if proc.returncode == 0 and proc.stdout.strip():
|
| 221 | return True
|
| 222 | head = self.head(repo)
|
| 223 | return bool(head) and self.is_ancestor(repo, commit, head)
|
| 224 |
|
| 225 | def is_ancestor(self, repo: Path, ancestor: str, descendant: str) -> bool:
|
| 226 | return self._run(repo, "merge-base", "--is-ancestor", ancestor, descendant).returncode == 0
|
| 227 |
|
| 228 | def fsck(self, repo: Path) -> str:
|
| 229 | proc = self._run(repo, "fsck", "--no-progress", "--no-dangling")
|
| 230 | if proc.returncode == 0:
|
| 231 | return "ok"
|
| 232 | return (proc.stderr.decode("utf-8", "replace") or proc.stdout.decode("utf-8", "replace")).strip()[-400:] or "fsck failed"
|
| 233 |
|
| 234 | def commit_tree(self, repo: Path, commit: str) -> tuple[Optional[dict], str]:
|
| 235 | proc = self._run(repo, "archive", "--format=tar", commit)
|
| 236 | if proc.returncode != 0:
|
| 237 | return None, "corrupt:" + proc.stderr.decode("utf-8", "replace")[-300:]
|
| 238 | with tempfile.TemporaryDirectory(prefix="p4-gittree-") as temp:
|
| 239 | with tarfile.open(fileobj=io.BytesIO(proc.stdout)) as tar:
|
| 240 | tar.extractall(temp) # noqa: S202 - our own fixture content
|
| 241 | return tree_manifest(Path(temp)), "ok"
|