| 1 | """Independent repository-state checks for benchmark correctness.
|
| 2 |
|
| 3 | These helpers inspect durable VCS metadata on disk. They deliberately do not
|
| 4 | parse measured command output, so a subject cannot satisfy them by printing a
|
| 5 | successful-looking transcript.
|
| 6 | """
|
| 7 |
|
| 8 | from __future__ import annotations
|
| 9 |
|
| 10 | import hashlib
|
| 11 | import os
|
| 12 | import shutil
|
| 13 | import subprocess
|
| 14 | import tempfile
|
| 15 | from pathlib import Path
|
| 16 | from typing import Any, Mapping
|
| 17 |
|
| 18 | METADATA_DIRS = (".git", ".oak")
|
| 19 | PAYLOAD_SAMPLE_LIMIT = 3
|
| 20 | CONTENT_FAILURE_LIMIT = 20
|
| 21 | OAK_READER_PAYLOAD_SOURCE = "oak_independent_reader_payload_sha256"
|
| 22 | STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES = frozenset({"git_head_payload_sha256", OAK_READER_PAYLOAD_SOURCE})
|
| 23 |
|
| 24 |
|
| 25 | def content_integrity_source_supports_public_trust(
|
| 26 | source: object,
|
| 27 | sources: object = None,
|
| 28 | ) -> bool:
|
| 29 | """Return whether a content oracle source can back a public trust claim.
|
| 30 |
|
| 31 | Worktree-backed checks are still useful measurements, but only committed
|
| 32 | HEAD bytes read through known-good Git are strong enough for public content
|
| 33 | trust. Mixed sources inherit their weakest member.
|
| 34 | """
|
| 35 | if isinstance(sources, list):
|
| 36 | normalized_sources = [item for item in sources if isinstance(item, str)]
|
| 37 | else:
|
| 38 | normalized_sources = []
|
| 39 | if not normalized_sources and isinstance(source, str):
|
| 40 | normalized_sources = [source]
|
| 41 | return bool(normalized_sources) and all(
|
| 42 | item in STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES for item in normalized_sources
|
| 43 | )
|
| 44 |
|
| 45 |
|
| 46 | def content_integrity_public_trust_passed(row: Mapping[str, Any]) -> bool:
|
| 47 | return (
|
| 48 | row.get("content_integrity_check_passed") is True
|
| 49 | and content_integrity_source_supports_public_trust(
|
| 50 | row.get("content_integrity_source"),
|
| 51 | row.get("content_integrity_sources"),
|
| 52 | )
|
| 53 | )
|
| 54 |
|
| 55 |
|
| 56 | def parse_manifest(text: str) -> dict[str, str]:
|
| 57 | """Parse MANIFEST.sha256 lines in sha256sum format."""
|
| 58 | entries: dict[str, str] = {}
|
| 59 | for line in text.splitlines():
|
| 60 | stripped = line.strip()
|
| 61 | if not stripped:
|
| 62 | continue
|
| 63 | digest, separator, rel = stripped.partition(" ")
|
| 64 | if not separator:
|
| 65 | digest, separator, rel = stripped.partition(" ")
|
| 66 | if separator and len(digest) == 64:
|
| 67 | entries[rel.strip()] = digest.lower()
|
| 68 | return entries
|
| 69 |
|
| 70 |
|
| 71 | def metadata_path(repo: Path) -> Path | None:
|
| 72 | for name in METADATA_DIRS:
|
| 73 | candidate = repo / name
|
| 74 | if candidate.exists():
|
| 75 | return candidate
|
| 76 | return None
|
| 77 |
|
| 78 |
|
| 79 | def repository_state_fingerprint(repo: Path) -> str | None:
|
| 80 | root = metadata_path(repo)
|
| 81 | if root is None:
|
| 82 | return None
|
| 83 |
|
| 84 | digest = hashlib.sha256()
|
| 85 | digest.update(root.name.encode("utf-8", "surrogateescape"))
|
| 86 | if root.is_file():
|
| 87 | _hash_file(root, root.name, digest)
|
| 88 | return digest.hexdigest()
|
| 89 |
|
| 90 | for dirpath, dirnames, filenames in os.walk(root):
|
| 91 | dirnames[:] = sorted(name for name in dirnames if not name.endswith(".lock"))
|
| 92 | for filename in sorted(name for name in filenames if not name.endswith(".lock")):
|
| 93 | path = Path(dirpath) / filename
|
| 94 | rel = path.relative_to(root).as_posix()
|
| 95 | _hash_file(path, rel, digest)
|
| 96 | return digest.hexdigest()
|
| 97 |
|
| 98 |
|
| 99 | def state_change_fields(
|
| 100 | repo: Path,
|
| 101 | before_fingerprint: str | None,
|
| 102 | *,
|
| 103 | operation: str,
|
| 104 | process_returncode: int,
|
| 105 | source: str = "direct_repo_metadata_fingerprint",
|
| 106 | ) -> dict[str, Any]:
|
| 107 | after_fingerprint = repository_state_fingerprint(repo)
|
| 108 | passed = process_returncode == 0 and before_fingerprint is not None and after_fingerprint is not None
|
| 109 | if passed:
|
| 110 | passed = before_fingerprint != after_fingerprint
|
| 111 |
|
| 112 | reason: str | None = None
|
| 113 | if process_returncode != 0:
|
| 114 | reason = "process_returncode_nonzero"
|
| 115 | elif before_fingerprint is None:
|
| 116 | reason = "repository_metadata_missing_before_operation"
|
| 117 | elif after_fingerprint is None:
|
| 118 | reason = "repository_metadata_missing_after_operation"
|
| 119 | elif before_fingerprint == after_fingerprint:
|
| 120 | reason = "repository_metadata_unchanged_after_successful_operation"
|
| 121 |
|
| 122 | return {
|
| 123 | "integrity_check_passed": passed,
|
| 124 | "integrity_failure_reason": reason,
|
| 125 | "integrity_check_source": source,
|
| 126 | "integrity_operation": operation,
|
| 127 | "repo_state_before": before_fingerprint,
|
| 128 | "repo_state_after": after_fingerprint,
|
| 129 | }
|
| 130 |
|
| 131 |
|
| 132 | def content_payload_fields(
|
| 133 | repo: Path,
|
| 134 | expected_payloads: list[tuple[str, str]],
|
| 135 | ) -> dict[str, Any]:
|
| 136 | """Check required worktree payloads for a specific workflow phase.
|
| 137 |
|
| 138 | Callers decide which payloads are phase-appropriate. This helper only
|
| 139 | verifies the requested path/needle pairs and reports exact missing paths.
|
| 140 | """
|
| 141 | missing: list[str] = []
|
| 142 | for rel_path, expected in expected_payloads:
|
| 143 | path = repo / rel_path
|
| 144 | try:
|
| 145 | text = path.read_text(errors="replace")
|
| 146 | except OSError:
|
| 147 | missing.append(rel_path)
|
| 148 | continue
|
| 149 | if expected not in text:
|
| 150 | missing.append(rel_path)
|
| 151 |
|
| 152 | reason = None
|
| 153 | if missing:
|
| 154 | reason = "content_payload_expected_payload_missing:" + ",".join(sorted(missing))
|
| 155 | return {
|
| 156 | "content_payload_check_passed": not missing,
|
| 157 | "content_payload_failure_reason": reason,
|
| 158 | "content_payload_expected_paths": [path for path, _needle in expected_payloads],
|
| 159 | "content_payload_missing_paths": missing,
|
| 160 | "content_payload_check_source": "worktree_content_payload_attestation",
|
| 161 | }
|
| 162 |
|
| 163 |
|
| 164 | def sample_worktree_payload_hashes(repo: Path, limit: int = PAYLOAD_SAMPLE_LIMIT) -> dict[str, str]:
|
| 165 | """Hash a small deterministic payload sample outside VCS metadata dirs."""
|
| 166 | paths = []
|
| 167 | root = Path(repo)
|
| 168 | for path in sorted(root.rglob("*")):
|
| 169 | if len(paths) >= limit:
|
| 170 | break
|
| 171 | if not path.is_file():
|
| 172 | continue
|
| 173 | rel = path.relative_to(root)
|
| 174 | if set(rel.parts).intersection(METADATA_DIRS):
|
| 175 | continue
|
| 176 | paths.append(rel.as_posix())
|
| 177 | return worktree_payload_hashes(root, paths)
|
| 178 |
|
| 179 |
|
| 180 | def worktree_payload_hashes(repo: Path, paths: list[str]) -> dict[str, str]:
|
| 181 | """Return sha256 hashes for explicit repo-relative worktree payloads."""
|
| 182 | hashes: dict[str, str] = {}
|
| 183 | root = Path(repo)
|
| 184 | for rel in paths:
|
| 185 | safe_rel = _safe_relative_path(rel)
|
| 186 | if safe_rel is None:
|
| 187 | continue
|
| 188 | path = root / safe_rel
|
| 189 | if not path.is_file():
|
| 190 | continue
|
| 191 | hashes[rel] = _file_sha256(path)
|
| 192 | return hashes
|
| 193 |
|
| 194 |
|
| 195 | def content_integrity_fields(
|
| 196 | repo: Path,
|
| 197 | *,
|
| 198 | operation: str,
|
| 199 | process_returncode: int,
|
| 200 | expected_hashes: Mapping[str, str] | None = None,
|
| 201 | expected_payloads: Mapping[str, str | bytes] | None = None,
|
| 202 | ) -> dict[str, Any]:
|
| 203 | """Verify expected payload content independently of command output.
|
| 204 |
|
| 205 | Git rows read committed content from HEAD with the known-good system Git.
|
| 206 | Oak rows read committed HEAD through the independent SQLite reader
|
| 207 | (no Oak code or binary). Other repositories, or an Oak reader that is
|
| 208 | unavailable, fall back to the worktree, which never earns public trust.
|
| 209 | """
|
| 210 | hashes = {str(path): digest.lower() for path, digest in (expected_hashes or {}).items()}
|
| 211 | payloads = dict(expected_payloads or {})
|
| 212 | paths = sorted(set(hashes) | set(payloads))
|
| 213 | fallback_source = "content_payload_sha256"
|
| 214 | sources: set[str] = set()
|
| 215 | failures: list[dict[str, str]] = []
|
| 216 | verified = 0
|
| 217 |
|
| 218 | if process_returncode != 0:
|
| 219 | return _content_fields(
|
| 220 | operation,
|
| 221 | False,
|
| 222 | "process_returncode_nonzero",
|
| 223 | fallback_source,
|
| 224 | len(paths),
|
| 225 | verified,
|
| 226 | failures,
|
| 227 | sources,
|
| 228 | )
|
| 229 | if not paths:
|
| 230 | return _content_fields(
|
| 231 | operation,
|
| 232 | False,
|
| 233 | "content_payload_expectation_missing",
|
| 234 | fallback_source,
|
| 235 | 0,
|
| 236 | verified,
|
| 237 | failures,
|
| 238 | sources,
|
| 239 | )
|
| 240 |
|
| 241 | oak_committed = _oak_committed_payloads(Path(repo), paths)
|
| 242 | for rel in paths:
|
| 243 | safe_rel = _safe_relative_path(rel)
|
| 244 | if safe_rel is None:
|
| 245 | failures.append({"path": rel, "reason": "unsafe_relative_path"})
|
| 246 | continue
|
| 247 | if oak_committed is not None:
|
| 248 | data, read_source, error = oak_committed.get(rel, (None, OAK_READER_PAYLOAD_SOURCE, f"oak_committed_payload_missing:{rel}"))
|
| 249 | else:
|
| 250 | data, read_source, error = _read_attested_payload(Path(repo), rel)
|
| 251 | sources.add(read_source)
|
| 252 | if error is not None or data is None:
|
| 253 | failures.append({"path": rel, "reason": error or "payload_missing"})
|
| 254 | continue
|
| 255 | expected_hash = hashes.get(rel)
|
| 256 | if expected_hash is not None:
|
| 257 | actual_hash = hashlib.sha256(data).hexdigest()
|
| 258 | if actual_hash != expected_hash:
|
| 259 | failures.append(
|
| 260 | {
|
| 261 | "path": rel,
|
| 262 | "reason": "sha256_mismatch",
|
| 263 | "expected_sha256": expected_hash,
|
| 264 | "actual_sha256": actual_hash,
|
| 265 | }
|
| 266 | )
|
| 267 | continue
|
| 268 | expected_payload = payloads.get(rel)
|
| 269 | if expected_payload is not None:
|
| 270 | needle = expected_payload.encode("utf-8") if isinstance(expected_payload, str) else expected_payload
|
| 271 | if needle not in data:
|
| 272 | failures.append({"path": rel, "reason": "expected_payload_missing"})
|
| 273 | continue
|
| 274 | verified += 1
|
| 275 |
|
| 276 | passed = not failures and verified == len(paths)
|
| 277 | reason = None if passed else _content_failure_reason(failures)
|
| 278 | return _content_fields(operation, passed, reason, fallback_source, len(paths), verified, failures, sources)
|
| 279 |
|
| 280 |
|
| 281 | def _content_fields(
|
| 282 | operation: str,
|
| 283 | passed: bool,
|
| 284 | reason: str | None,
|
| 285 | source: str,
|
| 286 | expected_files: int,
|
| 287 | verified_files: int,
|
| 288 | failures: list[dict[str, str]],
|
| 289 | sources: set[str] | None = None,
|
| 290 | ) -> dict[str, Any]:
|
| 291 | source_list = sorted(sources or [])
|
| 292 | reported_source = source_list[0] if len(source_list) == 1 else "mixed_payload_sources" if source_list else source
|
| 293 | public_trust_passed = passed and content_integrity_source_supports_public_trust(
|
| 294 | reported_source,
|
| 295 | source_list,
|
| 296 | )
|
| 297 | return {
|
| 298 | "content_integrity_check_passed": passed,
|
| 299 | "content_integrity_failure_reason": reason,
|
| 300 | "content_integrity_source": reported_source,
|
| 301 | "content_integrity_sources": source_list,
|
| 302 | "content_integrity_public_trust_passed": public_trust_passed,
|
| 303 | "content_integrity_operation": operation,
|
| 304 | "content_expected_files": expected_files,
|
| 305 | "content_verified_files": verified_files,
|
| 306 | "content_mismatches": failures[:CONTENT_FAILURE_LIMIT],
|
| 307 | }
|
| 308 |
|
| 309 |
|
| 310 | def _content_failure_reason(failures: list[dict[str, str]]) -> str:
|
| 311 | if not failures:
|
| 312 | return "content_payload_check_failed"
|
| 313 | first = failures[0]
|
| 314 | path = first.get("path", "?")
|
| 315 | reason = first.get("reason", "content_payload_check_failed")
|
| 316 | return f"content_payload_{reason}:{path}"
|
| 317 |
|
| 318 |
|
| 319 | def _oak_committed_payloads(repo: Path, paths: list[str]):
|
| 320 | """Committed HEAD bytes of an Oak repository via the independent reader.
|
| 321 |
|
| 322 | Returns None when the repository is not Oak or the reader is unavailable
|
| 323 | (format drift, bounds): the caller then falls back to the weak worktree
|
| 324 | source, which never earns public trust. Positively invalid stored state
|
| 325 | fails every requested path.
|
| 326 | """
|
| 327 | if not (repo / ".oak" / "oak.db").is_file() or (repo / ".git").exists():
|
| 328 | return None
|
| 329 | from oakbench import oak_sqlite_reader as reader
|
| 330 | scope = [rel for rel in paths if _safe_relative_path(rel) is not None]
|
| 331 | if not scope:
|
| 332 | return None
|
| 333 | try:
|
| 334 | with tempfile.TemporaryDirectory(prefix="oakbench-integrity-read-") as temporary:
|
| 335 | destination = Path(temporary) / "tree"
|
| 336 | result = reader.read_committed_tree(repo, destination, reader.normalize_scope(scope))
|
| 337 | payloads = {}
|
| 338 | for rel in scope:
|
| 339 | target = destination / rel
|
| 340 | if rel in result.get("nonregular_paths", []):
|
| 341 | payloads[rel] = (None, OAK_READER_PAYLOAD_SOURCE, f"oak_committed_payload_not_regular:{rel}")
|
| 342 | elif target.is_file() and not target.is_symlink():
|
| 343 | payloads[rel] = (target.read_bytes(), OAK_READER_PAYLOAD_SOURCE, None)
|
| 344 | return payloads
|
| 345 | except reader.Invalid as exc:
|
| 346 | reason = f"oak_committed_state_invalid:{exc}"
|
| 347 | return {rel: (None, OAK_READER_PAYLOAD_SOURCE, reason) for rel in scope}
|
| 348 | except (reader.Unavailable, OSError, ValueError):
|
| 349 | return None
|
| 350 |
|
| 351 |
|
| 352 | def _read_attested_payload(repo: Path, rel: str) -> tuple[bytes | None, str, str | None]:
|
| 353 | if (repo / ".git").exists():
|
| 354 | git = shutil.which("git")
|
| 355 | if git is None:
|
| 356 | return None, "git_head_payload_sha256", "known_good_git_unavailable_for_payload_check"
|
| 357 | try:
|
| 358 | proc = subprocess.run(
|
| 359 | [git, "-C", str(repo), "show", f"HEAD:{rel}"],
|
| 360 | stdout=subprocess.PIPE,
|
| 361 | stderr=subprocess.PIPE,
|
| 362 | check=False,
|
| 363 | )
|
| 364 | except OSError as exc:
|
| 365 | return None, "git_head_payload_sha256", f"known_good_git_payload_read_error:{exc}"
|
| 366 | if proc.returncode != 0:
|
| 367 | return None, "git_head_payload_sha256", f"git_head_payload_missing:{rel}"
|
| 368 | if _looks_like_lfs_pointer(proc.stdout):
|
| 369 | try:
|
| 370 | return (repo / rel).read_bytes(), "git_lfs_worktree_payload_sha256", None
|
| 371 | except OSError as exc:
|
| 372 | return None, "git_lfs_worktree_payload_sha256", str(exc)
|
| 373 | return proc.stdout, "git_head_payload_sha256", None
|
| 374 |
|
| 375 | path = repo / rel
|
| 376 | try:
|
| 377 | return path.read_bytes(), "worktree_payload_sha256", None
|
| 378 | except OSError as exc:
|
| 379 | return None, "worktree_payload_sha256", str(exc)
|
| 380 |
|
| 381 |
|
| 382 | def _safe_relative_path(rel: str) -> Path | None:
|
| 383 | candidate = Path(rel)
|
| 384 | if candidate.is_absolute() or ".." in candidate.parts:
|
| 385 | return None
|
| 386 | return candidate
|
| 387 |
|
| 388 |
|
| 389 | def _file_sha256(path: Path) -> str:
|
| 390 | digest = hashlib.sha256()
|
| 391 | with path.open("rb") as fh:
|
| 392 | for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
| 393 | digest.update(chunk)
|
| 394 | return digest.hexdigest()
|
| 395 |
|
| 396 |
|
| 397 | def _looks_like_lfs_pointer(data: bytes) -> bool:
|
| 398 | return data.startswith(b"version https://git-lfs.github.com/spec/v1\n") and b"\noid sha256:" in data
|
| 399 |
|
| 400 |
|
| 401 | def _hash_file(path: Path, rel: str, digest: "hashlib._Hash") -> None:
|
| 402 | digest.update(b"\0path:")
|
| 403 | digest.update(rel.encode("utf-8", "surrogateescape"))
|
| 404 | try:
|
| 405 | stat = path.lstat()
|
| 406 | except OSError:
|
| 407 | digest.update(b"\0missing")
|
| 408 | return
|
| 409 | if path.is_symlink():
|
| 410 | digest.update(b"\0symlink:")
|
| 411 | digest.update(os.readlink(path).encode("utf-8", "surrogateescape"))
|
| 412 | return
|
| 413 | if not path.is_file():
|
| 414 | digest.update(b"\0nonfile")
|
| 415 | return
|
| 416 | digest.update(b"\0size:")
|
| 417 | digest.update(str(stat.st_size).encode("ascii"))
|
| 418 | digest.update(b"\0content:")
|
| 419 | try:
|
| 420 | with path.open("rb") as fh:
|
| 421 | for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
| 422 | digest.update(chunk)
|
| 423 | except OSError as exc:
|
| 424 | digest.update(b"\0read_error:")
|
| 425 | digest.update(str(exc).encode("utf-8", "surrogateescape"))
|