| 1 | """Independent repository-state checks for benchmark correctness.
|
| 2 |
|
| 3 | These helpers inspect durable VCS metadata on disk. They deliberately do not
|
| 4 | parse measured command output, so a subject cannot satisfy them by printing a
|
| 5 | successful-looking transcript.
|
| 6 | """
|
| 7 |
|
| 8 | from __future__ import annotations
|
| 9 |
|
| 10 | import hashlib
|
| 11 | import os
|
| 12 | import shutil
|
| 13 | import subprocess
|
| 14 | from pathlib import Path
|
| 15 | from typing import Any, Mapping
|
| 16 |
|
| 17 | METADATA_DIRS = (".git", ".oak")
|
| 18 | PAYLOAD_SAMPLE_LIMIT = 3
|
| 19 | CONTENT_FAILURE_LIMIT = 20
|
| 20 | STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES = frozenset({"git_head_payload_sha256"})
|
| 21 |
|
| 22 |
|
| 23 | def content_integrity_source_supports_public_trust(
|
| 24 | source: object,
|
| 25 | sources: object = None,
|
| 26 | ) -> bool:
|
| 27 | """Return whether a content oracle source can back a public trust claim.
|
| 28 |
|
| 29 | Worktree-backed checks are still useful measurements, but only committed
|
| 30 | HEAD bytes read through known-good Git are strong enough for public content
|
| 31 | trust. Mixed sources inherit their weakest member.
|
| 32 | """
|
| 33 | if isinstance(sources, list):
|
| 34 | normalized_sources = [item for item in sources if isinstance(item, str)]
|
| 35 | else:
|
| 36 | normalized_sources = []
|
| 37 | if not normalized_sources and isinstance(source, str):
|
| 38 | normalized_sources = [source]
|
| 39 | return bool(normalized_sources) and all(
|
| 40 | item in STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES for item in normalized_sources
|
| 41 | )
|
| 42 |
|
| 43 |
|
| 44 | def content_integrity_public_trust_passed(row: Mapping[str, Any]) -> bool:
|
| 45 | return (
|
| 46 | row.get("content_integrity_check_passed") is True
|
| 47 | and content_integrity_source_supports_public_trust(
|
| 48 | row.get("content_integrity_source"),
|
| 49 | row.get("content_integrity_sources"),
|
| 50 | )
|
| 51 | )
|
| 52 |
|
| 53 |
|
| 54 | def parse_manifest(text: str) -> dict[str, str]:
|
| 55 | """Parse MANIFEST.sha256 lines in sha256sum format."""
|
| 56 | entries: dict[str, str] = {}
|
| 57 | for line in text.splitlines():
|
| 58 | stripped = line.strip()
|
| 59 | if not stripped:
|
| 60 | continue
|
| 61 | digest, separator, rel = stripped.partition(" ")
|
| 62 | if not separator:
|
| 63 | digest, separator, rel = stripped.partition(" ")
|
| 64 | if separator and len(digest) == 64:
|
| 65 | entries[rel.strip()] = digest.lower()
|
| 66 | return entries
|
| 67 |
|
| 68 |
|
| 69 | def metadata_path(repo: Path) -> Path | None:
|
| 70 | for name in METADATA_DIRS:
|
| 71 | candidate = repo / name
|
| 72 | if candidate.exists():
|
| 73 | return candidate
|
| 74 | return None
|
| 75 |
|
| 76 |
|
| 77 | def repository_state_fingerprint(repo: Path) -> str | None:
|
| 78 | root = metadata_path(repo)
|
| 79 | if root is None:
|
| 80 | return None
|
| 81 |
|
| 82 | digest = hashlib.sha256()
|
| 83 | digest.update(root.name.encode("utf-8", "surrogateescape"))
|
| 84 | if root.is_file():
|
| 85 | _hash_file(root, root.name, digest)
|
| 86 | return digest.hexdigest()
|
| 87 |
|
| 88 | for dirpath, dirnames, filenames in os.walk(root):
|
| 89 | dirnames[:] = sorted(name for name in dirnames if not name.endswith(".lock"))
|
| 90 | for filename in sorted(name for name in filenames if not name.endswith(".lock")):
|
| 91 | path = Path(dirpath) / filename
|
| 92 | rel = path.relative_to(root).as_posix()
|
| 93 | _hash_file(path, rel, digest)
|
| 94 | return digest.hexdigest()
|
| 95 |
|
| 96 |
|
| 97 | def state_change_fields(
|
| 98 | repo: Path,
|
| 99 | before_fingerprint: str | None,
|
| 100 | *,
|
| 101 | operation: str,
|
| 102 | process_returncode: int,
|
| 103 | source: str = "direct_repo_metadata_fingerprint",
|
| 104 | ) -> dict[str, Any]:
|
| 105 | after_fingerprint = repository_state_fingerprint(repo)
|
| 106 | passed = process_returncode == 0 and before_fingerprint is not None and after_fingerprint is not None
|
| 107 | if passed:
|
| 108 | passed = before_fingerprint != after_fingerprint
|
| 109 |
|
| 110 | reason: str | None = None
|
| 111 | if process_returncode != 0:
|
| 112 | reason = "process_returncode_nonzero"
|
| 113 | elif before_fingerprint is None:
|
| 114 | reason = "repository_metadata_missing_before_operation"
|
| 115 | elif after_fingerprint is None:
|
| 116 | reason = "repository_metadata_missing_after_operation"
|
| 117 | elif before_fingerprint == after_fingerprint:
|
| 118 | reason = "repository_metadata_unchanged_after_successful_operation"
|
| 119 |
|
| 120 | return {
|
| 121 | "integrity_check_passed": passed,
|
| 122 | "integrity_failure_reason": reason,
|
| 123 | "integrity_check_source": source,
|
| 124 | "integrity_operation": operation,
|
| 125 | "repo_state_before": before_fingerprint,
|
| 126 | "repo_state_after": after_fingerprint,
|
| 127 | }
|
| 128 |
|
| 129 |
|
| 130 | def content_payload_fields(
|
| 131 | repo: Path,
|
| 132 | expected_payloads: list[tuple[str, str]],
|
| 133 | ) -> dict[str, Any]:
|
| 134 | """Check required worktree payloads for a specific workflow phase.
|
| 135 |
|
| 136 | Callers decide which payloads are phase-appropriate. This helper only
|
| 137 | verifies the requested path/needle pairs and reports exact missing paths.
|
| 138 | """
|
| 139 | missing: list[str] = []
|
| 140 | for rel_path, expected in expected_payloads:
|
| 141 | path = repo / rel_path
|
| 142 | try:
|
| 143 | text = path.read_text(errors="replace")
|
| 144 | except OSError:
|
| 145 | missing.append(rel_path)
|
| 146 | continue
|
| 147 | if expected not in text:
|
| 148 | missing.append(rel_path)
|
| 149 |
|
| 150 | reason = None
|
| 151 | if missing:
|
| 152 | reason = "content_payload_expected_payload_missing:" + ",".join(sorted(missing))
|
| 153 | return {
|
| 154 | "content_payload_check_passed": not missing,
|
| 155 | "content_payload_failure_reason": reason,
|
| 156 | "content_payload_expected_paths": [path for path, _needle in expected_payloads],
|
| 157 | "content_payload_missing_paths": missing,
|
| 158 | "content_payload_check_source": "worktree_content_payload_attestation",
|
| 159 | }
|
| 160 |
|
| 161 |
|
| 162 | def sample_worktree_payload_hashes(repo: Path, limit: int = PAYLOAD_SAMPLE_LIMIT) -> dict[str, str]:
|
| 163 | """Hash a small deterministic payload sample outside VCS metadata dirs."""
|
| 164 | paths = []
|
| 165 | root = Path(repo)
|
| 166 | for path in sorted(root.rglob("*")):
|
| 167 | if len(paths) >= limit:
|
| 168 | break
|
| 169 | if not path.is_file():
|
| 170 | continue
|
| 171 | rel = path.relative_to(root)
|
| 172 | if set(rel.parts).intersection(METADATA_DIRS):
|
| 173 | continue
|
| 174 | paths.append(rel.as_posix())
|
| 175 | return worktree_payload_hashes(root, paths)
|
| 176 |
|
| 177 |
|
| 178 | def worktree_payload_hashes(repo: Path, paths: list[str]) -> dict[str, str]:
|
| 179 | """Return sha256 hashes for explicit repo-relative worktree payloads."""
|
| 180 | hashes: dict[str, str] = {}
|
| 181 | root = Path(repo)
|
| 182 | for rel in paths:
|
| 183 | safe_rel = _safe_relative_path(rel)
|
| 184 | if safe_rel is None:
|
| 185 | continue
|
| 186 | path = root / safe_rel
|
| 187 | if not path.is_file():
|
| 188 | continue
|
| 189 | hashes[rel] = _file_sha256(path)
|
| 190 | return hashes
|
| 191 |
|
| 192 |
|
| 193 | def content_integrity_fields(
|
| 194 | repo: Path,
|
| 195 | *,
|
| 196 | operation: str,
|
| 197 | process_returncode: int,
|
| 198 | expected_hashes: Mapping[str, str] | None = None,
|
| 199 | expected_payloads: Mapping[str, str | bytes] | None = None,
|
| 200 | ) -> dict[str, Any]:
|
| 201 | """Verify expected payload content independently of command output.
|
| 202 |
|
| 203 | Git rows read committed content from HEAD with the known-good system Git.
|
| 204 | Other repositories fall back to the worktree; paired with metadata-state
|
| 205 | attestation this still catches content corruption in snapshot-style rows.
|
| 206 | """
|
| 207 | hashes = {str(path): digest.lower() for path, digest in (expected_hashes or {}).items()}
|
| 208 | payloads = dict(expected_payloads or {})
|
| 209 | paths = sorted(set(hashes) | set(payloads))
|
| 210 | fallback_source = "content_payload_sha256"
|
| 211 | sources: set[str] = set()
|
| 212 | failures: list[dict[str, str]] = []
|
| 213 | verified = 0
|
| 214 |
|
| 215 | if process_returncode != 0:
|
| 216 | return _content_fields(
|
| 217 | operation,
|
| 218 | False,
|
| 219 | "process_returncode_nonzero",
|
| 220 | fallback_source,
|
| 221 | len(paths),
|
| 222 | verified,
|
| 223 | failures,
|
| 224 | sources,
|
| 225 | )
|
| 226 | if not paths:
|
| 227 | return _content_fields(
|
| 228 | operation,
|
| 229 | False,
|
| 230 | "content_payload_expectation_missing",
|
| 231 | fallback_source,
|
| 232 | 0,
|
| 233 | verified,
|
| 234 | failures,
|
| 235 | sources,
|
| 236 | )
|
| 237 |
|
| 238 | for rel in paths:
|
| 239 | safe_rel = _safe_relative_path(rel)
|
| 240 | if safe_rel is None:
|
| 241 | failures.append({"path": rel, "reason": "unsafe_relative_path"})
|
| 242 | continue
|
| 243 | data, read_source, error = _read_attested_payload(Path(repo), rel)
|
| 244 | sources.add(read_source)
|
| 245 | if error is not None or data is None:
|
| 246 | failures.append({"path": rel, "reason": error or "payload_missing"})
|
| 247 | continue
|
| 248 | expected_hash = hashes.get(rel)
|
| 249 | if expected_hash is not None:
|
| 250 | actual_hash = hashlib.sha256(data).hexdigest()
|
| 251 | if actual_hash != expected_hash:
|
| 252 | failures.append(
|
| 253 | {
|
| 254 | "path": rel,
|
| 255 | "reason": "sha256_mismatch",
|
| 256 | "expected_sha256": expected_hash,
|
| 257 | "actual_sha256": actual_hash,
|
| 258 | }
|
| 259 | )
|
| 260 | continue
|
| 261 | expected_payload = payloads.get(rel)
|
| 262 | if expected_payload is not None:
|
| 263 | needle = expected_payload.encode("utf-8") if isinstance(expected_payload, str) else expected_payload
|
| 264 | if needle not in data:
|
| 265 | failures.append({"path": rel, "reason": "expected_payload_missing"})
|
| 266 | continue
|
| 267 | verified += 1
|
| 268 |
|
| 269 | passed = not failures and verified == len(paths)
|
| 270 | reason = None if passed else _content_failure_reason(failures)
|
| 271 | return _content_fields(operation, passed, reason, fallback_source, len(paths), verified, failures, sources)
|
| 272 |
|
| 273 |
|
| 274 | def _content_fields(
|
| 275 | operation: str,
|
| 276 | passed: bool,
|
| 277 | reason: str | None,
|
| 278 | source: str,
|
| 279 | expected_files: int,
|
| 280 | verified_files: int,
|
| 281 | failures: list[dict[str, str]],
|
| 282 | sources: set[str] | None = None,
|
| 283 | ) -> dict[str, Any]:
|
| 284 | source_list = sorted(sources or [])
|
| 285 | reported_source = source_list[0] if len(source_list) == 1 else "mixed_payload_sources" if source_list else source
|
| 286 | public_trust_passed = passed and content_integrity_source_supports_public_trust(
|
| 287 | reported_source,
|
| 288 | source_list,
|
| 289 | )
|
| 290 | return {
|
| 291 | "content_integrity_check_passed": passed,
|
| 292 | "content_integrity_failure_reason": reason,
|
| 293 | "content_integrity_source": reported_source,
|
| 294 | "content_integrity_sources": source_list,
|
| 295 | "content_integrity_public_trust_passed": public_trust_passed,
|
| 296 | "content_integrity_operation": operation,
|
| 297 | "content_expected_files": expected_files,
|
| 298 | "content_verified_files": verified_files,
|
| 299 | "content_mismatches": failures[:CONTENT_FAILURE_LIMIT],
|
| 300 | }
|
| 301 |
|
| 302 |
|
| 303 | def _content_failure_reason(failures: list[dict[str, str]]) -> str:
|
| 304 | if not failures:
|
| 305 | return "content_payload_check_failed"
|
| 306 | first = failures[0]
|
| 307 | path = first.get("path", "?")
|
| 308 | reason = first.get("reason", "content_payload_check_failed")
|
| 309 | return f"content_payload_{reason}:{path}"
|
| 310 |
|
| 311 |
|
| 312 | def _read_attested_payload(repo: Path, rel: str) -> tuple[bytes | None, str, str | None]:
|
| 313 | if (repo / ".git").exists():
|
| 314 | git = shutil.which("git")
|
| 315 | if git is None:
|
| 316 | return None, "git_head_payload_sha256", "known_good_git_unavailable_for_payload_check"
|
| 317 | try:
|
| 318 | proc = subprocess.run(
|
| 319 | [git, "-C", str(repo), "show", f"HEAD:{rel}"],
|
| 320 | stdout=subprocess.PIPE,
|
| 321 | stderr=subprocess.PIPE,
|
| 322 | check=False,
|
| 323 | )
|
| 324 | except OSError as exc:
|
| 325 | return None, "git_head_payload_sha256", f"known_good_git_payload_read_error:{exc}"
|
| 326 | if proc.returncode != 0:
|
| 327 | return None, "git_head_payload_sha256", f"git_head_payload_missing:{rel}"
|
| 328 | if _looks_like_lfs_pointer(proc.stdout):
|
| 329 | try:
|
| 330 | return (repo / rel).read_bytes(), "git_lfs_worktree_payload_sha256", None
|
| 331 | except OSError as exc:
|
| 332 | return None, "git_lfs_worktree_payload_sha256", str(exc)
|
| 333 | return proc.stdout, "git_head_payload_sha256", None
|
| 334 |
|
| 335 | path = repo / rel
|
| 336 | try:
|
| 337 | return path.read_bytes(), "worktree_payload_sha256", None
|
| 338 | except OSError as exc:
|
| 339 | return None, "worktree_payload_sha256", str(exc)
|
| 340 |
|
| 341 |
|
| 342 | def _safe_relative_path(rel: str) -> Path | None:
|
| 343 | candidate = Path(rel)
|
| 344 | if candidate.is_absolute() or ".." in candidate.parts:
|
| 345 | return None
|
| 346 | return candidate
|
| 347 |
|
| 348 |
|
| 349 | def _file_sha256(path: Path) -> str:
|
| 350 | digest = hashlib.sha256()
|
| 351 | with path.open("rb") as fh:
|
| 352 | for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
| 353 | digest.update(chunk)
|
| 354 | return digest.hexdigest()
|
| 355 |
|
| 356 |
|
| 357 | def _looks_like_lfs_pointer(data: bytes) -> bool:
|
| 358 | return data.startswith(b"version https://git-lfs.github.com/spec/v1\n") and b"\noid sha256:" in data
|
| 359 |
|
| 360 |
|
| 361 | def _hash_file(path: Path, rel: str, digest: "hashlib._Hash") -> None:
|
| 362 | digest.update(b"\0path:")
|
| 363 | digest.update(rel.encode("utf-8", "surrogateescape"))
|
| 364 | try:
|
| 365 | stat = path.lstat()
|
| 366 | except OSError:
|
| 367 | digest.update(b"\0missing")
|
| 368 | return
|
| 369 | if path.is_symlink():
|
| 370 | digest.update(b"\0symlink:")
|
| 371 | digest.update(os.readlink(path).encode("utf-8", "surrogateescape"))
|
| 372 | return
|
| 373 | if not path.is_file():
|
| 374 | digest.update(b"\0nonfile")
|
| 375 | return
|
| 376 | digest.update(b"\0size:")
|
| 377 | digest.update(str(stat.st_size).encode("ascii"))
|
| 378 | digest.update(b"\0content:")
|
| 379 | try:
|
| 380 | with path.open("rb") as fh:
|
| 381 | for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
| 382 | digest.update(chunk)
|
| 383 | except OSError as exc:
|
| 384 | digest.update(b"\0read_error:")
|
| 385 | digest.update(str(exc).encode("utf-8", "surrogateescape"))
|