Viewing files at commit 62758990fe31Commit detailsView latest

scripts/oakbench/integrity.py 438 lines · 15.9 KB · python
1
"""Independent repository-state checks for benchmark correctness.
2
3
These helpers inspect durable VCS metadata on disk. They deliberately do not
4
parse measured command output, so a subject cannot satisfy them by printing a
5
successful-looking transcript.
6
"""
7
8
from __future__ import annotations
9
10
import hashlib
11
import os
12
import shutil
13
import subprocess
14
import tempfile
15
from pathlib import Path
16
from typing import Any, Mapping
17
18
METADATA_DIRS = (".git", ".oak")
19
PAYLOAD_SAMPLE_LIMIT = 3
20
CONTENT_FAILURE_LIMIT = 20
21
OAK_READER_PAYLOAD_SOURCE = "oak_independent_reader_payload_sha256"
22
STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES = frozenset({"git_head_payload_sha256", OAK_READER_PAYLOAD_SOURCE})
23
24
25
def content_integrity_source_supports_public_trust(
26
    source: object,
27
    sources: object = None,
28
) -> bool:
29
    """Return whether a content oracle source can back a public trust claim.
30
31
    Worktree-backed checks are still useful measurements, but only committed
32
    HEAD bytes read through known-good Git are strong enough for public content
33
    trust. Mixed sources inherit their weakest member.
34
    """
35
    if isinstance(sources, list):
36
        normalized_sources = [item for item in sources if isinstance(item, str)]
37
    else:
38
        normalized_sources = []
39
    if not normalized_sources and isinstance(source, str):
40
        normalized_sources = [source]
41
    return bool(normalized_sources) and all(
42
        item in STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES for item in normalized_sources
43
    )
44
45
46
def content_integrity_public_trust_passed(row: Mapping[str, Any]) -> bool:
47
    return (
48
        row.get("content_integrity_check_passed") is True
49
        and content_integrity_source_supports_public_trust(
50
            row.get("content_integrity_source"),
51
            row.get("content_integrity_sources"),
52
        )
53
    )
54
55
56
def parse_manifest(text: str) -> dict[str, str]:
57
    """Parse MANIFEST.sha256 lines in sha256sum format."""
58
    entries: dict[str, str] = {}
59
    for line in text.splitlines():
60
        stripped = line.strip()
61
        if not stripped:
62
            continue
63
        digest, separator, rel = stripped.partition("  ")
64
        if not separator:
65
            digest, separator, rel = stripped.partition(" ")
66
        if separator and len(digest) == 64:
67
            entries[rel.strip()] = digest.lower()
68
    return entries
69
70
71
def metadata_path(repo: Path) -> Path | None:
72
    for name in METADATA_DIRS:
73
        candidate = repo / name
74
        if candidate.exists():
75
            return candidate
76
    return None
77
78
79
def repository_state_fingerprint(repo: Path) -> str | None:
80
    root = metadata_path(repo)
81
    if root is None:
82
        return None
83
84
    digest = hashlib.sha256()
85
    digest.update(root.name.encode("utf-8", "surrogateescape"))
86
    if root.is_file():
87
        _hash_file(root, root.name, digest)
88
        return digest.hexdigest()
89
90
    for dirpath, dirnames, filenames in os.walk(root):
91
        dirnames[:] = sorted(name for name in dirnames if not name.endswith(".lock"))
92
        for filename in sorted(name for name in filenames if not name.endswith(".lock")):
93
            path = Path(dirpath) / filename
94
            rel = path.relative_to(root).as_posix()
95
            _hash_file(path, rel, digest)
96
    return digest.hexdigest()
97
98
99
def state_change_fields(
100
    repo: Path,
101
    before_fingerprint: str | None,
102
    *,
103
    operation: str,
104
    process_returncode: int,
105
    source: str = "direct_repo_metadata_fingerprint",
106
) -> dict[str, Any]:
107
    after_fingerprint = repository_state_fingerprint(repo)
108
    passed = process_returncode == 0 and before_fingerprint is not None and after_fingerprint is not None
109
    if passed:
110
        passed = before_fingerprint != after_fingerprint
111
112
    reason: str | None = None
113
    if process_returncode != 0:
114
        reason = "process_returncode_nonzero"
115
    elif before_fingerprint is None:
116
        reason = "repository_metadata_missing_before_operation"
117
    elif after_fingerprint is None:
118
        reason = "repository_metadata_missing_after_operation"
119
    elif before_fingerprint == after_fingerprint:
120
        reason = "repository_metadata_unchanged_after_successful_operation"
121
122
    return {
123
        "integrity_check_passed": passed,
124
        "integrity_failure_reason": reason,
125
        "integrity_check_source": source,
126
        "integrity_operation": operation,
127
        "repo_state_before": before_fingerprint,
128
        "repo_state_after": after_fingerprint,
129
    }
130
131
132
def content_payload_fields(
133
    repo: Path,
134
    expected_payloads: list[tuple[str, str]],
135
) -> dict[str, Any]:
136
    """Check required worktree payloads for a specific workflow phase.
137
138
    Callers decide which payloads are phase-appropriate. This helper only
139
    verifies the requested path/needle pairs and reports exact missing paths.
140
    """
141
    missing: list[str] = []
142
    for rel_path, expected in expected_payloads:
143
        path = repo / rel_path
144
        try:
145
            text = path.read_text(errors="replace")
146
        except OSError:
147
            missing.append(rel_path)
148
            continue
149
        if expected not in text:
150
            missing.append(rel_path)
151
152
    reason = None
153
    if missing:
154
        reason = "content_payload_expected_payload_missing:" + ",".join(sorted(missing))
155
    return {
156
        "content_payload_check_passed": not missing,
157
        "content_payload_failure_reason": reason,
158
        "content_payload_expected_paths": [path for path, _needle in expected_payloads],
159
        "content_payload_missing_paths": missing,
160
        "content_payload_check_source": "worktree_content_payload_attestation",
161
    }
162
163
164
def sample_worktree_payload_hashes(repo: Path, limit: int = PAYLOAD_SAMPLE_LIMIT) -> dict[str, str]:
165
    """Hash a small deterministic payload sample outside VCS metadata dirs."""
166
    paths = []
167
    root = Path(repo)
168
    for path in sorted(root.rglob("*")):
169
        if len(paths) >= limit:
170
            break
171
        if not path.is_file():
172
            continue
173
        rel = path.relative_to(root)
174
        if set(rel.parts).intersection(METADATA_DIRS):
175
            continue
176
        paths.append(rel.as_posix())
177
    return worktree_payload_hashes(root, paths)
178
179
180
def worktree_payload_hashes(repo: Path, paths: list[str]) -> dict[str, str]:
181
    """Return sha256 hashes for explicit repo-relative worktree payloads."""
182
    hashes: dict[str, str] = {}
183
    root = Path(repo)
184
    for rel in paths:
185
        safe_rel = _safe_relative_path(rel)
186
        if safe_rel is None:
187
            continue
188
        path = root / safe_rel
189
        if not path.is_file():
190
            continue
191
        hashes[rel] = _file_sha256(path)
192
    return hashes
193
194
195
def content_integrity_fields(
196
    repo: Path,
197
    *,
198
    operation: str,
199
    process_returncode: int,
200
    expected_hashes: Mapping[str, str] | None = None,
201
    expected_payloads: Mapping[str, str | bytes] | None = None,
202
) -> dict[str, Any]:
203
    """Verify expected payload content independently of command output.
204
205
    Git rows read committed content from HEAD with the known-good system Git.
206
    Oak rows read committed HEAD through the independent SQLite reader
207
    (no Oak code or binary). Other repositories, or an Oak reader that is
208
    unavailable, fall back to the worktree, which never earns public trust.
209
    """
210
    hashes = {str(path): digest.lower() for path, digest in (expected_hashes or {}).items()}
211
    payloads = dict(expected_payloads or {})
212
    paths = sorted(set(hashes) | set(payloads))
213
    fallback_source = "content_payload_sha256"
214
    sources: set[str] = set()
215
    failures: list[dict[str, str]] = []
216
    verified = 0
217
218
    if process_returncode != 0:
219
        return _content_fields(
220
            operation,
221
            False,
222
            "process_returncode_nonzero",
223
            fallback_source,
224
            len(paths),
225
            verified,
226
            failures,
227
            sources,
228
        )
229
    if not paths:
230
        return _content_fields(
231
            operation,
232
            False,
233
            "content_payload_expectation_missing",
234
            fallback_source,
235
            0,
236
            verified,
237
            failures,
238
            sources,
239
        )
240
241
    oak_committed = _oak_committed_payloads(Path(repo), paths)
242
    if isinstance(oak_committed, _ReaderUnavailable):
243
        # Never fall back to the worktree for an Oak repository: an
244
        # unreadable committed state is unmeasured (ADR-0002), not a pass.
245
        fields = _content_fields(operation, False, oak_committed.reason, OAK_READER_PAYLOAD_SOURCE,
246
                                 len(paths), 0, [], {OAK_READER_PAYLOAD_SOURCE})
247
        fields.update(content_integrity_check_passed=None, content_integrity_public_trust_passed=False,
248
                      content_integrity_unmeasured_reason=oak_committed.reason)
249
        return fields
250
    for rel in paths:
251
        safe_rel = _safe_relative_path(rel)
252
        if safe_rel is None:
253
            failures.append({"path": rel, "reason": "unsafe_relative_path"})
254
            continue
255
        if oak_committed is not None:
256
            data, read_source, error = oak_committed.get(rel, (None, OAK_READER_PAYLOAD_SOURCE, f"oak_committed_payload_missing:{rel}"))
257
        else:
258
            data, read_source, error = _read_attested_payload(Path(repo), rel)
259
        sources.add(read_source)
260
        if error is not None or data is None:
261
            failures.append({"path": rel, "reason": error or "payload_missing"})
262
            continue
263
        expected_hash = hashes.get(rel)
264
        if expected_hash is not None:
265
            actual_hash = hashlib.sha256(data).hexdigest()
266
            if actual_hash != expected_hash:
267
                failures.append(
268
                    {
269
                        "path": rel,
270
                        "reason": "sha256_mismatch",
271
                        "expected_sha256": expected_hash,
272
                        "actual_sha256": actual_hash,
273
                    }
274
                )
275
                continue
276
        expected_payload = payloads.get(rel)
277
        if expected_payload is not None:
278
            needle = expected_payload.encode("utf-8") if isinstance(expected_payload, str) else expected_payload
279
            if needle not in data:
280
                failures.append({"path": rel, "reason": "expected_payload_missing"})
281
                continue
282
        verified += 1
283
284
    passed = not failures and verified == len(paths)
285
    reason = None if passed else _content_failure_reason(failures)
286
    return _content_fields(operation, passed, reason, fallback_source, len(paths), verified, failures, sources)
287
288
289
def _content_fields(
290
    operation: str,
291
    passed: bool,
292
    reason: str | None,
293
    source: str,
294
    expected_files: int,
295
    verified_files: int,
296
    failures: list[dict[str, str]],
297
    sources: set[str] | None = None,
298
) -> dict[str, Any]:
299
    source_list = sorted(sources or [])
300
    reported_source = source_list[0] if len(source_list) == 1 else "mixed_payload_sources" if source_list else source
301
    public_trust_passed = passed and content_integrity_source_supports_public_trust(
302
        reported_source,
303
        source_list,
304
    )
305
    return {
306
        "content_integrity_check_passed": passed,
307
        "content_integrity_failure_reason": reason,
308
        "content_integrity_source": reported_source,
309
        "content_integrity_sources": source_list,
310
        "content_integrity_public_trust_passed": public_trust_passed,
311
        "content_integrity_operation": operation,
312
        "content_expected_files": expected_files,
313
        "content_verified_files": verified_files,
314
        "content_mismatches": failures[:CONTENT_FAILURE_LIMIT],
315
    }
316
317
318
def _content_failure_reason(failures: list[dict[str, str]]) -> str:
319
    if not failures:
320
        return "content_payload_check_failed"
321
    first = failures[0]
322
    path = first.get("path", "?")
323
    reason = first.get("reason", "content_payload_check_failed")
324
    return f"content_payload_{reason}:{path}"
325
326
327
class _ReaderUnavailable:
328
    def __init__(self, reason: str):
329
        self.reason = reason
330
331
332
def _oak_committed_payloads(repo: Path, paths: list[str]):
333
    """Committed HEAD bytes of an Oak repository via the independent reader.
334
335
    Returns None when the repository is not Oak (the caller uses its other
336
    sources), ``_ReaderUnavailable`` when the reader cannot measure (format
337
    drift, bounds): the check is then unmeasured, never a worktree pass.
338
    Positively invalid stored state fails every requested path.
339
    """
340
    if not (repo / ".oak" / "oak.db").is_file() or (repo / ".git").exists():
341
        return None
342
    from oakbench import oak_sqlite_reader as reader
343
    scope = [rel for rel in paths if _safe_relative_path(rel) is not None]
344
    if not scope:
345
        return None
346
    try:
347
        with tempfile.TemporaryDirectory(prefix="oakbench-integrity-read-") as temporary:
348
            destination = Path(temporary) / "tree"
349
            result = reader.read_committed_tree(repo, destination, reader.normalize_scope(scope))
350
            payloads = {}
351
            for rel in scope:
352
                target = destination / rel
353
                if rel in result.get("nonregular_paths", []):
354
                    payloads[rel] = (None, OAK_READER_PAYLOAD_SOURCE, f"oak_committed_payload_not_regular:{rel}")
355
                elif target.is_file() and not target.is_symlink():
356
                    payloads[rel] = (target.read_bytes(), OAK_READER_PAYLOAD_SOURCE, None)
357
            return payloads
358
    except reader.Invalid as exc:
359
        reason = f"oak_committed_state_invalid:{exc}"
360
        return {rel: (None, OAK_READER_PAYLOAD_SOURCE, reason) for rel in scope}
361
    except (reader.Unavailable, OSError, ValueError) as exc:
362
        return _ReaderUnavailable(f"oak_reader_unavailable:{exc}")
363
364
365
def _read_attested_payload(repo: Path, rel: str) -> tuple[bytes | None, str, str | None]:
366
    if (repo / ".git").exists():
367
        git = shutil.which("git")
368
        if git is None:
369
            return None, "git_head_payload_sha256", "known_good_git_unavailable_for_payload_check"
370
        try:
371
            proc = subprocess.run(
372
                [git, "-C", str(repo), "show", f"HEAD:{rel}"],
373
                stdout=subprocess.PIPE,
374
                stderr=subprocess.PIPE,
375
                check=False,
376
            )
377
        except OSError as exc:
378
            return None, "git_head_payload_sha256", f"known_good_git_payload_read_error:{exc}"
379
        if proc.returncode != 0:
380
            return None, "git_head_payload_sha256", f"git_head_payload_missing:{rel}"
381
        if _looks_like_lfs_pointer(proc.stdout):
382
            try:
383
                return (repo / rel).read_bytes(), "git_lfs_worktree_payload_sha256", None
384
            except OSError as exc:
385
                return None, "git_lfs_worktree_payload_sha256", str(exc)
386
        return proc.stdout, "git_head_payload_sha256", None
387
388
    path = repo / rel
389
    try:
390
        return path.read_bytes(), "worktree_payload_sha256", None
391
    except OSError as exc:
392
        return None, "worktree_payload_sha256", str(exc)
393
394
395
def _safe_relative_path(rel: str) -> Path | None:
396
    candidate = Path(rel)
397
    if candidate.is_absolute() or ".." in candidate.parts:
398
        return None
399
    return candidate
400
401
402
def _file_sha256(path: Path) -> str:
403
    digest = hashlib.sha256()
404
    with path.open("rb") as fh:
405
        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
406
            digest.update(chunk)
407
    return digest.hexdigest()
408
409
410
def _looks_like_lfs_pointer(data: bytes) -> bool:
411
    return data.startswith(b"version https://git-lfs.github.com/spec/v1\n") and b"\noid sha256:" in data
412
413
414
def _hash_file(path: Path, rel: str, digest: "hashlib._Hash") -> None:
415
    digest.update(b"\0path:")
416
    digest.update(rel.encode("utf-8", "surrogateescape"))
417
    try:
418
        stat = path.lstat()
419
    except OSError:
420
        digest.update(b"\0missing")
421
        return
422
    if path.is_symlink():
423
        digest.update(b"\0symlink:")
424
        digest.update(os.readlink(path).encode("utf-8", "surrogateescape"))
425
        return
426
    if not path.is_file():
427
        digest.update(b"\0nonfile")
428
        return
429
    digest.update(b"\0size:")
430
    digest.update(str(stat.st_size).encode("ascii"))
431
    digest.update(b"\0content:")
432
    try:
433
        with path.open("rb") as fh:
434
            for chunk in iter(lambda: fh.read(1024 * 1024), b""):
435
                digest.update(chunk)
436
    except OSError as exc:
437
        digest.update(b"\0read_error:")
438
        digest.update(str(exc).encode("utf-8", "surrogateescape"))