Viewing files at commit 21316fe59317Commit detailsView latest

scripts/oakbench/integrity.py 425 lines · 15.2 KB · python
1
"""Independent repository-state checks for benchmark correctness.
2
3
These helpers inspect durable VCS metadata on disk. They deliberately do not
4
parse measured command output, so a subject cannot satisfy them by printing a
5
successful-looking transcript.
6
"""
7
8
from __future__ import annotations
9
10
import hashlib
11
import os
12
import shutil
13
import subprocess
14
import tempfile
15
from pathlib import Path
16
from typing import Any, Mapping
17
18
METADATA_DIRS = (".git", ".oak")
19
PAYLOAD_SAMPLE_LIMIT = 3
20
CONTENT_FAILURE_LIMIT = 20
21
OAK_READER_PAYLOAD_SOURCE = "oak_independent_reader_payload_sha256"
22
STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES = frozenset({"git_head_payload_sha256", OAK_READER_PAYLOAD_SOURCE})
23
24
25
def content_integrity_source_supports_public_trust(
26
    source: object,
27
    sources: object = None,
28
) -> bool:
29
    """Return whether a content oracle source can back a public trust claim.
30
31
    Worktree-backed checks are still useful measurements, but only committed
32
    HEAD bytes read through known-good Git are strong enough for public content
33
    trust. Mixed sources inherit their weakest member.
34
    """
35
    if isinstance(sources, list):
36
        normalized_sources = [item for item in sources if isinstance(item, str)]
37
    else:
38
        normalized_sources = []
39
    if not normalized_sources and isinstance(source, str):
40
        normalized_sources = [source]
41
    return bool(normalized_sources) and all(
42
        item in STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES for item in normalized_sources
43
    )
44
45
46
def content_integrity_public_trust_passed(row: Mapping[str, Any]) -> bool:
47
    return (
48
        row.get("content_integrity_check_passed") is True
49
        and content_integrity_source_supports_public_trust(
50
            row.get("content_integrity_source"),
51
            row.get("content_integrity_sources"),
52
        )
53
    )
54
55
56
def parse_manifest(text: str) -> dict[str, str]:
57
    """Parse MANIFEST.sha256 lines in sha256sum format."""
58
    entries: dict[str, str] = {}
59
    for line in text.splitlines():
60
        stripped = line.strip()
61
        if not stripped:
62
            continue
63
        digest, separator, rel = stripped.partition("  ")
64
        if not separator:
65
            digest, separator, rel = stripped.partition(" ")
66
        if separator and len(digest) == 64:
67
            entries[rel.strip()] = digest.lower()
68
    return entries
69
70
71
def metadata_path(repo: Path) -> Path | None:
72
    for name in METADATA_DIRS:
73
        candidate = repo / name
74
        if candidate.exists():
75
            return candidate
76
    return None
77
78
79
def repository_state_fingerprint(repo: Path) -> str | None:
80
    root = metadata_path(repo)
81
    if root is None:
82
        return None
83
84
    digest = hashlib.sha256()
85
    digest.update(root.name.encode("utf-8", "surrogateescape"))
86
    if root.is_file():
87
        _hash_file(root, root.name, digest)
88
        return digest.hexdigest()
89
90
    for dirpath, dirnames, filenames in os.walk(root):
91
        dirnames[:] = sorted(name for name in dirnames if not name.endswith(".lock"))
92
        for filename in sorted(name for name in filenames if not name.endswith(".lock")):
93
            path = Path(dirpath) / filename
94
            rel = path.relative_to(root).as_posix()
95
            _hash_file(path, rel, digest)
96
    return digest.hexdigest()
97
98
99
def state_change_fields(
100
    repo: Path,
101
    before_fingerprint: str | None,
102
    *,
103
    operation: str,
104
    process_returncode: int,
105
    source: str = "direct_repo_metadata_fingerprint",
106
) -> dict[str, Any]:
107
    after_fingerprint = repository_state_fingerprint(repo)
108
    passed = process_returncode == 0 and before_fingerprint is not None and after_fingerprint is not None
109
    if passed:
110
        passed = before_fingerprint != after_fingerprint
111
112
    reason: str | None = None
113
    if process_returncode != 0:
114
        reason = "process_returncode_nonzero"
115
    elif before_fingerprint is None:
116
        reason = "repository_metadata_missing_before_operation"
117
    elif after_fingerprint is None:
118
        reason = "repository_metadata_missing_after_operation"
119
    elif before_fingerprint == after_fingerprint:
120
        reason = "repository_metadata_unchanged_after_successful_operation"
121
122
    return {
123
        "integrity_check_passed": passed,
124
        "integrity_failure_reason": reason,
125
        "integrity_check_source": source,
126
        "integrity_operation": operation,
127
        "repo_state_before": before_fingerprint,
128
        "repo_state_after": after_fingerprint,
129
    }
130
131
132
def content_payload_fields(
133
    repo: Path,
134
    expected_payloads: list[tuple[str, str]],
135
) -> dict[str, Any]:
136
    """Check required worktree payloads for a specific workflow phase.
137
138
    Callers decide which payloads are phase-appropriate. This helper only
139
    verifies the requested path/needle pairs and reports exact missing paths.
140
    """
141
    missing: list[str] = []
142
    for rel_path, expected in expected_payloads:
143
        path = repo / rel_path
144
        try:
145
            text = path.read_text(errors="replace")
146
        except OSError:
147
            missing.append(rel_path)
148
            continue
149
        if expected not in text:
150
            missing.append(rel_path)
151
152
    reason = None
153
    if missing:
154
        reason = "content_payload_expected_payload_missing:" + ",".join(sorted(missing))
155
    return {
156
        "content_payload_check_passed": not missing,
157
        "content_payload_failure_reason": reason,
158
        "content_payload_expected_paths": [path for path, _needle in expected_payloads],
159
        "content_payload_missing_paths": missing,
160
        "content_payload_check_source": "worktree_content_payload_attestation",
161
    }
162
163
164
def sample_worktree_payload_hashes(repo: Path, limit: int = PAYLOAD_SAMPLE_LIMIT) -> dict[str, str]:
165
    """Hash a small deterministic payload sample outside VCS metadata dirs."""
166
    paths = []
167
    root = Path(repo)
168
    for path in sorted(root.rglob("*")):
169
        if len(paths) >= limit:
170
            break
171
        if not path.is_file():
172
            continue
173
        rel = path.relative_to(root)
174
        if set(rel.parts).intersection(METADATA_DIRS):
175
            continue
176
        paths.append(rel.as_posix())
177
    return worktree_payload_hashes(root, paths)
178
179
180
def worktree_payload_hashes(repo: Path, paths: list[str]) -> dict[str, str]:
181
    """Return sha256 hashes for explicit repo-relative worktree payloads."""
182
    hashes: dict[str, str] = {}
183
    root = Path(repo)
184
    for rel in paths:
185
        safe_rel = _safe_relative_path(rel)
186
        if safe_rel is None:
187
            continue
188
        path = root / safe_rel
189
        if not path.is_file():
190
            continue
191
        hashes[rel] = _file_sha256(path)
192
    return hashes
193
194
195
def content_integrity_fields(
196
    repo: Path,
197
    *,
198
    operation: str,
199
    process_returncode: int,
200
    expected_hashes: Mapping[str, str] | None = None,
201
    expected_payloads: Mapping[str, str | bytes] | None = None,
202
) -> dict[str, Any]:
203
    """Verify expected payload content independently of command output.
204
205
    Git rows read committed content from HEAD with the known-good system Git.
206
    Oak rows read committed HEAD through the independent SQLite reader
207
    (no Oak code or binary). Other repositories, or an Oak reader that is
208
    unavailable, fall back to the worktree, which never earns public trust.
209
    """
210
    hashes = {str(path): digest.lower() for path, digest in (expected_hashes or {}).items()}
211
    payloads = dict(expected_payloads or {})
212
    paths = sorted(set(hashes) | set(payloads))
213
    fallback_source = "content_payload_sha256"
214
    sources: set[str] = set()
215
    failures: list[dict[str, str]] = []
216
    verified = 0
217
218
    if process_returncode != 0:
219
        return _content_fields(
220
            operation,
221
            False,
222
            "process_returncode_nonzero",
223
            fallback_source,
224
            len(paths),
225
            verified,
226
            failures,
227
            sources,
228
        )
229
    if not paths:
230
        return _content_fields(
231
            operation,
232
            False,
233
            "content_payload_expectation_missing",
234
            fallback_source,
235
            0,
236
            verified,
237
            failures,
238
            sources,
239
        )
240
241
    oak_committed = _oak_committed_payloads(Path(repo), paths)
242
    for rel in paths:
243
        safe_rel = _safe_relative_path(rel)
244
        if safe_rel is None:
245
            failures.append({"path": rel, "reason": "unsafe_relative_path"})
246
            continue
247
        if oak_committed is not None:
248
            data, read_source, error = oak_committed.get(rel, (None, OAK_READER_PAYLOAD_SOURCE, f"oak_committed_payload_missing:{rel}"))
249
        else:
250
            data, read_source, error = _read_attested_payload(Path(repo), rel)
251
        sources.add(read_source)
252
        if error is not None or data is None:
253
            failures.append({"path": rel, "reason": error or "payload_missing"})
254
            continue
255
        expected_hash = hashes.get(rel)
256
        if expected_hash is not None:
257
            actual_hash = hashlib.sha256(data).hexdigest()
258
            if actual_hash != expected_hash:
259
                failures.append(
260
                    {
261
                        "path": rel,
262
                        "reason": "sha256_mismatch",
263
                        "expected_sha256": expected_hash,
264
                        "actual_sha256": actual_hash,
265
                    }
266
                )
267
                continue
268
        expected_payload = payloads.get(rel)
269
        if expected_payload is not None:
270
            needle = expected_payload.encode("utf-8") if isinstance(expected_payload, str) else expected_payload
271
            if needle not in data:
272
                failures.append({"path": rel, "reason": "expected_payload_missing"})
273
                continue
274
        verified += 1
275
276
    passed = not failures and verified == len(paths)
277
    reason = None if passed else _content_failure_reason(failures)
278
    return _content_fields(operation, passed, reason, fallback_source, len(paths), verified, failures, sources)
279
280
281
def _content_fields(
282
    operation: str,
283
    passed: bool,
284
    reason: str | None,
285
    source: str,
286
    expected_files: int,
287
    verified_files: int,
288
    failures: list[dict[str, str]],
289
    sources: set[str] | None = None,
290
) -> dict[str, Any]:
291
    source_list = sorted(sources or [])
292
    reported_source = source_list[0] if len(source_list) == 1 else "mixed_payload_sources" if source_list else source
293
    public_trust_passed = passed and content_integrity_source_supports_public_trust(
294
        reported_source,
295
        source_list,
296
    )
297
    return {
298
        "content_integrity_check_passed": passed,
299
        "content_integrity_failure_reason": reason,
300
        "content_integrity_source": reported_source,
301
        "content_integrity_sources": source_list,
302
        "content_integrity_public_trust_passed": public_trust_passed,
303
        "content_integrity_operation": operation,
304
        "content_expected_files": expected_files,
305
        "content_verified_files": verified_files,
306
        "content_mismatches": failures[:CONTENT_FAILURE_LIMIT],
307
    }
308
309
310
def _content_failure_reason(failures: list[dict[str, str]]) -> str:
311
    if not failures:
312
        return "content_payload_check_failed"
313
    first = failures[0]
314
    path = first.get("path", "?")
315
    reason = first.get("reason", "content_payload_check_failed")
316
    return f"content_payload_{reason}:{path}"
317
318
319
def _oak_committed_payloads(repo: Path, paths: list[str]):
320
    """Committed HEAD bytes of an Oak repository via the independent reader.
321
322
    Returns None when the repository is not Oak or the reader is unavailable
323
    (format drift, bounds): the caller then falls back to the weak worktree
324
    source, which never earns public trust. Positively invalid stored state
325
    fails every requested path.
326
    """
327
    if not (repo / ".oak" / "oak.db").is_file() or (repo / ".git").exists():
328
        return None
329
    from oakbench import oak_sqlite_reader as reader
330
    scope = [rel for rel in paths if _safe_relative_path(rel) is not None]
331
    if not scope:
332
        return None
333
    try:
334
        with tempfile.TemporaryDirectory(prefix="oakbench-integrity-read-") as temporary:
335
            destination = Path(temporary) / "tree"
336
            result = reader.read_committed_tree(repo, destination, reader.normalize_scope(scope))
337
            payloads = {}
338
            for rel in scope:
339
                target = destination / rel
340
                if rel in result.get("nonregular_paths", []):
341
                    payloads[rel] = (None, OAK_READER_PAYLOAD_SOURCE, f"oak_committed_payload_not_regular:{rel}")
342
                elif target.is_file() and not target.is_symlink():
343
                    payloads[rel] = (target.read_bytes(), OAK_READER_PAYLOAD_SOURCE, None)
344
            return payloads
345
    except reader.Invalid as exc:
346
        reason = f"oak_committed_state_invalid:{exc}"
347
        return {rel: (None, OAK_READER_PAYLOAD_SOURCE, reason) for rel in scope}
348
    except (reader.Unavailable, OSError, ValueError):
349
        return None
350
351
352
def _read_attested_payload(repo: Path, rel: str) -> tuple[bytes | None, str, str | None]:
353
    if (repo / ".git").exists():
354
        git = shutil.which("git")
355
        if git is None:
356
            return None, "git_head_payload_sha256", "known_good_git_unavailable_for_payload_check"
357
        try:
358
            proc = subprocess.run(
359
                [git, "-C", str(repo), "show", f"HEAD:{rel}"],
360
                stdout=subprocess.PIPE,
361
                stderr=subprocess.PIPE,
362
                check=False,
363
            )
364
        except OSError as exc:
365
            return None, "git_head_payload_sha256", f"known_good_git_payload_read_error:{exc}"
366
        if proc.returncode != 0:
367
            return None, "git_head_payload_sha256", f"git_head_payload_missing:{rel}"
368
        if _looks_like_lfs_pointer(proc.stdout):
369
            try:
370
                return (repo / rel).read_bytes(), "git_lfs_worktree_payload_sha256", None
371
            except OSError as exc:
372
                return None, "git_lfs_worktree_payload_sha256", str(exc)
373
        return proc.stdout, "git_head_payload_sha256", None
374
375
    path = repo / rel
376
    try:
377
        return path.read_bytes(), "worktree_payload_sha256", None
378
    except OSError as exc:
379
        return None, "worktree_payload_sha256", str(exc)
380
381
382
def _safe_relative_path(rel: str) -> Path | None:
383
    candidate = Path(rel)
384
    if candidate.is_absolute() or ".." in candidate.parts:
385
        return None
386
    return candidate
387
388
389
def _file_sha256(path: Path) -> str:
390
    digest = hashlib.sha256()
391
    with path.open("rb") as fh:
392
        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
393
            digest.update(chunk)
394
    return digest.hexdigest()
395
396
397
def _looks_like_lfs_pointer(data: bytes) -> bool:
398
    return data.startswith(b"version https://git-lfs.github.com/spec/v1\n") and b"\noid sha256:" in data
399
400
401
def _hash_file(path: Path, rel: str, digest: "hashlib._Hash") -> None:
402
    digest.update(b"\0path:")
403
    digest.update(rel.encode("utf-8", "surrogateescape"))
404
    try:
405
        stat = path.lstat()
406
    except OSError:
407
        digest.update(b"\0missing")
408
        return
409
    if path.is_symlink():
410
        digest.update(b"\0symlink:")
411
        digest.update(os.readlink(path).encode("utf-8", "surrogateescape"))
412
        return
413
    if not path.is_file():
414
        digest.update(b"\0nonfile")
415
        return
416
    digest.update(b"\0size:")
417
    digest.update(str(stat.st_size).encode("ascii"))
418
    digest.update(b"\0content:")
419
    try:
420
        with path.open("rb") as fh:
421
            for chunk in iter(lambda: fh.read(1024 * 1024), b""):
422
                digest.update(chunk)
423
    except OSError as exc:
424
        digest.update(b"\0read_error:")
425
        digest.update(str(exc).encode("utf-8", "surrogateescape"))