Viewing files at commit c8b43cc46596Commit detailsView latest

scripts/oakbench/integrity.py 385 lines · 13.2 KB · python
1
"""Independent repository-state checks for benchmark correctness.
2
3
These helpers inspect durable VCS metadata on disk. They deliberately do not
4
parse measured command output, so a subject cannot satisfy them by printing a
5
successful-looking transcript.
6
"""
7
8
from __future__ import annotations
9
10
import hashlib
11
import os
12
import shutil
13
import subprocess
14
from pathlib import Path
15
from typing import Any, Mapping
16
17
METADATA_DIRS = (".git", ".oak")
18
PAYLOAD_SAMPLE_LIMIT = 3
19
CONTENT_FAILURE_LIMIT = 20
20
STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES = frozenset({"git_head_payload_sha256"})
21
22
23
def content_integrity_source_supports_public_trust(
24
    source: object,
25
    sources: object = None,
26
) -> bool:
27
    """Return whether a content oracle source can back a public trust claim.
28
29
    Worktree-backed checks are still useful measurements, but only committed
30
    HEAD bytes read through known-good Git are strong enough for public content
31
    trust. Mixed sources inherit their weakest member.
32
    """
33
    if isinstance(sources, list):
34
        normalized_sources = [item for item in sources if isinstance(item, str)]
35
    else:
36
        normalized_sources = []
37
    if not normalized_sources and isinstance(source, str):
38
        normalized_sources = [source]
39
    return bool(normalized_sources) and all(
40
        item in STRONG_PUBLIC_CONTENT_INTEGRITY_SOURCES for item in normalized_sources
41
    )
42
43
44
def content_integrity_public_trust_passed(row: Mapping[str, Any]) -> bool:
45
    return (
46
        row.get("content_integrity_check_passed") is True
47
        and content_integrity_source_supports_public_trust(
48
            row.get("content_integrity_source"),
49
            row.get("content_integrity_sources"),
50
        )
51
    )
52
53
54
def parse_manifest(text: str) -> dict[str, str]:
55
    """Parse MANIFEST.sha256 lines in sha256sum format."""
56
    entries: dict[str, str] = {}
57
    for line in text.splitlines():
58
        stripped = line.strip()
59
        if not stripped:
60
            continue
61
        digest, separator, rel = stripped.partition("  ")
62
        if not separator:
63
            digest, separator, rel = stripped.partition(" ")
64
        if separator and len(digest) == 64:
65
            entries[rel.strip()] = digest.lower()
66
    return entries
67
68
69
def metadata_path(repo: Path) -> Path | None:
70
    for name in METADATA_DIRS:
71
        candidate = repo / name
72
        if candidate.exists():
73
            return candidate
74
    return None
75
76
77
def repository_state_fingerprint(repo: Path) -> str | None:
78
    root = metadata_path(repo)
79
    if root is None:
80
        return None
81
82
    digest = hashlib.sha256()
83
    digest.update(root.name.encode("utf-8", "surrogateescape"))
84
    if root.is_file():
85
        _hash_file(root, root.name, digest)
86
        return digest.hexdigest()
87
88
    for dirpath, dirnames, filenames in os.walk(root):
89
        dirnames[:] = sorted(name for name in dirnames if not name.endswith(".lock"))
90
        for filename in sorted(name for name in filenames if not name.endswith(".lock")):
91
            path = Path(dirpath) / filename
92
            rel = path.relative_to(root).as_posix()
93
            _hash_file(path, rel, digest)
94
    return digest.hexdigest()
95
96
97
def state_change_fields(
98
    repo: Path,
99
    before_fingerprint: str | None,
100
    *,
101
    operation: str,
102
    process_returncode: int,
103
    source: str = "direct_repo_metadata_fingerprint",
104
) -> dict[str, Any]:
105
    after_fingerprint = repository_state_fingerprint(repo)
106
    passed = process_returncode == 0 and before_fingerprint is not None and after_fingerprint is not None
107
    if passed:
108
        passed = before_fingerprint != after_fingerprint
109
110
    reason: str | None = None
111
    if process_returncode != 0:
112
        reason = "process_returncode_nonzero"
113
    elif before_fingerprint is None:
114
        reason = "repository_metadata_missing_before_operation"
115
    elif after_fingerprint is None:
116
        reason = "repository_metadata_missing_after_operation"
117
    elif before_fingerprint == after_fingerprint:
118
        reason = "repository_metadata_unchanged_after_successful_operation"
119
120
    return {
121
        "integrity_check_passed": passed,
122
        "integrity_failure_reason": reason,
123
        "integrity_check_source": source,
124
        "integrity_operation": operation,
125
        "repo_state_before": before_fingerprint,
126
        "repo_state_after": after_fingerprint,
127
    }
128
129
130
def content_payload_fields(
131
    repo: Path,
132
    expected_payloads: list[tuple[str, str]],
133
) -> dict[str, Any]:
134
    """Check required worktree payloads for a specific workflow phase.
135
136
    Callers decide which payloads are phase-appropriate. This helper only
137
    verifies the requested path/needle pairs and reports exact missing paths.
138
    """
139
    missing: list[str] = []
140
    for rel_path, expected in expected_payloads:
141
        path = repo / rel_path
142
        try:
143
            text = path.read_text(errors="replace")
144
        except OSError:
145
            missing.append(rel_path)
146
            continue
147
        if expected not in text:
148
            missing.append(rel_path)
149
150
    reason = None
151
    if missing:
152
        reason = "content_payload_expected_payload_missing:" + ",".join(sorted(missing))
153
    return {
154
        "content_payload_check_passed": not missing,
155
        "content_payload_failure_reason": reason,
156
        "content_payload_expected_paths": [path for path, _needle in expected_payloads],
157
        "content_payload_missing_paths": missing,
158
        "content_payload_check_source": "worktree_content_payload_attestation",
159
    }
160
161
162
def sample_worktree_payload_hashes(repo: Path, limit: int = PAYLOAD_SAMPLE_LIMIT) -> dict[str, str]:
163
    """Hash a small deterministic payload sample outside VCS metadata dirs."""
164
    paths = []
165
    root = Path(repo)
166
    for path in sorted(root.rglob("*")):
167
        if len(paths) >= limit:
168
            break
169
        if not path.is_file():
170
            continue
171
        rel = path.relative_to(root)
172
        if set(rel.parts).intersection(METADATA_DIRS):
173
            continue
174
        paths.append(rel.as_posix())
175
    return worktree_payload_hashes(root, paths)
176
177
178
def worktree_payload_hashes(repo: Path, paths: list[str]) -> dict[str, str]:
179
    """Return sha256 hashes for explicit repo-relative worktree payloads."""
180
    hashes: dict[str, str] = {}
181
    root = Path(repo)
182
    for rel in paths:
183
        safe_rel = _safe_relative_path(rel)
184
        if safe_rel is None:
185
            continue
186
        path = root / safe_rel
187
        if not path.is_file():
188
            continue
189
        hashes[rel] = _file_sha256(path)
190
    return hashes
191
192
193
def content_integrity_fields(
194
    repo: Path,
195
    *,
196
    operation: str,
197
    process_returncode: int,
198
    expected_hashes: Mapping[str, str] | None = None,
199
    expected_payloads: Mapping[str, str | bytes] | None = None,
200
) -> dict[str, Any]:
201
    """Verify expected payload content independently of command output.
202
203
    Git rows read committed content from HEAD with the known-good system Git.
204
    Other repositories fall back to the worktree; paired with metadata-state
205
    attestation this still catches content corruption in snapshot-style rows.
206
    """
207
    hashes = {str(path): digest.lower() for path, digest in (expected_hashes or {}).items()}
208
    payloads = dict(expected_payloads or {})
209
    paths = sorted(set(hashes) | set(payloads))
210
    fallback_source = "content_payload_sha256"
211
    sources: set[str] = set()
212
    failures: list[dict[str, str]] = []
213
    verified = 0
214
215
    if process_returncode != 0:
216
        return _content_fields(
217
            operation,
218
            False,
219
            "process_returncode_nonzero",
220
            fallback_source,
221
            len(paths),
222
            verified,
223
            failures,
224
            sources,
225
        )
226
    if not paths:
227
        return _content_fields(
228
            operation,
229
            False,
230
            "content_payload_expectation_missing",
231
            fallback_source,
232
            0,
233
            verified,
234
            failures,
235
            sources,
236
        )
237
238
    for rel in paths:
239
        safe_rel = _safe_relative_path(rel)
240
        if safe_rel is None:
241
            failures.append({"path": rel, "reason": "unsafe_relative_path"})
242
            continue
243
        data, read_source, error = _read_attested_payload(Path(repo), rel)
244
        sources.add(read_source)
245
        if error is not None or data is None:
246
            failures.append({"path": rel, "reason": error or "payload_missing"})
247
            continue
248
        expected_hash = hashes.get(rel)
249
        if expected_hash is not None:
250
            actual_hash = hashlib.sha256(data).hexdigest()
251
            if actual_hash != expected_hash:
252
                failures.append(
253
                    {
254
                        "path": rel,
255
                        "reason": "sha256_mismatch",
256
                        "expected_sha256": expected_hash,
257
                        "actual_sha256": actual_hash,
258
                    }
259
                )
260
                continue
261
        expected_payload = payloads.get(rel)
262
        if expected_payload is not None:
263
            needle = expected_payload.encode("utf-8") if isinstance(expected_payload, str) else expected_payload
264
            if needle not in data:
265
                failures.append({"path": rel, "reason": "expected_payload_missing"})
266
                continue
267
        verified += 1
268
269
    passed = not failures and verified == len(paths)
270
    reason = None if passed else _content_failure_reason(failures)
271
    return _content_fields(operation, passed, reason, fallback_source, len(paths), verified, failures, sources)
272
273
274
def _content_fields(
275
    operation: str,
276
    passed: bool,
277
    reason: str | None,
278
    source: str,
279
    expected_files: int,
280
    verified_files: int,
281
    failures: list[dict[str, str]],
282
    sources: set[str] | None = None,
283
) -> dict[str, Any]:
284
    source_list = sorted(sources or [])
285
    reported_source = source_list[0] if len(source_list) == 1 else "mixed_payload_sources" if source_list else source
286
    public_trust_passed = passed and content_integrity_source_supports_public_trust(
287
        reported_source,
288
        source_list,
289
    )
290
    return {
291
        "content_integrity_check_passed": passed,
292
        "content_integrity_failure_reason": reason,
293
        "content_integrity_source": reported_source,
294
        "content_integrity_sources": source_list,
295
        "content_integrity_public_trust_passed": public_trust_passed,
296
        "content_integrity_operation": operation,
297
        "content_expected_files": expected_files,
298
        "content_verified_files": verified_files,
299
        "content_mismatches": failures[:CONTENT_FAILURE_LIMIT],
300
    }
301
302
303
def _content_failure_reason(failures: list[dict[str, str]]) -> str:
304
    if not failures:
305
        return "content_payload_check_failed"
306
    first = failures[0]
307
    path = first.get("path", "?")
308
    reason = first.get("reason", "content_payload_check_failed")
309
    return f"content_payload_{reason}:{path}"
310
311
312
def _read_attested_payload(repo: Path, rel: str) -> tuple[bytes | None, str, str | None]:
313
    if (repo / ".git").exists():
314
        git = shutil.which("git")
315
        if git is None:
316
            return None, "git_head_payload_sha256", "known_good_git_unavailable_for_payload_check"
317
        try:
318
            proc = subprocess.run(
319
                [git, "-C", str(repo), "show", f"HEAD:{rel}"],
320
                stdout=subprocess.PIPE,
321
                stderr=subprocess.PIPE,
322
                check=False,
323
            )
324
        except OSError as exc:
325
            return None, "git_head_payload_sha256", f"known_good_git_payload_read_error:{exc}"
326
        if proc.returncode != 0:
327
            return None, "git_head_payload_sha256", f"git_head_payload_missing:{rel}"
328
        if _looks_like_lfs_pointer(proc.stdout):
329
            try:
330
                return (repo / rel).read_bytes(), "git_lfs_worktree_payload_sha256", None
331
            except OSError as exc:
332
                return None, "git_lfs_worktree_payload_sha256", str(exc)
333
        return proc.stdout, "git_head_payload_sha256", None
334
335
    path = repo / rel
336
    try:
337
        return path.read_bytes(), "worktree_payload_sha256", None
338
    except OSError as exc:
339
        return None, "worktree_payload_sha256", str(exc)
340
341
342
def _safe_relative_path(rel: str) -> Path | None:
343
    candidate = Path(rel)
344
    if candidate.is_absolute() or ".." in candidate.parts:
345
        return None
346
    return candidate
347
348
349
def _file_sha256(path: Path) -> str:
350
    digest = hashlib.sha256()
351
    with path.open("rb") as fh:
352
        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
353
            digest.update(chunk)
354
    return digest.hexdigest()
355
356
357
def _looks_like_lfs_pointer(data: bytes) -> bool:
358
    return data.startswith(b"version https://git-lfs.github.com/spec/v1\n") and b"\noid sha256:" in data
359
360
361
def _hash_file(path: Path, rel: str, digest: "hashlib._Hash") -> None:
362
    digest.update(b"\0path:")
363
    digest.update(rel.encode("utf-8", "surrogateescape"))
364
    try:
365
        stat = path.lstat()
366
    except OSError:
367
        digest.update(b"\0missing")
368
        return
369
    if path.is_symlink():
370
        digest.update(b"\0symlink:")
371
        digest.update(os.readlink(path).encode("utf-8", "surrogateescape"))
372
        return
373
    if not path.is_file():
374
        digest.update(b"\0nonfile")
375
        return
376
    digest.update(b"\0size:")
377
    digest.update(str(stat.st_size).encode("ascii"))
378
    digest.update(b"\0content:")
379
    try:
380
        with path.open("rb") as fh:
381
            for chunk in iter(lambda: fh.read(1024 * 1024), b""):
382
                digest.update(chunk)
383
    except OSError as exc:
384
        digest.update(b"\0read_error:")
385
        digest.update(str(exc).encode("utf-8", "surrogateescape"))