Viewing files at commit f08b4a249972Commit detailsView latest

Browse files
scripts/oakbench/workflow_oracles.py 643 lines · 32.9 KB · python
1
"""Trusted exact task payload checks, outside the agent-editable checkout.
2
3
This module is stdlib-only and can run with Python -I as an external oracle.
4
It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
5
600 canonical service configs and four 16 MiB deterministic binary assets.
6
"""
7
from __future__ import annotations
8
9
from functools import lru_cache
10
import hashlib
11
import json
12
import os
13
from pathlib import Path
14
import sys
15
import shutil
16
import subprocess
17
import tempfile
18
import re
19
20
if __package__ in (None, ""):
21
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
from oakbench.environment import base_env
23
24
VERSION = "2026-09-28.2"
25
EXACT_SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
26
COMMITTED_PAYLOADS = {
27
    "bugfix_test_loop": {"app/pricing.py": "return cents - ((cents * percent) // 100)"},
28
    "vcs_error_recovery": {"README.md": "# recovered"},
29
    "sync_push_divergence": {"work.txt": "local change", "notes.txt": "upstream change"},
30
    "sync_pull_upstream": {"notes.txt": "upstream change"},
31
    "sync_pull_dirty": {"notes.txt": "upstream change"},
32
    "sync_non_ff_after_amend": {"work.txt": "amended line"},
33
}
34
SCENARIOS = EXACT_SCENARIOS | frozenset(COMMITTED_PAYLOADS) | {"history_archaeology"}
35
PAYLOAD_LIMIT_BYTES = 1024 * 1024
36
ASSET_BYTES = 16 * 1024 * 1024
37
38
# Preserved fixture instructions, not programs executed by this oracle.
39
CONFIG_VALIDATOR = "\n".join([
40
    "from pathlib import Path", "bad = []",
41
    "for p in Path('services').glob('*/config.toml'):",
42
    "    text = p.read_text()", "    if 'timeout_ms = 2500' in text:",
43
    "        bad.append(str(p))", "print(f'bad={len(bad)}')",
44
    "raise SystemExit(1 if bad else 0)", "",
45
]).encode()
46
ASSET_VALIDATOR = "\n".join([
47
    "import json", "from pathlib import Path",
48
    "data = json.loads(Path('manifest.json').read_text())",
49
    "for item in data['assets']:", "    p = Path(item['path'])",
50
    "    assert p.exists(), p", "    assert p.stat().st_size == item['bytes'], p",
51
    "    assert item['reviewed'] is True, p", "print('manifest ok')", "",
52
]).encode()
53
54
55
EXACT_SCOPES = {
56
    "wide_config_refactor": ("services", "validate_configs.py"),
57
    "large_asset_manifest": ("assets", "manifest.json", "validate_manifest.py"),
58
}
59
60
61
def committed_blob_limit(scenario: str) -> int:
62
    """Largest committed blob the Git check accepts for a scenario."""
63
    return {"wide_config_refactor": 4096, "large_asset_manifest": ASSET_BYTES}.get(scenario, PAYLOAD_LIMIT_BYTES)
64
65
66
def committed_scope(scenario: str) -> tuple[str, ...] | None:
67
    """The committed paths every reader checks for a scenario, Git or Oak.
68
69
    One definition, so the Git ls-tree check and the independent Oak reader
70
    read the SAME scope at the same strength and may share an evidence class.
71
    """
72
    if scenario in COMMITTED_PAYLOADS:
73
        return tuple(sorted(COMMITTED_PAYLOADS[scenario]))
74
    return EXACT_SCOPES.get(scenario)
75
76
77
class _InstrumentUnavailable(Exception):
78
    """The instrument could not run to completion (signal); never a subject verdict."""
79
80
81
def _regular_file(path: Path) -> bool:
82
    return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
83
84
85
def _tree_paths(root: Path) -> set[str]:
86
    """Return every descendant identity, rejecting links and special files."""
87
    if root.is_symlink() or not root.is_dir():
88
        raise ValueError("payload_directory_missing_or_link")
89
    result = set()
90
    for parent, dirs, files in os.walk(root, followlinks=False):
91
        for name in dirs + files:
92
            path = Path(parent) / name
93
            if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
94
                raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
95
            result.add(path.relative_to(root).as_posix())
96
    return result
97
98
99
def _config_failure(repo: Path) -> str | None:
100
    expected_paths = {f"service-{i:04d}" for i in range(600)}
101
    expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
102
    actual = _tree_paths(repo / "services")
103
    if actual != expected_paths:
104
        return "config_path_set_mismatch"
105
    for i in range(600):
106
        rel = f"services/service-{i:04d}/config.toml"
107
        expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
108
        if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
109
            return "config_payload_mismatch:" + rel
110
    return None
111
112
113
def _unique_object(pairs):
114
    result = {}
115
    for key, value in pairs:
116
        if key in result:
117
            raise ValueError("duplicate_json_key:" + key)
118
        result[key] = value
119
    return result
120
121
122
@lru_cache(maxsize=4)
123
def _expected_asset_hash(index: int) -> str:
124
    unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
125
    digest = hashlib.sha256()
126
    remaining = ASSET_BYTES
127
    while remaining:
128
        chunk = unit[:min(len(unit), remaining)]
129
        digest.update(chunk)
130
        remaining -= len(chunk)
131
    return digest.hexdigest()
132
133
134
def _asset_failure(repo: Path) -> str | None:
135
    expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
136
    if _tree_paths(repo / "assets") != expected_paths:
137
        return "asset_path_set_mismatch"
138
    manifest = repo / "manifest.json"
139
    if not _regular_file(manifest):
140
        return "manifest_missing_or_wrong_type"
141
    if manifest.stat().st_size > 64 * 1024:
142
        return "manifest_size_out_of_bounds"
143
    data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
144
    if not isinstance(data, dict) or set(data) != {"assets"}:
145
        return "manifest_object_shape"
146
    entries = data["assets"]
147
    if not isinstance(entries, list) or len(entries) != 4:
148
        return "manifest_asset_count"
149
    seen = set()
150
    for item in entries:
151
        if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
152
            return "manifest_entry_shape"
153
        path = item["path"]
154
        if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
155
            return "manifest_asset_identity"
156
        if path in seen:
157
            return "manifest_duplicate_asset:" + path
158
        seen.add(path)
159
        if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
160
            return "manifest_asset_fields:" + path
161
    for index in range(4):
162
        path = repo / "assets" / f"asset-{index:02d}.bin"
163
        if path.stat().st_size != ASSET_BYTES:
164
            return "asset_size_mismatch:" + path.name
165
        digest = hashlib.sha256()
166
        with path.open("rb") as stream:
167
            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
168
                digest.update(chunk)
169
        if digest.hexdigest() != _expected_asset_hash(index):
170
            return "asset_content_mismatch:" + path.name
171
    return None
172
173
174
def validate(scenario: str, repo: Path) -> str | None:
175
    """Return a concrete failure reason, or None on a fully verified task."""
176
    if scenario not in SCENARIOS:
177
        return "unsupported_trusted_task_oracle:" + scenario
178
    if scenario == "history_archaeology":
179
        return "history_answer_oracle_unavailable"
180
    if scenario in COMMITTED_PAYLOADS:
181
        expected = dict(COMMITTED_PAYLOADS[scenario])
182
        if scenario == "sync_pull_dirty":
183
            expected["work.txt"] = "dirty local edit"
184
        try:
185
            for relative, marker in expected.items():
186
                path = repo / relative
187
                if (any((repo.joinpath(*Path(relative).parts[:index])).is_symlink() for index in range(1, len(Path(relative).parts) + 1))
188
                        or not _regular_file(path) or path.stat().st_size > PAYLOAD_LIMIT_BYTES
189
                        or marker not in path.read_text()):
190
                    return "worktree_payload_missing:" + relative
191
        except (OSError, UnicodeError):
192
            return "worktree_payload_unreadable"
193
        return None
194
    name, expected_validator = (
195
        ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
196
        else ("validate_manifest.py", ASSET_VALIDATOR)
197
    )
198
    try:
199
        path = repo / name
200
        if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
201
                or path.read_bytes() != expected_validator):
202
            return "fixture_validator_modified_or_missing:" + name
203
        return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
204
    except (OSError, ValueError, UnicodeError, RecursionError) as exc:
205
        return "trusted_payload_unreadable_or_invalid:" + str(exc)
206
207
208
def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
209
    """Resolve an explicitly configured instrument or the invoking PATH once."""
210
    requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
211
    resolved = shutil.which(requested)
212
    # The xcrun trampoline execs the real Git; identify the binary that runs
213
    # so evidence classes compare the same bytes the Git subject executes.
214
    from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
215
    if resolved and is_xcrun_git_shim(resolved):
216
        real = resolve_real_git()
217
        resolved = str(real) if real else resolved
218
    metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
219
                "git_head_instrument_version": None}
220
    if resolved is None:
221
        return None, metadata
222
    binary = str(Path(resolved).resolve())
223
    try:
224
        version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
225
        if version.returncode:
226
            return None, metadata
227
        metadata.update(git_head_instrument_binary=binary,
228
                        git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
229
                        git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
230
        return binary, metadata
231
    except (OSError, subprocess.TimeoutExpired):
232
        return None, metadata
233
234
235
def git_head_text(repo: Path, path: str, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> tuple[str | None, str | None]:
236
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
237
    if git is None:
238
        return None, "git_head_oracle_unavailable"
239
    try:
240
        result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
241
                                env=base_env(), capture_output=True, timeout=30)
242
        if result.returncode < 0:
243
            return None, "git_head_oracle_unavailable"
244
        return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
245
    except (OSError, subprocess.TimeoutExpired):
246
        return None, "git_head_oracle_unavailable"
247
248
249
def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> str | None:
250
    """Verify the entire task subtree from committed objects using system Git.
251
252
    Never checks out subject-controlled hooks/filters or extracts an archive.
253
    Enumerated blob paths and bounded sizes are checked before materialization.
254
    Git object replacement and inherited Git environment overrides are disabled.
255
    """
256
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
257
    if git is None:
258
        return "git_head_oracle_unavailable"
259
    if scenario in COMMITTED_PAYLOADS:
260
        try:
261
            expected = COMMITTED_PAYLOADS[scenario]
262
            command = [git, "-C", str(repo.resolve())]
263
            # One tree read binds every required path to the same committed tree.
264
            tree = subprocess.run([*command, "ls-tree", "-r", "-z", "HEAD", "--", *committed_scope(scenario)],
265
                env=base_env(), capture_output=True, timeout=30)
266
            if tree.returncode < 0:
267
                return "git_head_oracle_unavailable"
268
            if tree.returncode:
269
                return "git_head_tree_unreadable"
270
            entries = {}
271
            for raw in tree.stdout.split(b"\0"):
272
                if not raw:
273
                    continue
274
                header, relative = raw.split(b"\t", 1)
275
                mode, kind, oid = header.split()
276
                path = relative.decode("utf-8")
277
                if mode != b"100644" or kind != b"blob":
278
                    return "committed_payload_path_type:" + path
279
                if path not in expected or path in entries:
280
                    return "committed_payload_path_set_mismatch"
281
                if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
282
                    return "git_head_invalid_object_id"
283
                entries[path] = oid.decode("ascii")
284
            if set(entries) != set(expected):
285
                return "committed_payload_path_set_mismatch"
286
            for path, marker in expected.items():
287
                oid = entries[path]
288
                size = subprocess.run([*command, "cat-file", "-s", oid],
289
                    env=base_env(), capture_output=True, timeout=30)
290
                if size.returncode < 0:
291
                    return "git_head_oracle_unavailable"
292
                if size.returncode or not size.stdout.strip().isdigit():
293
                    return "committed_payload_missing:" + path
294
                length = int(size.stdout.strip())
295
                if length > PAYLOAD_LIMIT_BYTES:
296
                    return "committed_payload_over_limit:" + path
297
                blob = subprocess.run([*command, "cat-file", "blob", oid],
298
                    env=base_env(), capture_output=True, timeout=30)
299
                if blob.returncode < 0:
300
                    return "git_head_oracle_unavailable"
301
                if blob.returncode or len(blob.stdout) != length:
302
                    return "git_head_blob_unreadable"
303
                try:
304
                    text = blob.stdout.decode("utf-8")
305
                except UnicodeError:
306
                    return "committed_payload_invalid_utf8:" + path
307
                if marker not in text:
308
                    return "committed_payload_missing:" + path
309
            return None
310
        except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired):
311
            return "git_head_oracle_unavailable"
312
    if scenario == "wide_config_refactor":
313
        paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
314
        paths.add("validate_configs.py")
315
        scope = list(committed_scope(scenario))
316
        max_blob_size = 4096
317
    elif scenario == "large_asset_manifest":
318
        paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
319
        paths.update({"manifest.json", "validate_manifest.py"})
320
        scope = list(committed_scope(scenario))
321
        max_blob_size = ASSET_BYTES
322
    else:
323
        return "unsupported_trusted_task_oracle:" + scenario
324
    env = base_env()
325
    command = [git, "-C", str(repo.resolve())]
326
    try:
327
        tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
328
                              env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
329
        if tree.returncode < 0:
330
            return "git_head_oracle_unavailable"
331
        if tree.returncode:
332
            return "git_head_tree_unreadable"
333
        entries = {}
334
        for raw in tree.stdout.split(b"\0"):
335
            if not raw:
336
                continue
337
            header, rel = raw.split(b"\t", 1)
338
            mode, kind, oid = header.split()
339
            path = rel.decode("utf-8")
340
            if mode != b"100644" or kind != b"blob":
341
                return "git_head_path_type:" + path
342
            if path not in paths or path in entries:
343
                return "git_head_path_set_mismatch"
344
            if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
345
                return "git_head_invalid_object_id"
346
            entries[path] = oid
347
        if set(entries) != paths:
348
            return "git_head_path_set_mismatch"
349
        ordered = sorted(entries)
350
        request = b"\n".join(entries[path] for path in ordered) + b"\n"
351
        sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
352
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
353
        if sizes.returncode < 0:
354
            return "git_head_oracle_unavailable"
355
        if sizes.returncode:
356
            return "git_head_blob_metadata_unreadable"
357
        headers = sizes.stdout.splitlines()
358
        if len(headers) != len(ordered):
359
            return "git_head_blob_metadata_incomplete"
360
        for header in headers:
361
            fields = header.split()
362
            if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
363
                return "git_head_blob_metadata_invalid"
364
            if int(fields[2]) > max_blob_size:
365
                return "git_head_blob_size_out_of_bounds"
366
        blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
367
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
368
        if blobs.returncode < 0:
369
            return "git_head_oracle_unavailable"
370
        if blobs.returncode:
371
            return "git_head_blob_unreadable"
372
        offset = 0
373
        with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
374
            root = Path(tmp)
375
            for path, expected_header in zip(ordered, headers):
376
                end = blobs.stdout.find(b"\n", offset)
377
                if end < 0 or blobs.stdout[offset:end] != expected_header:
378
                    return "git_head_blob_framing_invalid"
379
                size = int(expected_header.split()[2])
380
                offset = end + 1
381
                payload = blobs.stdout[offset:offset + size]
382
                if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
383
                    return "git_head_blob_framing_invalid"
384
                offset += size + 1
385
                destination = root / path
386
                destination.parent.mkdir(parents=True, exist_ok=True)
387
                destination.write_bytes(payload)
388
            if offset != len(blobs.stdout):
389
                return "git_head_blob_trailing_bytes"
390
            reason = validate(scenario, root)
391
            return "git_head_" + reason if reason else None
392
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
393
        return "git_head_oracle_error:" + str(exc)
394
395
396
def reader_evidence(kind: str, binary: str | None, metadata: dict) -> dict:
397
    """Classify an executed reader by observed binary bytes, never its label."""
398
    subject_hash = metadata.get("workflow_subject_binary_sha256")
399
    if kind == "oak":
400
        subject_hash = subject_hash or metadata.get("oak_export_instrument_sha256")
401
    try:
402
        resolved = shutil.which(binary or kind) if subject_hash is None else None
403
        if resolved and kind == "git":
404
            # The xcrun shim execs the real Git: hash the binary that runs,
405
            # exactly as resolve_git_instrument does for the instrument.
406
            from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
407
            if is_xcrun_git_shim(resolved):
408
                real = resolve_real_git()
409
                resolved = str(real) if real else resolved
410
        if resolved:
411
            if str(Path(resolved).resolve()) == metadata.get("git_head_instrument_binary"):
412
                subject_hash = metadata.get("git_head_instrument_sha256")
413
            else:
414
                subject_hash = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
415
    except OSError:
416
        pass
417
    instrument_hash = metadata.get("git_head_instrument_sha256")
418
    evidence_class = "unverified_committed_tree"
419
    if kind == "oak":
420
        evidence_class = "subject_produced_committed_tree"
421
    elif subject_hash and instrument_hash:
422
        evidence_class = "subject_binary_committed_tree" if subject_hash == instrument_hash else "independent_committed_tree"
423
    return {"workflow_subject_binary_sha256": subject_hash, "workflow_integrity_evidence_class": evidence_class}
424
425
426
def reader_timeout_s(repo: Path) -> float:
427
    """Subprocess budget for the independent reader, scaled by stored bytes.
428
429
    Matches the reader's own byte-scaled deadline (60 s + 1.5 s/MiB verified)
430
    with margin; the database size bounds the bytes it can verify.
431
    """
432
    try:
433
        size = (Path(repo) / ".oak" / "oak.db").stat().st_size
434
    except OSError:
435
        size = 0
436
    return 120 + 3.0 * size / (1024 * 1024)
437
438
439
def independent_oak_evidence(scenario: str, repo: Path, python: str, evidence: dict) -> dict:
440
    """Run a separately configured read-only decoder; never invoke Oak here."""
441
    from oakbench.oak_sqlite_reader import READER_VERSION
442
    helper = Path(__file__).with_name("oak_sqlite_reader.py")
443
    binary = shutil.which(python)
444
    if binary is None:
445
        evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_python_unavailable"
446
        return evidence
447
    evidence["workflow_integrity_source"] += ";independent_oak_sqlite_read_attempt"
448
    evidence["workflow_integrity_evidence_class"] = "independent_committed_read_attempt"
449
    try:
450
        helper_hash = hashlib.sha256(helper.read_bytes()).hexdigest()
451
        evidence["oak_reader_python_sha256"] = hashlib.sha256(Path(binary).read_bytes()).hexdigest()
452
        with tempfile.TemporaryDirectory(prefix="oakbench-independent-read-") as temporary:
453
            tree = Path(temporary) / "tree"
454
            scope = committed_scope(scenario)
455
            evidence["workflow_integrity_committed_scope"] = list(scope)
456
            result = subprocess.run([binary, "-I", str(helper), str(repo), str(tree),
457
                                     "--max-blob-bytes", str(committed_blob_limit(scenario)), "--scope", *scope],
458
                env=base_env(), capture_output=True, timeout=reader_timeout_s(repo))
459
            metadata = json.loads(result.stdout)
460
            if result.returncode == 77:
461
                evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_unavailable:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
462
                return evidence
463
            if result.returncode != 0:
464
                # A decoder failure is an instrument gap unless its contract
465
                # positively identifies invalid stored content.
466
                if result.returncode == 1 and metadata.get("corrupt") is True:
467
                    evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason="oak_stored_content_invalid:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure")))))
468
                else:
469
                    evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_failed:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
470
                return evidence
471
            if (metadata.get("reader_version") != READER_VERSION or metadata.get("reader_sha256") != helper_hash
472
                    or not re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("head", "")))
473
                    or metadata.get("scope") != list(scope)):
474
                raise ValueError("oak_reader_identity_mismatch")
475
            evidence.update({"oak_reader_version": READER_VERSION, "oak_reader_sha256": helper_hash,
476
                "oak_reader_head": metadata["head"], "oak_reader_backend": metadata.get("reader_backend"),
477
                "oak_reader_codec_sha256": metadata.get("codec_sha256"),
478
                "oak_reader_blake3_version": metadata.get("blake3_version"),
479
                "oak_reader_zstandard_version": metadata.get("zstandard_version")})
480
            nonregular = metadata.get("nonregular_paths") or []
481
            oversized = metadata.get("oversized_paths") or []
482
            collisions = metadata.get("collision_paths") or []
483
            exact = scenario in EXACT_SCENARIOS
484
            if nonregular:
485
                # Same outcome as Git's scoped mode check (mode != 100644).
486
                reason = ("git_head_path_type:" if exact else "committed_payload_path_type:") + str(nonregular[0])
487
            elif collisions:
488
                reason = "git_head_path_set_mismatch" if exact else "committed_payload_path_set_mismatch"
489
            elif oversized:
490
                reason = "git_head_blob_size_out_of_bounds" if exact else "committed_payload_over_limit:" + str(oversized[0])
491
            elif scenario in EXACT_SCENARIOS:
492
                reason = validate(scenario, tree)
493
            else:
494
                reason = None
495
                for relative, marker in COMMITTED_PAYLOADS[scenario].items():
496
                    path = tree / relative
497
                    # Reason strings mirror validate_git_head's so the two
498
                    # readers report the same failure for the same content.
499
                    if not path.exists() and not path.is_symlink():
500
                        reason = "committed_payload_path_set_mismatch"
501
                        break
502
                    if not _regular_file(path):
503
                        reason = "committed_payload_path_type:" + relative
504
                        break
505
                    if path.stat().st_size > PAYLOAD_LIMIT_BYTES:
506
                        reason = "committed_payload_over_limit:" + relative
507
                        break
508
                    try:
509
                        text = path.read_text(encoding="utf-8")
510
                    except UnicodeError:
511
                        reason = "committed_payload_invalid_utf8:" + relative
512
                        break
513
                    if marker not in text:
514
                        reason = "committed_payload_missing:" + relative
515
                        break
516
            evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
517
            if reason is None:
518
                evidence["workflow_integrity_evidence_class"] = "independent_committed_tree" if scenario in EXACT_SCENARIOS else "independent_committed_payload"
519
                evidence["workflow_integrity_source"] = "trusted_task_worktree;independent_oak_sqlite_reader"
520
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
521
        evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_instrument_error:" + str(exc)
522
    return evidence
523
524
525
def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None, oak_reader_python: str | None = None) -> dict:
526
    source = "trusted_exact_task_worktree"
527
    evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
528
                "workflow_integrity_evidence_class": "worktree_payload",
529
                "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
530
                "workflow_integrity_unmeasured_reason": None}
531
    reason = validate(scenario, repo)
532
    if reason == "history_answer_oracle_unavailable":
533
        evidence["workflow_integrity_evidence_class"] = None
534
        evidence["workflow_integrity_unmeasured_reason"] = reason
535
        return evidence
536
    if reason:
537
        evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
538
        return evidence
539
    configured_reader = oak_reader_python or os.environ.get("OAK_BENCH_OAK_READER_PYTHON")
540
    if kind == "oak" and not configured_reader and os.environ.get("OAK_BENCH_OAK_COMMITTED_READER") != "export":
541
        # The reader is stdlib-only (pure BLAKE3/zstd): the harness interpreter
542
        # runs it, isolated (-I). Oak export (subject-produced evidence) is an
543
        # explicit opt-in only: OAK_BENCH_OAK_COMMITTED_READER=export.
544
        configured_reader = sys.executable
545
    if kind == "oak" and configured_reader:
546
        return independent_oak_evidence(scenario, repo, configured_reader, evidence)
547
    if kind in {"git", "oak"}:
548
        git, metadata = resolve_git_instrument(git_instrument)
549
        evidence.update(metadata)
550
        if git is None:
551
            evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
552
            return evidence
553
        evidence["workflow_integrity_committed_scope"] = list(committed_scope(scenario) or ())
554
        if kind == "git":
555
            evidence["workflow_integrity_evidence_class"] = "committed_tree_attempt"
556
            evidence["workflow_integrity_source"] = source + ";git_head_read_attempt"
557
            reason = validate_git_head(scenario, repo, resolved_instrument=(git, metadata))
558
        else:
559
            oak = shutil.which(binary or "oak")
560
            if oak is None:
561
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
562
                return evidence
563
            oak = str(Path(oak).resolve())
564
            evidence["oak_export_instrument_binary"] = oak
565
            # Same classification as the Git arm (validate_git_head): a
566
            # positive nonzero exit or unusable output from the subject, or
567
            # subject-controlled repository state, is a MEASURED failure; only
568
            # an instrument that cannot run (launch error, signal, timeout) is
569
            # unmeasured. A broken `oak status`/`oak export` must never turn an
570
            # Oak failure into a skip.
571
            reason = None
572
            try:
573
                evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
574
575
                def run(command, timeout):
576
                    result = subprocess.run(command, cwd=repo, env=base_env(), capture_output=True, timeout=timeout)
577
                    if result.returncode < 0:
578
                        raise _InstrumentUnavailable("oak_instrument_signalled")
579
                    return result
580
581
                def identity(result):
582
                    if result.returncode:
583
                        return None, None
584
                    try:
585
                        data = json.loads(result.stdout)
586
                    except ValueError:
587
                        return None, None
588
                    if not isinstance(data, dict):
589
                        return None, None
590
                    return data.get("head"), data.get("branch")
591
592
                head, branch = identity(run([oak, "status", "--json"], 30))
593
                if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
594
                    reason = "oak_head_identity_unreadable"
595
                else:
596
                    evidence["oak_export_head"] = head
597
                    with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
598
                        destination = Path(tmp) / "export"
599
                        evidence["workflow_integrity_evidence_class"] = "subject_produced_committed_tree_attempt"
600
                        evidence["workflow_integrity_source"] = source + ";subject_produced_oak_head_export_attempt"
601
                        exported = run([oak, "export", "--branch", branch, str(destination)], 120)
602
                        if exported.returncode:
603
                            reason = "oak_head_export_failed"
604
                        elif identity(run([oak, "status", "--json"], 30)) != (head, branch):
605
                            reason = "oak_head_changed_during_export"
606
                        else:
607
                            reason = validate_git_head(scenario, destination, resolved_instrument=(git, metadata))
608
            except (OSError, subprocess.TimeoutExpired, _InstrumentUnavailable) as exc:
609
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_error:" + str(exc)
610
                return evidence
611
    if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
612
        evidence["workflow_integrity_unmeasured_reason"] = reason
613
        return evidence
614
    if reason is None and kind in {"git", "oak"}:
615
        evidence.update(reader_evidence(kind, binary, evidence))
616
        if scenario not in EXACT_SCENARIOS:
617
            evidence["workflow_integrity_evidence_class"] = evidence["workflow_integrity_evidence_class"].replace("_tree", "_payload")
618
        suffix = "subject_produced_oak_head_export" if kind == "oak" else evidence["workflow_integrity_evidence_class"]
619
        evidence["workflow_integrity_source"] = source + ";" + suffix
620
    evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
621
    return evidence
622
623
624
def main() -> int:
625
    import argparse
626
    parser = argparse.ArgumentParser()
627
    parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
628
    parser.add_argument("--git-head", action="store_true")
629
    parser.add_argument("--oak-head", action="store_true")
630
    parser.add_argument("--binary")
631
    parser.add_argument("--evidence-json", action="store_true")
632
    parser.add_argument("--git-instrument")
633
    parser.add_argument("--oak-reader-python")
634
    args = parser.parse_args()
635
    evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument, args.oak_reader_python)
636
    reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
637
    print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
638
    passed = evidence["workflow_integrity_passed"]
639
    return 77 if passed is None else 0 if passed else 1
640
641
642
if __name__ == "__main__":
643
    raise SystemExit(main())