Viewing files at commit 538751fb07e6Commit detailsView latest

Browse files
scripts/oakbench/workflow_oracles.py 623 lines · 31.9 KB · python
1
"""Trusted exact task payload checks, outside the agent-editable checkout.
2
3
This module is stdlib-only and can run with Python -I as an external oracle.
4
It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
5
600 canonical service configs and four 16 MiB deterministic binary assets.
6
"""
7
from __future__ import annotations
8
9
from functools import lru_cache
10
import hashlib
11
import json
12
import os
13
from pathlib import Path
14
import sys
15
import shutil
16
import subprocess
17
import tempfile
18
import re
19
20
if __package__ in (None, ""):
21
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
from oakbench.environment import base_env
23
24
VERSION = "2026-09-27.1"
25
EXACT_SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
26
COMMITTED_PAYLOADS = {
27
    "bugfix_test_loop": {"app/pricing.py": "return cents - ((cents * percent) // 100)"},
28
    "vcs_error_recovery": {"README.md": "# recovered"},
29
    "sync_push_divergence": {"work.txt": "local change", "notes.txt": "upstream change"},
30
    "sync_pull_upstream": {"notes.txt": "upstream change"},
31
    "sync_pull_dirty": {"notes.txt": "upstream change"},
32
    "sync_non_ff_after_amend": {"work.txt": "amended line"},
33
}
34
SCENARIOS = EXACT_SCENARIOS | frozenset(COMMITTED_PAYLOADS) | {"history_archaeology"}
35
PAYLOAD_LIMIT_BYTES = 1024 * 1024
36
ASSET_BYTES = 16 * 1024 * 1024
37
38
# Preserved fixture instructions, not programs executed by this oracle.
39
CONFIG_VALIDATOR = "\n".join([
40
    "from pathlib import Path", "bad = []",
41
    "for p in Path('services').glob('*/config.toml'):",
42
    "    text = p.read_text()", "    if 'timeout_ms = 2500' in text:",
43
    "        bad.append(str(p))", "print(f'bad={len(bad)}')",
44
    "raise SystemExit(1 if bad else 0)", "",
45
]).encode()
46
ASSET_VALIDATOR = "\n".join([
47
    "import json", "from pathlib import Path",
48
    "data = json.loads(Path('manifest.json').read_text())",
49
    "for item in data['assets']:", "    p = Path(item['path'])",
50
    "    assert p.exists(), p", "    assert p.stat().st_size == item['bytes'], p",
51
    "    assert item['reviewed'] is True, p", "print('manifest ok')", "",
52
]).encode()
53
54
55
EXACT_SCOPES = {
56
    "wide_config_refactor": ("services", "validate_configs.py"),
57
    "large_asset_manifest": ("assets", "manifest.json", "validate_manifest.py"),
58
}
59
60
61
def committed_blob_limit(scenario: str) -> int:
62
    """Largest committed blob the Git check accepts for a scenario."""
63
    return {"wide_config_refactor": 4096, "large_asset_manifest": ASSET_BYTES}.get(scenario, PAYLOAD_LIMIT_BYTES)
64
65
66
def committed_scope(scenario: str) -> tuple[str, ...] | None:
67
    """The committed paths every reader checks for a scenario, Git or Oak.
68
69
    One definition, so the Git ls-tree check and the independent Oak reader
70
    read the SAME scope at the same strength and may share an evidence class.
71
    """
72
    if scenario in COMMITTED_PAYLOADS:
73
        return tuple(sorted(COMMITTED_PAYLOADS[scenario]))
74
    return EXACT_SCOPES.get(scenario)
75
76
77
class _InstrumentUnavailable(Exception):
78
    """The instrument could not run to completion (signal); never a subject verdict."""
79
80
81
def _regular_file(path: Path) -> bool:
82
    return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
83
84
85
def _tree_paths(root: Path) -> set[str]:
86
    """Return every descendant identity, rejecting links and special files."""
87
    if root.is_symlink() or not root.is_dir():
88
        raise ValueError("payload_directory_missing_or_link")
89
    result = set()
90
    for parent, dirs, files in os.walk(root, followlinks=False):
91
        for name in dirs + files:
92
            path = Path(parent) / name
93
            if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
94
                raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
95
            result.add(path.relative_to(root).as_posix())
96
    return result
97
98
99
def _config_failure(repo: Path) -> str | None:
100
    expected_paths = {f"service-{i:04d}" for i in range(600)}
101
    expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
102
    actual = _tree_paths(repo / "services")
103
    if actual != expected_paths:
104
        return "config_path_set_mismatch"
105
    for i in range(600):
106
        rel = f"services/service-{i:04d}/config.toml"
107
        expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
108
        if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
109
            return "config_payload_mismatch:" + rel
110
    return None
111
112
113
def _unique_object(pairs):
114
    result = {}
115
    for key, value in pairs:
116
        if key in result:
117
            raise ValueError("duplicate_json_key:" + key)
118
        result[key] = value
119
    return result
120
121
122
@lru_cache(maxsize=4)
123
def _expected_asset_hash(index: int) -> str:
124
    unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
125
    digest = hashlib.sha256()
126
    remaining = ASSET_BYTES
127
    while remaining:
128
        chunk = unit[:min(len(unit), remaining)]
129
        digest.update(chunk)
130
        remaining -= len(chunk)
131
    return digest.hexdigest()
132
133
134
def _asset_failure(repo: Path) -> str | None:
135
    expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
136
    if _tree_paths(repo / "assets") != expected_paths:
137
        return "asset_path_set_mismatch"
138
    manifest = repo / "manifest.json"
139
    if not _regular_file(manifest):
140
        return "manifest_missing_or_wrong_type"
141
    if manifest.stat().st_size > 64 * 1024:
142
        return "manifest_size_out_of_bounds"
143
    data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
144
    if not isinstance(data, dict) or set(data) != {"assets"}:
145
        return "manifest_object_shape"
146
    entries = data["assets"]
147
    if not isinstance(entries, list) or len(entries) != 4:
148
        return "manifest_asset_count"
149
    seen = set()
150
    for item in entries:
151
        if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
152
            return "manifest_entry_shape"
153
        path = item["path"]
154
        if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
155
            return "manifest_asset_identity"
156
        if path in seen:
157
            return "manifest_duplicate_asset:" + path
158
        seen.add(path)
159
        if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
160
            return "manifest_asset_fields:" + path
161
    for index in range(4):
162
        path = repo / "assets" / f"asset-{index:02d}.bin"
163
        if path.stat().st_size != ASSET_BYTES:
164
            return "asset_size_mismatch:" + path.name
165
        digest = hashlib.sha256()
166
        with path.open("rb") as stream:
167
            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
168
                digest.update(chunk)
169
        if digest.hexdigest() != _expected_asset_hash(index):
170
            return "asset_content_mismatch:" + path.name
171
    return None
172
173
174
def validate(scenario: str, repo: Path) -> str | None:
175
    """Return a concrete failure reason, or None on a fully verified task."""
176
    if scenario not in SCENARIOS:
177
        return "unsupported_trusted_task_oracle:" + scenario
178
    if scenario == "history_archaeology":
179
        return "history_answer_oracle_unavailable"
180
    if scenario in COMMITTED_PAYLOADS:
181
        expected = dict(COMMITTED_PAYLOADS[scenario])
182
        if scenario == "sync_pull_dirty":
183
            expected["work.txt"] = "dirty local edit"
184
        try:
185
            for relative, marker in expected.items():
186
                path = repo / relative
187
                if (any((repo.joinpath(*Path(relative).parts[:index])).is_symlink() for index in range(1, len(Path(relative).parts) + 1))
188
                        or not _regular_file(path) or path.stat().st_size > PAYLOAD_LIMIT_BYTES
189
                        or marker not in path.read_text()):
190
                    return "worktree_payload_missing:" + relative
191
        except (OSError, UnicodeError):
192
            return "worktree_payload_unreadable"
193
        return None
194
    name, expected_validator = (
195
        ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
196
        else ("validate_manifest.py", ASSET_VALIDATOR)
197
    )
198
    try:
199
        path = repo / name
200
        if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
201
                or path.read_bytes() != expected_validator):
202
            return "fixture_validator_modified_or_missing:" + name
203
        return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
204
    except (OSError, ValueError, UnicodeError, RecursionError) as exc:
205
        return "trusted_payload_unreadable_or_invalid:" + str(exc)
206
207
208
def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
209
    """Resolve an explicitly configured instrument or the invoking PATH once."""
210
    requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
211
    resolved = shutil.which(requested)
212
    # The xcrun trampoline execs the real Git; identify the binary that runs
213
    # so evidence classes compare the same bytes the Git subject executes.
214
    from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
215
    if resolved and is_xcrun_git_shim(resolved):
216
        real = resolve_real_git()
217
        resolved = str(real) if real else resolved
218
    metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
219
                "git_head_instrument_version": None}
220
    if resolved is None:
221
        return None, metadata
222
    binary = str(Path(resolved).resolve())
223
    try:
224
        version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
225
        if version.returncode:
226
            return None, metadata
227
        metadata.update(git_head_instrument_binary=binary,
228
                        git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
229
                        git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
230
        return binary, metadata
231
    except (OSError, subprocess.TimeoutExpired):
232
        return None, metadata
233
234
235
def git_head_text(repo: Path, path: str, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> tuple[str | None, str | None]:
236
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
237
    if git is None:
238
        return None, "git_head_oracle_unavailable"
239
    try:
240
        result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
241
                                env=base_env(), capture_output=True, timeout=30)
242
        if result.returncode < 0:
243
            return None, "git_head_oracle_unavailable"
244
        return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
245
    except (OSError, subprocess.TimeoutExpired):
246
        return None, "git_head_oracle_unavailable"
247
248
249
def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> str | None:
250
    """Verify the entire task subtree from committed objects using system Git.
251
252
    Never checks out subject-controlled hooks/filters or extracts an archive.
253
    Enumerated blob paths and bounded sizes are checked before materialization.
254
    Git object replacement and inherited Git environment overrides are disabled.
255
    """
256
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
257
    if git is None:
258
        return "git_head_oracle_unavailable"
259
    if scenario in COMMITTED_PAYLOADS:
260
        try:
261
            expected = COMMITTED_PAYLOADS[scenario]
262
            command = [git, "-C", str(repo.resolve())]
263
            # One tree read binds every required path to the same committed tree.
264
            tree = subprocess.run([*command, "ls-tree", "-r", "-z", "HEAD", "--", *committed_scope(scenario)],
265
                env=base_env(), capture_output=True, timeout=30)
266
            if tree.returncode < 0:
267
                return "git_head_oracle_unavailable"
268
            if tree.returncode:
269
                return "git_head_tree_unreadable"
270
            entries = {}
271
            for raw in tree.stdout.split(b"\0"):
272
                if not raw:
273
                    continue
274
                header, relative = raw.split(b"\t", 1)
275
                mode, kind, oid = header.split()
276
                path = relative.decode("utf-8")
277
                if mode != b"100644" or kind != b"blob":
278
                    return "committed_payload_path_type:" + path
279
                if path not in expected or path in entries:
280
                    return "committed_payload_path_set_mismatch"
281
                if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
282
                    return "git_head_invalid_object_id"
283
                entries[path] = oid.decode("ascii")
284
            if set(entries) != set(expected):
285
                return "committed_payload_path_set_mismatch"
286
            for path, marker in expected.items():
287
                oid = entries[path]
288
                size = subprocess.run([*command, "cat-file", "-s", oid],
289
                    env=base_env(), capture_output=True, timeout=30)
290
                if size.returncode < 0:
291
                    return "git_head_oracle_unavailable"
292
                if size.returncode or not size.stdout.strip().isdigit():
293
                    return "committed_payload_missing:" + path
294
                length = int(size.stdout.strip())
295
                if length > PAYLOAD_LIMIT_BYTES:
296
                    return "committed_payload_over_limit:" + path
297
                blob = subprocess.run([*command, "cat-file", "blob", oid],
298
                    env=base_env(), capture_output=True, timeout=30)
299
                if blob.returncode < 0:
300
                    return "git_head_oracle_unavailable"
301
                if blob.returncode or len(blob.stdout) != length:
302
                    return "git_head_blob_unreadable"
303
                try:
304
                    text = blob.stdout.decode("utf-8")
305
                except UnicodeError:
306
                    return "committed_payload_invalid_utf8:" + path
307
                if marker not in text:
308
                    return "committed_payload_missing:" + path
309
            return None
310
        except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired):
311
            return "git_head_oracle_unavailable"
312
    if scenario == "wide_config_refactor":
313
        paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
314
        paths.add("validate_configs.py")
315
        scope = list(committed_scope(scenario))
316
        max_blob_size = 4096
317
    elif scenario == "large_asset_manifest":
318
        paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
319
        paths.update({"manifest.json", "validate_manifest.py"})
320
        scope = list(committed_scope(scenario))
321
        max_blob_size = ASSET_BYTES
322
    else:
323
        return "unsupported_trusted_task_oracle:" + scenario
324
    env = base_env()
325
    command = [git, "-C", str(repo.resolve())]
326
    try:
327
        tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
328
                              env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
329
        if tree.returncode < 0:
330
            return "git_head_oracle_unavailable"
331
        if tree.returncode:
332
            return "git_head_tree_unreadable"
333
        entries = {}
334
        for raw in tree.stdout.split(b"\0"):
335
            if not raw:
336
                continue
337
            header, rel = raw.split(b"\t", 1)
338
            mode, kind, oid = header.split()
339
            path = rel.decode("utf-8")
340
            if mode != b"100644" or kind != b"blob":
341
                return "git_head_path_type:" + path
342
            if path not in paths or path in entries:
343
                return "git_head_path_set_mismatch"
344
            if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
345
                return "git_head_invalid_object_id"
346
            entries[path] = oid
347
        if set(entries) != paths:
348
            return "git_head_path_set_mismatch"
349
        ordered = sorted(entries)
350
        request = b"\n".join(entries[path] for path in ordered) + b"\n"
351
        sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
352
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
353
        if sizes.returncode < 0:
354
            return "git_head_oracle_unavailable"
355
        if sizes.returncode:
356
            return "git_head_blob_metadata_unreadable"
357
        headers = sizes.stdout.splitlines()
358
        if len(headers) != len(ordered):
359
            return "git_head_blob_metadata_incomplete"
360
        for header in headers:
361
            fields = header.split()
362
            if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
363
                return "git_head_blob_metadata_invalid"
364
            if int(fields[2]) > max_blob_size:
365
                return "git_head_blob_size_out_of_bounds"
366
        blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
367
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
368
        if blobs.returncode < 0:
369
            return "git_head_oracle_unavailable"
370
        if blobs.returncode:
371
            return "git_head_blob_unreadable"
372
        offset = 0
373
        with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
374
            root = Path(tmp)
375
            for path, expected_header in zip(ordered, headers):
376
                end = blobs.stdout.find(b"\n", offset)
377
                if end < 0 or blobs.stdout[offset:end] != expected_header:
378
                    return "git_head_blob_framing_invalid"
379
                size = int(expected_header.split()[2])
380
                offset = end + 1
381
                payload = blobs.stdout[offset:offset + size]
382
                if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
383
                    return "git_head_blob_framing_invalid"
384
                offset += size + 1
385
                destination = root / path
386
                destination.parent.mkdir(parents=True, exist_ok=True)
387
                destination.write_bytes(payload)
388
            if offset != len(blobs.stdout):
389
                return "git_head_blob_trailing_bytes"
390
            reason = validate(scenario, root)
391
            return "git_head_" + reason if reason else None
392
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
393
        return "git_head_oracle_error:" + str(exc)
394
395
396
def reader_evidence(kind: str, binary: str | None, metadata: dict) -> dict:
397
    """Classify an executed reader by observed binary bytes, never its label."""
398
    subject_hash = metadata.get("workflow_subject_binary_sha256")
399
    if kind == "oak":
400
        subject_hash = subject_hash or metadata.get("oak_export_instrument_sha256")
401
    try:
402
        resolved = shutil.which(binary or kind) if subject_hash is None else None
403
        if resolved and kind == "git":
404
            # The xcrun shim execs the real Git: hash the binary that runs,
405
            # exactly as resolve_git_instrument does for the instrument.
406
            from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
407
            if is_xcrun_git_shim(resolved):
408
                real = resolve_real_git()
409
                resolved = str(real) if real else resolved
410
        if resolved:
411
            if str(Path(resolved).resolve()) == metadata.get("git_head_instrument_binary"):
412
                subject_hash = metadata.get("git_head_instrument_sha256")
413
            else:
414
                subject_hash = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
415
    except OSError:
416
        pass
417
    instrument_hash = metadata.get("git_head_instrument_sha256")
418
    evidence_class = "unverified_committed_tree"
419
    if kind == "oak":
420
        evidence_class = "subject_produced_committed_tree"
421
    elif subject_hash and instrument_hash:
422
        evidence_class = "subject_binary_committed_tree" if subject_hash == instrument_hash else "independent_committed_tree"
423
    return {"workflow_subject_binary_sha256": subject_hash, "workflow_integrity_evidence_class": evidence_class}
424
425
426
def independent_oak_evidence(scenario: str, repo: Path, python: str, evidence: dict) -> dict:
427
    """Run a separately configured read-only decoder; never invoke Oak here."""
428
    from oakbench.oak_sqlite_reader import READER_VERSION
429
    helper = Path(__file__).with_name("oak_sqlite_reader.py")
430
    binary = shutil.which(python)
431
    if binary is None:
432
        evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_python_unavailable"
433
        return evidence
434
    evidence["workflow_integrity_source"] += ";independent_oak_sqlite_read_attempt"
435
    evidence["workflow_integrity_evidence_class"] = "independent_committed_read_attempt"
436
    try:
437
        helper_hash = hashlib.sha256(helper.read_bytes()).hexdigest()
438
        evidence["oak_reader_python_sha256"] = hashlib.sha256(Path(binary).read_bytes()).hexdigest()
439
        with tempfile.TemporaryDirectory(prefix="oakbench-independent-read-") as temporary:
440
            tree = Path(temporary) / "tree"
441
            scope = committed_scope(scenario)
442
            evidence["workflow_integrity_committed_scope"] = list(scope)
443
            result = subprocess.run([binary, "-I", str(helper), str(repo), str(tree),
444
                                     "--max-blob-bytes", str(committed_blob_limit(scenario)), "--scope", *scope],
445
                env=base_env(), capture_output=True, timeout=120)
446
            metadata = json.loads(result.stdout)
447
            if result.returncode == 77:
448
                evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_unavailable:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
449
                return evidence
450
            if result.returncode != 0:
451
                # A decoder failure is an instrument gap unless its contract
452
                # positively identifies invalid stored content.
453
                if result.returncode == 1 and metadata.get("corrupt") is True:
454
                    evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason="oak_stored_content_invalid:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure")))))
455
                else:
456
                    evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_failed:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
457
                return evidence
458
            if (metadata.get("reader_version") != READER_VERSION or metadata.get("reader_sha256") != helper_hash
459
                    or not re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("head", "")))
460
                    or metadata.get("scope") != list(scope)):
461
                raise ValueError("oak_reader_identity_mismatch")
462
            evidence.update({"oak_reader_version": READER_VERSION, "oak_reader_sha256": helper_hash,
463
                "oak_reader_head": metadata["head"], "oak_reader_blake3_version": metadata.get("blake3_version"),
464
                "oak_reader_zstandard_version": metadata.get("zstandard_version")})
465
            nonregular = metadata.get("nonregular_paths") or []
466
            oversized = metadata.get("oversized_paths") or []
467
            collisions = metadata.get("collision_paths") or []
468
            exact = scenario in EXACT_SCENARIOS
469
            if nonregular:
470
                # Same outcome as Git's scoped mode check (mode != 100644).
471
                reason = ("git_head_path_type:" if exact else "committed_payload_path_type:") + str(nonregular[0])
472
            elif collisions:
473
                reason = "git_head_path_set_mismatch" if exact else "committed_payload_path_set_mismatch"
474
            elif oversized:
475
                reason = "git_head_blob_size_out_of_bounds" if exact else "committed_payload_over_limit:" + str(oversized[0])
476
            elif scenario in EXACT_SCENARIOS:
477
                reason = validate(scenario, tree)
478
            else:
479
                reason = None
480
                for relative, marker in COMMITTED_PAYLOADS[scenario].items():
481
                    path = tree / relative
482
                    # Reason strings mirror validate_git_head's so the two
483
                    # readers report the same failure for the same content.
484
                    if not path.exists() and not path.is_symlink():
485
                        reason = "committed_payload_path_set_mismatch"
486
                        break
487
                    if not _regular_file(path):
488
                        reason = "committed_payload_path_type:" + relative
489
                        break
490
                    if path.stat().st_size > PAYLOAD_LIMIT_BYTES:
491
                        reason = "committed_payload_over_limit:" + relative
492
                        break
493
                    try:
494
                        text = path.read_text(encoding="utf-8")
495
                    except UnicodeError:
496
                        reason = "committed_payload_invalid_utf8:" + relative
497
                        break
498
                    if marker not in text:
499
                        reason = "committed_payload_missing:" + relative
500
                        break
501
            evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
502
            if reason is None:
503
                evidence["workflow_integrity_evidence_class"] = "independent_committed_tree" if scenario in EXACT_SCENARIOS else "independent_committed_payload"
504
                evidence["workflow_integrity_source"] = "trusted_task_worktree;independent_oak_sqlite_reader"
505
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
506
        evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_instrument_error:" + str(exc)
507
    return evidence
508
509
510
def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None, oak_reader_python: str | None = None) -> dict:
511
    source = "trusted_exact_task_worktree"
512
    evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
513
                "workflow_integrity_evidence_class": "worktree_payload",
514
                "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
515
                "workflow_integrity_unmeasured_reason": None}
516
    reason = validate(scenario, repo)
517
    if reason == "history_answer_oracle_unavailable":
518
        evidence["workflow_integrity_evidence_class"] = None
519
        evidence["workflow_integrity_unmeasured_reason"] = reason
520
        return evidence
521
    if reason:
522
        evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
523
        return evidence
524
    configured_reader = oak_reader_python or os.environ.get("OAK_BENCH_OAK_READER_PYTHON")
525
    if kind == "oak" and configured_reader:
526
        return independent_oak_evidence(scenario, repo, configured_reader, evidence)
527
    if kind in {"git", "oak"}:
528
        git, metadata = resolve_git_instrument(git_instrument)
529
        evidence.update(metadata)
530
        if git is None:
531
            evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
532
            return evidence
533
        evidence["workflow_integrity_committed_scope"] = list(committed_scope(scenario) or ())
534
        if kind == "git":
535
            evidence["workflow_integrity_evidence_class"] = "committed_tree_attempt"
536
            evidence["workflow_integrity_source"] = source + ";git_head_read_attempt"
537
            reason = validate_git_head(scenario, repo, resolved_instrument=(git, metadata))
538
        else:
539
            oak = shutil.which(binary or "oak")
540
            if oak is None:
541
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
542
                return evidence
543
            oak = str(Path(oak).resolve())
544
            evidence["oak_export_instrument_binary"] = oak
545
            # Same classification as the Git arm (validate_git_head): a
546
            # positive nonzero exit or unusable output from the subject, or
547
            # subject-controlled repository state, is a MEASURED failure; only
548
            # an instrument that cannot run (launch error, signal, timeout) is
549
            # unmeasured. A broken `oak status`/`oak export` must never turn an
550
            # Oak failure into a skip.
551
            reason = None
552
            try:
553
                evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
554
555
                def run(command, timeout):
556
                    result = subprocess.run(command, cwd=repo, env=base_env(), capture_output=True, timeout=timeout)
557
                    if result.returncode < 0:
558
                        raise _InstrumentUnavailable("oak_instrument_signalled")
559
                    return result
560
561
                def identity(result):
562
                    if result.returncode:
563
                        return None, None
564
                    try:
565
                        data = json.loads(result.stdout)
566
                    except ValueError:
567
                        return None, None
568
                    if not isinstance(data, dict):
569
                        return None, None
570
                    return data.get("head"), data.get("branch")
571
572
                head, branch = identity(run([oak, "status", "--json"], 30))
573
                if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
574
                    reason = "oak_head_identity_unreadable"
575
                else:
576
                    evidence["oak_export_head"] = head
577
                    with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
578
                        destination = Path(tmp) / "export"
579
                        evidence["workflow_integrity_evidence_class"] = "subject_produced_committed_tree_attempt"
580
                        evidence["workflow_integrity_source"] = source + ";subject_produced_oak_head_export_attempt"
581
                        exported = run([oak, "export", "--branch", branch, str(destination)], 120)
582
                        if exported.returncode:
583
                            reason = "oak_head_export_failed"
584
                        elif identity(run([oak, "status", "--json"], 30)) != (head, branch):
585
                            reason = "oak_head_changed_during_export"
586
                        else:
587
                            reason = validate_git_head(scenario, destination, resolved_instrument=(git, metadata))
588
            except (OSError, subprocess.TimeoutExpired, _InstrumentUnavailable) as exc:
589
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_error:" + str(exc)
590
                return evidence
591
    if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
592
        evidence["workflow_integrity_unmeasured_reason"] = reason
593
        return evidence
594
    if reason is None and kind in {"git", "oak"}:
595
        evidence.update(reader_evidence(kind, binary, evidence))
596
        if scenario not in EXACT_SCENARIOS:
597
            evidence["workflow_integrity_evidence_class"] = evidence["workflow_integrity_evidence_class"].replace("_tree", "_payload")
598
        suffix = "subject_produced_oak_head_export" if kind == "oak" else evidence["workflow_integrity_evidence_class"]
599
        evidence["workflow_integrity_source"] = source + ";" + suffix
600
    evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
601
    return evidence
602
603
604
def main() -> int:
605
    import argparse
606
    parser = argparse.ArgumentParser()
607
    parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
608
    parser.add_argument("--git-head", action="store_true")
609
    parser.add_argument("--oak-head", action="store_true")
610
    parser.add_argument("--binary")
611
    parser.add_argument("--evidence-json", action="store_true")
612
    parser.add_argument("--git-instrument")
613
    parser.add_argument("--oak-reader-python")
614
    args = parser.parse_args()
615
    evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument, args.oak_reader_python)
616
    reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
617
    print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
618
    passed = evidence["workflow_integrity_passed"]
619
    return 77 if passed is None else 0 if passed else 1
620
621
622
if __name__ == "__main__":
623
    raise SystemExit(main())