Viewing files at commit 21316fe59317Commit detailsView latest

Browse files
scripts/oakbench/workflow_oracles.py 630 lines · 32.5 KB · python
1
"""Trusted exact task payload checks, outside the agent-editable checkout.
2
3
This module is stdlib-only and can run with Python -I as an external oracle.
4
It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
5
600 canonical service configs and four 16 MiB deterministic binary assets.
6
"""
7
from __future__ import annotations
8
9
from functools import lru_cache
10
import hashlib
11
import json
12
import os
13
from pathlib import Path
14
import sys
15
import shutil
16
import subprocess
17
import tempfile
18
import re
19
20
if __package__ in (None, ""):
21
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
from oakbench.environment import base_env
23
24
VERSION = "2026-09-28.1"
25
EXACT_SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
26
COMMITTED_PAYLOADS = {
27
    "bugfix_test_loop": {"app/pricing.py": "return cents - ((cents * percent) // 100)"},
28
    "vcs_error_recovery": {"README.md": "# recovered"},
29
    "sync_push_divergence": {"work.txt": "local change", "notes.txt": "upstream change"},
30
    "sync_pull_upstream": {"notes.txt": "upstream change"},
31
    "sync_pull_dirty": {"notes.txt": "upstream change"},
32
    "sync_non_ff_after_amend": {"work.txt": "amended line"},
33
}
34
SCENARIOS = EXACT_SCENARIOS | frozenset(COMMITTED_PAYLOADS) | {"history_archaeology"}
35
PAYLOAD_LIMIT_BYTES = 1024 * 1024
36
ASSET_BYTES = 16 * 1024 * 1024
37
38
# Preserved fixture instructions, not programs executed by this oracle.
39
CONFIG_VALIDATOR = "\n".join([
40
    "from pathlib import Path", "bad = []",
41
    "for p in Path('services').glob('*/config.toml'):",
42
    "    text = p.read_text()", "    if 'timeout_ms = 2500' in text:",
43
    "        bad.append(str(p))", "print(f'bad={len(bad)}')",
44
    "raise SystemExit(1 if bad else 0)", "",
45
]).encode()
46
ASSET_VALIDATOR = "\n".join([
47
    "import json", "from pathlib import Path",
48
    "data = json.loads(Path('manifest.json').read_text())",
49
    "for item in data['assets']:", "    p = Path(item['path'])",
50
    "    assert p.exists(), p", "    assert p.stat().st_size == item['bytes'], p",
51
    "    assert item['reviewed'] is True, p", "print('manifest ok')", "",
52
]).encode()
53
54
55
EXACT_SCOPES = {
56
    "wide_config_refactor": ("services", "validate_configs.py"),
57
    "large_asset_manifest": ("assets", "manifest.json", "validate_manifest.py"),
58
}
59
60
61
def committed_blob_limit(scenario: str) -> int:
62
    """Largest committed blob the Git check accepts for a scenario."""
63
    return {"wide_config_refactor": 4096, "large_asset_manifest": ASSET_BYTES}.get(scenario, PAYLOAD_LIMIT_BYTES)
64
65
66
def committed_scope(scenario: str) -> tuple[str, ...] | None:
67
    """The committed paths every reader checks for a scenario, Git or Oak.
68
69
    One definition, so the Git ls-tree check and the independent Oak reader
70
    read the SAME scope at the same strength and may share an evidence class.
71
    """
72
    if scenario in COMMITTED_PAYLOADS:
73
        return tuple(sorted(COMMITTED_PAYLOADS[scenario]))
74
    return EXACT_SCOPES.get(scenario)
75
76
77
class _InstrumentUnavailable(Exception):
78
    """The instrument could not run to completion (signal); never a subject verdict."""
79
80
81
def _regular_file(path: Path) -> bool:
82
    return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
83
84
85
def _tree_paths(root: Path) -> set[str]:
86
    """Return every descendant identity, rejecting links and special files."""
87
    if root.is_symlink() or not root.is_dir():
88
        raise ValueError("payload_directory_missing_or_link")
89
    result = set()
90
    for parent, dirs, files in os.walk(root, followlinks=False):
91
        for name in dirs + files:
92
            path = Path(parent) / name
93
            if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
94
                raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
95
            result.add(path.relative_to(root).as_posix())
96
    return result
97
98
99
def _config_failure(repo: Path) -> str | None:
100
    expected_paths = {f"service-{i:04d}" for i in range(600)}
101
    expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
102
    actual = _tree_paths(repo / "services")
103
    if actual != expected_paths:
104
        return "config_path_set_mismatch"
105
    for i in range(600):
106
        rel = f"services/service-{i:04d}/config.toml"
107
        expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
108
        if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
109
            return "config_payload_mismatch:" + rel
110
    return None
111
112
113
def _unique_object(pairs):
114
    result = {}
115
    for key, value in pairs:
116
        if key in result:
117
            raise ValueError("duplicate_json_key:" + key)
118
        result[key] = value
119
    return result
120
121
122
@lru_cache(maxsize=4)
123
def _expected_asset_hash(index: int) -> str:
124
    unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
125
    digest = hashlib.sha256()
126
    remaining = ASSET_BYTES
127
    while remaining:
128
        chunk = unit[:min(len(unit), remaining)]
129
        digest.update(chunk)
130
        remaining -= len(chunk)
131
    return digest.hexdigest()
132
133
134
def _asset_failure(repo: Path) -> str | None:
135
    expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
136
    if _tree_paths(repo / "assets") != expected_paths:
137
        return "asset_path_set_mismatch"
138
    manifest = repo / "manifest.json"
139
    if not _regular_file(manifest):
140
        return "manifest_missing_or_wrong_type"
141
    if manifest.stat().st_size > 64 * 1024:
142
        return "manifest_size_out_of_bounds"
143
    data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
144
    if not isinstance(data, dict) or set(data) != {"assets"}:
145
        return "manifest_object_shape"
146
    entries = data["assets"]
147
    if not isinstance(entries, list) or len(entries) != 4:
148
        return "manifest_asset_count"
149
    seen = set()
150
    for item in entries:
151
        if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
152
            return "manifest_entry_shape"
153
        path = item["path"]
154
        if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
155
            return "manifest_asset_identity"
156
        if path in seen:
157
            return "manifest_duplicate_asset:" + path
158
        seen.add(path)
159
        if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
160
            return "manifest_asset_fields:" + path
161
    for index in range(4):
162
        path = repo / "assets" / f"asset-{index:02d}.bin"
163
        if path.stat().st_size != ASSET_BYTES:
164
            return "asset_size_mismatch:" + path.name
165
        digest = hashlib.sha256()
166
        with path.open("rb") as stream:
167
            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
168
                digest.update(chunk)
169
        if digest.hexdigest() != _expected_asset_hash(index):
170
            return "asset_content_mismatch:" + path.name
171
    return None
172
173
174
def validate(scenario: str, repo: Path) -> str | None:
175
    """Return a concrete failure reason, or None on a fully verified task."""
176
    if scenario not in SCENARIOS:
177
        return "unsupported_trusted_task_oracle:" + scenario
178
    if scenario == "history_archaeology":
179
        return "history_answer_oracle_unavailable"
180
    if scenario in COMMITTED_PAYLOADS:
181
        expected = dict(COMMITTED_PAYLOADS[scenario])
182
        if scenario == "sync_pull_dirty":
183
            expected["work.txt"] = "dirty local edit"
184
        try:
185
            for relative, marker in expected.items():
186
                path = repo / relative
187
                if (any((repo.joinpath(*Path(relative).parts[:index])).is_symlink() for index in range(1, len(Path(relative).parts) + 1))
188
                        or not _regular_file(path) or path.stat().st_size > PAYLOAD_LIMIT_BYTES
189
                        or marker not in path.read_text()):
190
                    return "worktree_payload_missing:" + relative
191
        except (OSError, UnicodeError):
192
            return "worktree_payload_unreadable"
193
        return None
194
    name, expected_validator = (
195
        ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
196
        else ("validate_manifest.py", ASSET_VALIDATOR)
197
    )
198
    try:
199
        path = repo / name
200
        if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
201
                or path.read_bytes() != expected_validator):
202
            return "fixture_validator_modified_or_missing:" + name
203
        return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
204
    except (OSError, ValueError, UnicodeError, RecursionError) as exc:
205
        return "trusted_payload_unreadable_or_invalid:" + str(exc)
206
207
208
def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
209
    """Resolve an explicitly configured instrument or the invoking PATH once."""
210
    requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
211
    resolved = shutil.which(requested)
212
    # The xcrun trampoline execs the real Git; identify the binary that runs
213
    # so evidence classes compare the same bytes the Git subject executes.
214
    from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
215
    if resolved and is_xcrun_git_shim(resolved):
216
        real = resolve_real_git()
217
        resolved = str(real) if real else resolved
218
    metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
219
                "git_head_instrument_version": None}
220
    if resolved is None:
221
        return None, metadata
222
    binary = str(Path(resolved).resolve())
223
    try:
224
        version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
225
        if version.returncode:
226
            return None, metadata
227
        metadata.update(git_head_instrument_binary=binary,
228
                        git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
229
                        git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
230
        return binary, metadata
231
    except (OSError, subprocess.TimeoutExpired):
232
        return None, metadata
233
234
235
def git_head_text(repo: Path, path: str, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> tuple[str | None, str | None]:
236
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
237
    if git is None:
238
        return None, "git_head_oracle_unavailable"
239
    try:
240
        result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
241
                                env=base_env(), capture_output=True, timeout=30)
242
        if result.returncode < 0:
243
            return None, "git_head_oracle_unavailable"
244
        return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
245
    except (OSError, subprocess.TimeoutExpired):
246
        return None, "git_head_oracle_unavailable"
247
248
249
def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> str | None:
250
    """Verify the entire task subtree from committed objects using system Git.
251
252
    Never checks out subject-controlled hooks/filters or extracts an archive.
253
    Enumerated blob paths and bounded sizes are checked before materialization.
254
    Git object replacement and inherited Git environment overrides are disabled.
255
    """
256
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
257
    if git is None:
258
        return "git_head_oracle_unavailable"
259
    if scenario in COMMITTED_PAYLOADS:
260
        try:
261
            expected = COMMITTED_PAYLOADS[scenario]
262
            command = [git, "-C", str(repo.resolve())]
263
            # One tree read binds every required path to the same committed tree.
264
            tree = subprocess.run([*command, "ls-tree", "-r", "-z", "HEAD", "--", *committed_scope(scenario)],
265
                env=base_env(), capture_output=True, timeout=30)
266
            if tree.returncode < 0:
267
                return "git_head_oracle_unavailable"
268
            if tree.returncode:
269
                return "git_head_tree_unreadable"
270
            entries = {}
271
            for raw in tree.stdout.split(b"\0"):
272
                if not raw:
273
                    continue
274
                header, relative = raw.split(b"\t", 1)
275
                mode, kind, oid = header.split()
276
                path = relative.decode("utf-8")
277
                if mode != b"100644" or kind != b"blob":
278
                    return "committed_payload_path_type:" + path
279
                if path not in expected or path in entries:
280
                    return "committed_payload_path_set_mismatch"
281
                if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
282
                    return "git_head_invalid_object_id"
283
                entries[path] = oid.decode("ascii")
284
            if set(entries) != set(expected):
285
                return "committed_payload_path_set_mismatch"
286
            for path, marker in expected.items():
287
                oid = entries[path]
288
                size = subprocess.run([*command, "cat-file", "-s", oid],
289
                    env=base_env(), capture_output=True, timeout=30)
290
                if size.returncode < 0:
291
                    return "git_head_oracle_unavailable"
292
                if size.returncode or not size.stdout.strip().isdigit():
293
                    return "committed_payload_missing:" + path
294
                length = int(size.stdout.strip())
295
                if length > PAYLOAD_LIMIT_BYTES:
296
                    return "committed_payload_over_limit:" + path
297
                blob = subprocess.run([*command, "cat-file", "blob", oid],
298
                    env=base_env(), capture_output=True, timeout=30)
299
                if blob.returncode < 0:
300
                    return "git_head_oracle_unavailable"
301
                if blob.returncode or len(blob.stdout) != length:
302
                    return "git_head_blob_unreadable"
303
                try:
304
                    text = blob.stdout.decode("utf-8")
305
                except UnicodeError:
306
                    return "committed_payload_invalid_utf8:" + path
307
                if marker not in text:
308
                    return "committed_payload_missing:" + path
309
            return None
310
        except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired):
311
            return "git_head_oracle_unavailable"
312
    if scenario == "wide_config_refactor":
313
        paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
314
        paths.add("validate_configs.py")
315
        scope = list(committed_scope(scenario))
316
        max_blob_size = 4096
317
    elif scenario == "large_asset_manifest":
318
        paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
319
        paths.update({"manifest.json", "validate_manifest.py"})
320
        scope = list(committed_scope(scenario))
321
        max_blob_size = ASSET_BYTES
322
    else:
323
        return "unsupported_trusted_task_oracle:" + scenario
324
    env = base_env()
325
    command = [git, "-C", str(repo.resolve())]
326
    try:
327
        tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
328
                              env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
329
        if tree.returncode < 0:
330
            return "git_head_oracle_unavailable"
331
        if tree.returncode:
332
            return "git_head_tree_unreadable"
333
        entries = {}
334
        for raw in tree.stdout.split(b"\0"):
335
            if not raw:
336
                continue
337
            header, rel = raw.split(b"\t", 1)
338
            mode, kind, oid = header.split()
339
            path = rel.decode("utf-8")
340
            if mode != b"100644" or kind != b"blob":
341
                return "git_head_path_type:" + path
342
            if path not in paths or path in entries:
343
                return "git_head_path_set_mismatch"
344
            if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
345
                return "git_head_invalid_object_id"
346
            entries[path] = oid
347
        if set(entries) != paths:
348
            return "git_head_path_set_mismatch"
349
        ordered = sorted(entries)
350
        request = b"\n".join(entries[path] for path in ordered) + b"\n"
351
        sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
352
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
353
        if sizes.returncode < 0:
354
            return "git_head_oracle_unavailable"
355
        if sizes.returncode:
356
            return "git_head_blob_metadata_unreadable"
357
        headers = sizes.stdout.splitlines()
358
        if len(headers) != len(ordered):
359
            return "git_head_blob_metadata_incomplete"
360
        for header in headers:
361
            fields = header.split()
362
            if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
363
                return "git_head_blob_metadata_invalid"
364
            if int(fields[2]) > max_blob_size:
365
                return "git_head_blob_size_out_of_bounds"
366
        blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
367
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
368
        if blobs.returncode < 0:
369
            return "git_head_oracle_unavailable"
370
        if blobs.returncode:
371
            return "git_head_blob_unreadable"
372
        offset = 0
373
        with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
374
            root = Path(tmp)
375
            for path, expected_header in zip(ordered, headers):
376
                end = blobs.stdout.find(b"\n", offset)
377
                if end < 0 or blobs.stdout[offset:end] != expected_header:
378
                    return "git_head_blob_framing_invalid"
379
                size = int(expected_header.split()[2])
380
                offset = end + 1
381
                payload = blobs.stdout[offset:offset + size]
382
                if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
383
                    return "git_head_blob_framing_invalid"
384
                offset += size + 1
385
                destination = root / path
386
                destination.parent.mkdir(parents=True, exist_ok=True)
387
                destination.write_bytes(payload)
388
            if offset != len(blobs.stdout):
389
                return "git_head_blob_trailing_bytes"
390
            reason = validate(scenario, root)
391
            return "git_head_" + reason if reason else None
392
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
393
        return "git_head_oracle_error:" + str(exc)
394
395
396
def reader_evidence(kind: str, binary: str | None, metadata: dict) -> dict:
397
    """Classify an executed reader by observed binary bytes, never its label."""
398
    subject_hash = metadata.get("workflow_subject_binary_sha256")
399
    if kind == "oak":
400
        subject_hash = subject_hash or metadata.get("oak_export_instrument_sha256")
401
    try:
402
        resolved = shutil.which(binary or kind) if subject_hash is None else None
403
        if resolved and kind == "git":
404
            # The xcrun shim execs the real Git: hash the binary that runs,
405
            # exactly as resolve_git_instrument does for the instrument.
406
            from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
407
            if is_xcrun_git_shim(resolved):
408
                real = resolve_real_git()
409
                resolved = str(real) if real else resolved
410
        if resolved:
411
            if str(Path(resolved).resolve()) == metadata.get("git_head_instrument_binary"):
412
                subject_hash = metadata.get("git_head_instrument_sha256")
413
            else:
414
                subject_hash = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
415
    except OSError:
416
        pass
417
    instrument_hash = metadata.get("git_head_instrument_sha256")
418
    evidence_class = "unverified_committed_tree"
419
    if kind == "oak":
420
        evidence_class = "subject_produced_committed_tree"
421
    elif subject_hash and instrument_hash:
422
        evidence_class = "subject_binary_committed_tree" if subject_hash == instrument_hash else "independent_committed_tree"
423
    return {"workflow_subject_binary_sha256": subject_hash, "workflow_integrity_evidence_class": evidence_class}
424
425
426
def independent_oak_evidence(scenario: str, repo: Path, python: str, evidence: dict) -> dict:
427
    """Run a separately configured read-only decoder; never invoke Oak here."""
428
    from oakbench.oak_sqlite_reader import READER_VERSION
429
    helper = Path(__file__).with_name("oak_sqlite_reader.py")
430
    binary = shutil.which(python)
431
    if binary is None:
432
        evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_python_unavailable"
433
        return evidence
434
    evidence["workflow_integrity_source"] += ";independent_oak_sqlite_read_attempt"
435
    evidence["workflow_integrity_evidence_class"] = "independent_committed_read_attempt"
436
    try:
437
        helper_hash = hashlib.sha256(helper.read_bytes()).hexdigest()
438
        evidence["oak_reader_python_sha256"] = hashlib.sha256(Path(binary).read_bytes()).hexdigest()
439
        with tempfile.TemporaryDirectory(prefix="oakbench-independent-read-") as temporary:
440
            tree = Path(temporary) / "tree"
441
            scope = committed_scope(scenario)
442
            evidence["workflow_integrity_committed_scope"] = list(scope)
443
            result = subprocess.run([binary, "-I", str(helper), str(repo), str(tree),
444
                                     "--max-blob-bytes", str(committed_blob_limit(scenario)), "--scope", *scope],
445
                env=base_env(), capture_output=True, timeout=120)
446
            metadata = json.loads(result.stdout)
447
            if result.returncode == 77:
448
                evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_unavailable:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
449
                return evidence
450
            if result.returncode != 0:
451
                # A decoder failure is an instrument gap unless its contract
452
                # positively identifies invalid stored content.
453
                if result.returncode == 1 and metadata.get("corrupt") is True:
454
                    evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason="oak_stored_content_invalid:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure")))))
455
                else:
456
                    evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_failed:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
457
                return evidence
458
            if (metadata.get("reader_version") != READER_VERSION or metadata.get("reader_sha256") != helper_hash
459
                    or not re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("head", "")))
460
                    or metadata.get("scope") != list(scope)):
461
                raise ValueError("oak_reader_identity_mismatch")
462
            evidence.update({"oak_reader_version": READER_VERSION, "oak_reader_sha256": helper_hash,
463
                "oak_reader_head": metadata["head"], "oak_reader_backend": metadata.get("reader_backend"),
464
                "oak_reader_codec_sha256": metadata.get("codec_sha256"),
465
                "oak_reader_blake3_version": metadata.get("blake3_version"),
466
                "oak_reader_zstandard_version": metadata.get("zstandard_version")})
467
            nonregular = metadata.get("nonregular_paths") or []
468
            oversized = metadata.get("oversized_paths") or []
469
            collisions = metadata.get("collision_paths") or []
470
            exact = scenario in EXACT_SCENARIOS
471
            if nonregular:
472
                # Same outcome as Git's scoped mode check (mode != 100644).
473
                reason = ("git_head_path_type:" if exact else "committed_payload_path_type:") + str(nonregular[0])
474
            elif collisions:
475
                reason = "git_head_path_set_mismatch" if exact else "committed_payload_path_set_mismatch"
476
            elif oversized:
477
                reason = "git_head_blob_size_out_of_bounds" if exact else "committed_payload_over_limit:" + str(oversized[0])
478
            elif scenario in EXACT_SCENARIOS:
479
                reason = validate(scenario, tree)
480
            else:
481
                reason = None
482
                for relative, marker in COMMITTED_PAYLOADS[scenario].items():
483
                    path = tree / relative
484
                    # Reason strings mirror validate_git_head's so the two
485
                    # readers report the same failure for the same content.
486
                    if not path.exists() and not path.is_symlink():
487
                        reason = "committed_payload_path_set_mismatch"
488
                        break
489
                    if not _regular_file(path):
490
                        reason = "committed_payload_path_type:" + relative
491
                        break
492
                    if path.stat().st_size > PAYLOAD_LIMIT_BYTES:
493
                        reason = "committed_payload_over_limit:" + relative
494
                        break
495
                    try:
496
                        text = path.read_text(encoding="utf-8")
497
                    except UnicodeError:
498
                        reason = "committed_payload_invalid_utf8:" + relative
499
                        break
500
                    if marker not in text:
501
                        reason = "committed_payload_missing:" + relative
502
                        break
503
            evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
504
            if reason is None:
505
                evidence["workflow_integrity_evidence_class"] = "independent_committed_tree" if scenario in EXACT_SCENARIOS else "independent_committed_payload"
506
                evidence["workflow_integrity_source"] = "trusted_task_worktree;independent_oak_sqlite_reader"
507
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
508
        evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_instrument_error:" + str(exc)
509
    return evidence
510
511
512
def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None, oak_reader_python: str | None = None) -> dict:
513
    source = "trusted_exact_task_worktree"
514
    evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
515
                "workflow_integrity_evidence_class": "worktree_payload",
516
                "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
517
                "workflow_integrity_unmeasured_reason": None}
518
    reason = validate(scenario, repo)
519
    if reason == "history_answer_oracle_unavailable":
520
        evidence["workflow_integrity_evidence_class"] = None
521
        evidence["workflow_integrity_unmeasured_reason"] = reason
522
        return evidence
523
    if reason:
524
        evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
525
        return evidence
526
    configured_reader = oak_reader_python or os.environ.get("OAK_BENCH_OAK_READER_PYTHON")
527
    if kind == "oak" and not configured_reader and os.environ.get("OAK_BENCH_OAK_COMMITTED_READER") != "export":
528
        # The reader is stdlib-only (pure BLAKE3/zstd): the harness interpreter
529
        # runs it, isolated (-I). Oak export (subject-produced evidence) is an
530
        # explicit opt-in only: OAK_BENCH_OAK_COMMITTED_READER=export.
531
        configured_reader = sys.executable
532
    if kind == "oak" and configured_reader:
533
        return independent_oak_evidence(scenario, repo, configured_reader, evidence)
534
    if kind in {"git", "oak"}:
535
        git, metadata = resolve_git_instrument(git_instrument)
536
        evidence.update(metadata)
537
        if git is None:
538
            evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
539
            return evidence
540
        evidence["workflow_integrity_committed_scope"] = list(committed_scope(scenario) or ())
541
        if kind == "git":
542
            evidence["workflow_integrity_evidence_class"] = "committed_tree_attempt"
543
            evidence["workflow_integrity_source"] = source + ";git_head_read_attempt"
544
            reason = validate_git_head(scenario, repo, resolved_instrument=(git, metadata))
545
        else:
546
            oak = shutil.which(binary or "oak")
547
            if oak is None:
548
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
549
                return evidence
550
            oak = str(Path(oak).resolve())
551
            evidence["oak_export_instrument_binary"] = oak
552
            # Same classification as the Git arm (validate_git_head): a
553
            # positive nonzero exit or unusable output from the subject, or
554
            # subject-controlled repository state, is a MEASURED failure; only
555
            # an instrument that cannot run (launch error, signal, timeout) is
556
            # unmeasured. A broken `oak status`/`oak export` must never turn an
557
            # Oak failure into a skip.
558
            reason = None
559
            try:
560
                evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
561
562
                def run(command, timeout):
563
                    result = subprocess.run(command, cwd=repo, env=base_env(), capture_output=True, timeout=timeout)
564
                    if result.returncode < 0:
565
                        raise _InstrumentUnavailable("oak_instrument_signalled")
566
                    return result
567
568
                def identity(result):
569
                    if result.returncode:
570
                        return None, None
571
                    try:
572
                        data = json.loads(result.stdout)
573
                    except ValueError:
574
                        return None, None
575
                    if not isinstance(data, dict):
576
                        return None, None
577
                    return data.get("head"), data.get("branch")
578
579
                head, branch = identity(run([oak, "status", "--json"], 30))
580
                if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
581
                    reason = "oak_head_identity_unreadable"
582
                else:
583
                    evidence["oak_export_head"] = head
584
                    with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
585
                        destination = Path(tmp) / "export"
586
                        evidence["workflow_integrity_evidence_class"] = "subject_produced_committed_tree_attempt"
587
                        evidence["workflow_integrity_source"] = source + ";subject_produced_oak_head_export_attempt"
588
                        exported = run([oak, "export", "--branch", branch, str(destination)], 120)
589
                        if exported.returncode:
590
                            reason = "oak_head_export_failed"
591
                        elif identity(run([oak, "status", "--json"], 30)) != (head, branch):
592
                            reason = "oak_head_changed_during_export"
593
                        else:
594
                            reason = validate_git_head(scenario, destination, resolved_instrument=(git, metadata))
595
            except (OSError, subprocess.TimeoutExpired, _InstrumentUnavailable) as exc:
596
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_error:" + str(exc)
597
                return evidence
598
    if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
599
        evidence["workflow_integrity_unmeasured_reason"] = reason
600
        return evidence
601
    if reason is None and kind in {"git", "oak"}:
602
        evidence.update(reader_evidence(kind, binary, evidence))
603
        if scenario not in EXACT_SCENARIOS:
604
            evidence["workflow_integrity_evidence_class"] = evidence["workflow_integrity_evidence_class"].replace("_tree", "_payload")
605
        suffix = "subject_produced_oak_head_export" if kind == "oak" else evidence["workflow_integrity_evidence_class"]
606
        evidence["workflow_integrity_source"] = source + ";" + suffix
607
    evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
608
    return evidence
609
610
611
def main() -> int:
612
    import argparse
613
    parser = argparse.ArgumentParser()
614
    parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
615
    parser.add_argument("--git-head", action="store_true")
616
    parser.add_argument("--oak-head", action="store_true")
617
    parser.add_argument("--binary")
618
    parser.add_argument("--evidence-json", action="store_true")
619
    parser.add_argument("--git-instrument")
620
    parser.add_argument("--oak-reader-python")
621
    args = parser.parse_args()
622
    evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument, args.oak_reader_python)
623
    reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
624
    print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
625
    passed = evidence["workflow_integrity_passed"]
626
    return 77 if passed is None else 0 if passed else 1
627
628
629
if __name__ == "__main__":
630
    raise SystemExit(main())