Viewing files at commit 1c9b819ab86bCommit detailsView latest

Browse files
scripts/oakbench/workflow_oracles.py 585 lines · 30.3 KB · python
1
"""Trusted exact task payload checks, outside the agent-editable checkout.
2
3
This module is stdlib-only and can run with Python -I as an external oracle.
4
It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
5
600 canonical service configs and four 16 MiB deterministic binary assets.
6
"""
7
from __future__ import annotations
8
9
from functools import lru_cache
10
import hashlib
11
import json
12
import os
13
from pathlib import Path
14
import sys
15
import shutil
16
import subprocess
17
import tempfile
18
import re
19
20
if __package__ in (None, ""):
21
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
from oakbench.environment import base_env
23
24
VERSION = "2026-09-27.1"
25
EXACT_SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
26
COMMITTED_PAYLOADS = {
27
    "bugfix_test_loop": {"app/pricing.py": "return cents - ((cents * percent) // 100)"},
28
    "vcs_error_recovery": {"README.md": "# recovered"},
29
    "sync_push_divergence": {"work.txt": "local change", "notes.txt": "upstream change"},
30
    "sync_pull_upstream": {"notes.txt": "upstream change"},
31
    "sync_pull_dirty": {"notes.txt": "upstream change"},
32
    "sync_non_ff_after_amend": {"work.txt": "amended line"},
33
}
34
SCENARIOS = EXACT_SCENARIOS | frozenset(COMMITTED_PAYLOADS) | {"history_archaeology"}
35
PAYLOAD_LIMIT_BYTES = 1024 * 1024
36
ASSET_BYTES = 16 * 1024 * 1024
37
38
# Preserved fixture instructions, not programs executed by this oracle.
39
CONFIG_VALIDATOR = "\n".join([
40
    "from pathlib import Path", "bad = []",
41
    "for p in Path('services').glob('*/config.toml'):",
42
    "    text = p.read_text()", "    if 'timeout_ms = 2500' in text:",
43
    "        bad.append(str(p))", "print(f'bad={len(bad)}')",
44
    "raise SystemExit(1 if bad else 0)", "",
45
]).encode()
46
ASSET_VALIDATOR = "\n".join([
47
    "import json", "from pathlib import Path",
48
    "data = json.loads(Path('manifest.json').read_text())",
49
    "for item in data['assets']:", "    p = Path(item['path'])",
50
    "    assert p.exists(), p", "    assert p.stat().st_size == item['bytes'], p",
51
    "    assert item['reviewed'] is True, p", "print('manifest ok')", "",
52
]).encode()
53
54
55
EXACT_SCOPES = {
56
    "wide_config_refactor": ("services", "validate_configs.py"),
57
    "large_asset_manifest": ("assets", "manifest.json", "validate_manifest.py"),
58
}
59
60
61
def committed_scope(scenario: str) -> tuple[str, ...] | None:
62
    """The committed paths every reader checks for a scenario, Git or Oak.
63
64
    One definition, so the Git ls-tree check and the independent Oak reader
65
    read the SAME scope at the same strength and may share an evidence class.
66
    """
67
    if scenario in COMMITTED_PAYLOADS:
68
        return tuple(sorted(COMMITTED_PAYLOADS[scenario]))
69
    return EXACT_SCOPES.get(scenario)
70
71
72
def _regular_file(path: Path) -> bool:
73
    return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
74
75
76
def _tree_paths(root: Path) -> set[str]:
77
    """Return every descendant identity, rejecting links and special files."""
78
    if root.is_symlink() or not root.is_dir():
79
        raise ValueError("payload_directory_missing_or_link")
80
    result = set()
81
    for parent, dirs, files in os.walk(root, followlinks=False):
82
        for name in dirs + files:
83
            path = Path(parent) / name
84
            if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
85
                raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
86
            result.add(path.relative_to(root).as_posix())
87
    return result
88
89
90
def _config_failure(repo: Path) -> str | None:
91
    expected_paths = {f"service-{i:04d}" for i in range(600)}
92
    expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
93
    actual = _tree_paths(repo / "services")
94
    if actual != expected_paths:
95
        return "config_path_set_mismatch"
96
    for i in range(600):
97
        rel = f"services/service-{i:04d}/config.toml"
98
        expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
99
        if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
100
            return "config_payload_mismatch:" + rel
101
    return None
102
103
104
def _unique_object(pairs):
105
    result = {}
106
    for key, value in pairs:
107
        if key in result:
108
            raise ValueError("duplicate_json_key:" + key)
109
        result[key] = value
110
    return result
111
112
113
@lru_cache(maxsize=4)
114
def _expected_asset_hash(index: int) -> str:
115
    unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
116
    digest = hashlib.sha256()
117
    remaining = ASSET_BYTES
118
    while remaining:
119
        chunk = unit[:min(len(unit), remaining)]
120
        digest.update(chunk)
121
        remaining -= len(chunk)
122
    return digest.hexdigest()
123
124
125
def _asset_failure(repo: Path) -> str | None:
126
    expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
127
    if _tree_paths(repo / "assets") != expected_paths:
128
        return "asset_path_set_mismatch"
129
    manifest = repo / "manifest.json"
130
    if not _regular_file(manifest):
131
        return "manifest_missing_or_wrong_type"
132
    if manifest.stat().st_size > 64 * 1024:
133
        return "manifest_size_out_of_bounds"
134
    data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
135
    if not isinstance(data, dict) or set(data) != {"assets"}:
136
        return "manifest_object_shape"
137
    entries = data["assets"]
138
    if not isinstance(entries, list) or len(entries) != 4:
139
        return "manifest_asset_count"
140
    seen = set()
141
    for item in entries:
142
        if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
143
            return "manifest_entry_shape"
144
        path = item["path"]
145
        if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
146
            return "manifest_asset_identity"
147
        if path in seen:
148
            return "manifest_duplicate_asset:" + path
149
        seen.add(path)
150
        if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
151
            return "manifest_asset_fields:" + path
152
    for index in range(4):
153
        path = repo / "assets" / f"asset-{index:02d}.bin"
154
        if path.stat().st_size != ASSET_BYTES:
155
            return "asset_size_mismatch:" + path.name
156
        digest = hashlib.sha256()
157
        with path.open("rb") as stream:
158
            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
159
                digest.update(chunk)
160
        if digest.hexdigest() != _expected_asset_hash(index):
161
            return "asset_content_mismatch:" + path.name
162
    return None
163
164
165
def validate(scenario: str, repo: Path) -> str | None:
166
    """Return a concrete failure reason, or None on a fully verified task."""
167
    if scenario not in SCENARIOS:
168
        return "unsupported_trusted_task_oracle:" + scenario
169
    if scenario == "history_archaeology":
170
        return "history_answer_oracle_unavailable"
171
    if scenario in COMMITTED_PAYLOADS:
172
        expected = dict(COMMITTED_PAYLOADS[scenario])
173
        if scenario == "sync_pull_dirty":
174
            expected["work.txt"] = "dirty local edit"
175
        try:
176
            for relative, marker in expected.items():
177
                path = repo / relative
178
                if (any((repo.joinpath(*Path(relative).parts[:index])).is_symlink() for index in range(1, len(Path(relative).parts) + 1))
179
                        or not _regular_file(path) or path.stat().st_size > PAYLOAD_LIMIT_BYTES
180
                        or marker not in path.read_text()):
181
                    return "worktree_payload_missing:" + relative
182
        except (OSError, UnicodeError):
183
            return "worktree_payload_unreadable"
184
        return None
185
    name, expected_validator = (
186
        ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
187
        else ("validate_manifest.py", ASSET_VALIDATOR)
188
    )
189
    try:
190
        path = repo / name
191
        if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
192
                or path.read_bytes() != expected_validator):
193
            return "fixture_validator_modified_or_missing:" + name
194
        return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
195
    except (OSError, ValueError, UnicodeError, RecursionError) as exc:
196
        return "trusted_payload_unreadable_or_invalid:" + str(exc)
197
198
199
def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
200
    """Resolve an explicitly configured instrument or the invoking PATH once."""
201
    requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
202
    resolved = shutil.which(requested)
203
    # The xcrun trampoline execs the real Git; identify the binary that runs
204
    # so evidence classes compare the same bytes the Git subject executes.
205
    from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
206
    if resolved and is_xcrun_git_shim(resolved):
207
        real = resolve_real_git()
208
        resolved = str(real) if real else resolved
209
    metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
210
                "git_head_instrument_version": None}
211
    if resolved is None:
212
        return None, metadata
213
    binary = str(Path(resolved).resolve())
214
    try:
215
        version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
216
        if version.returncode:
217
            return None, metadata
218
        metadata.update(git_head_instrument_binary=binary,
219
                        git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
220
                        git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
221
        return binary, metadata
222
    except (OSError, subprocess.TimeoutExpired):
223
        return None, metadata
224
225
226
def git_head_text(repo: Path, path: str, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> tuple[str | None, str | None]:
227
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
228
    if git is None:
229
        return None, "git_head_oracle_unavailable"
230
    try:
231
        result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
232
                                env=base_env(), capture_output=True, timeout=30)
233
        if result.returncode < 0:
234
            return None, "git_head_oracle_unavailable"
235
        return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
236
    except (OSError, subprocess.TimeoutExpired):
237
        return None, "git_head_oracle_unavailable"
238
239
240
def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> str | None:
241
    """Verify the entire task subtree from committed objects using system Git.
242
243
    Never checks out subject-controlled hooks/filters or extracts an archive.
244
    Enumerated blob paths and bounded sizes are checked before materialization.
245
    Git object replacement and inherited Git environment overrides are disabled.
246
    """
247
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
248
    if git is None:
249
        return "git_head_oracle_unavailable"
250
    if scenario in COMMITTED_PAYLOADS:
251
        try:
252
            expected = COMMITTED_PAYLOADS[scenario]
253
            command = [git, "-C", str(repo.resolve())]
254
            # One tree read binds every required path to the same committed tree.
255
            tree = subprocess.run([*command, "ls-tree", "-r", "-z", "HEAD", "--", *committed_scope(scenario)],
256
                env=base_env(), capture_output=True, timeout=30)
257
            if tree.returncode < 0:
258
                return "git_head_oracle_unavailable"
259
            if tree.returncode:
260
                return "git_head_tree_unreadable"
261
            entries = {}
262
            for raw in tree.stdout.split(b"\0"):
263
                if not raw:
264
                    continue
265
                header, relative = raw.split(b"\t", 1)
266
                mode, kind, oid = header.split()
267
                path = relative.decode("utf-8")
268
                if mode != b"100644" or kind != b"blob":
269
                    return "committed_payload_path_type:" + path
270
                if path not in expected or path in entries:
271
                    return "committed_payload_path_set_mismatch"
272
                if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
273
                    return "git_head_invalid_object_id"
274
                entries[path] = oid.decode("ascii")
275
            if set(entries) != set(expected):
276
                return "committed_payload_path_set_mismatch"
277
            for path, marker in expected.items():
278
                oid = entries[path]
279
                size = subprocess.run([*command, "cat-file", "-s", oid],
280
                    env=base_env(), capture_output=True, timeout=30)
281
                if size.returncode < 0:
282
                    return "git_head_oracle_unavailable"
283
                if size.returncode or not size.stdout.strip().isdigit():
284
                    return "committed_payload_missing:" + path
285
                length = int(size.stdout.strip())
286
                if length > PAYLOAD_LIMIT_BYTES:
287
                    return "committed_payload_over_limit:" + path
288
                blob = subprocess.run([*command, "cat-file", "blob", oid],
289
                    env=base_env(), capture_output=True, timeout=30)
290
                if blob.returncode < 0:
291
                    return "git_head_oracle_unavailable"
292
                if blob.returncode or len(blob.stdout) != length:
293
                    return "git_head_blob_unreadable"
294
                try:
295
                    text = blob.stdout.decode("utf-8")
296
                except UnicodeError:
297
                    return "committed_payload_invalid_utf8:" + path
298
                if marker not in text:
299
                    return "committed_payload_missing:" + path
300
            return None
301
        except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired):
302
            return "git_head_oracle_unavailable"
303
    if scenario == "wide_config_refactor":
304
        paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
305
        paths.add("validate_configs.py")
306
        scope = list(committed_scope(scenario))
307
        max_blob_size = 4096
308
    elif scenario == "large_asset_manifest":
309
        paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
310
        paths.update({"manifest.json", "validate_manifest.py"})
311
        scope = list(committed_scope(scenario))
312
        max_blob_size = ASSET_BYTES
313
    else:
314
        return "unsupported_trusted_task_oracle:" + scenario
315
    env = base_env()
316
    command = [git, "-C", str(repo.resolve())]
317
    try:
318
        tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
319
                              env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
320
        if tree.returncode < 0:
321
            return "git_head_oracle_unavailable"
322
        if tree.returncode:
323
            return "git_head_tree_unreadable"
324
        entries = {}
325
        for raw in tree.stdout.split(b"\0"):
326
            if not raw:
327
                continue
328
            header, rel = raw.split(b"\t", 1)
329
            mode, kind, oid = header.split()
330
            path = rel.decode("utf-8")
331
            if mode != b"100644" or kind != b"blob":
332
                return "git_head_path_type:" + path
333
            if path not in paths or path in entries:
334
                return "git_head_path_set_mismatch"
335
            if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
336
                return "git_head_invalid_object_id"
337
            entries[path] = oid
338
        if set(entries) != paths:
339
            return "git_head_path_set_mismatch"
340
        ordered = sorted(entries)
341
        request = b"\n".join(entries[path] for path in ordered) + b"\n"
342
        sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
343
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
344
        if sizes.returncode < 0:
345
            return "git_head_oracle_unavailable"
346
        if sizes.returncode:
347
            return "git_head_blob_metadata_unreadable"
348
        headers = sizes.stdout.splitlines()
349
        if len(headers) != len(ordered):
350
            return "git_head_blob_metadata_incomplete"
351
        for header in headers:
352
            fields = header.split()
353
            if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
354
                return "git_head_blob_metadata_invalid"
355
            if int(fields[2]) > max_blob_size:
356
                return "git_head_blob_size_out_of_bounds"
357
        blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
358
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
359
        if blobs.returncode < 0:
360
            return "git_head_oracle_unavailable"
361
        if blobs.returncode:
362
            return "git_head_blob_unreadable"
363
        offset = 0
364
        with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
365
            root = Path(tmp)
366
            for path, expected_header in zip(ordered, headers):
367
                end = blobs.stdout.find(b"\n", offset)
368
                if end < 0 or blobs.stdout[offset:end] != expected_header:
369
                    return "git_head_blob_framing_invalid"
370
                size = int(expected_header.split()[2])
371
                offset = end + 1
372
                payload = blobs.stdout[offset:offset + size]
373
                if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
374
                    return "git_head_blob_framing_invalid"
375
                offset += size + 1
376
                destination = root / path
377
                destination.parent.mkdir(parents=True, exist_ok=True)
378
                destination.write_bytes(payload)
379
            if offset != len(blobs.stdout):
380
                return "git_head_blob_trailing_bytes"
381
            reason = validate(scenario, root)
382
            return "git_head_" + reason if reason else None
383
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
384
        return "git_head_oracle_error:" + str(exc)
385
386
387
def reader_evidence(kind: str, binary: str | None, metadata: dict) -> dict:
388
    """Classify an executed reader by observed binary bytes, never its label."""
389
    subject_hash = metadata.get("workflow_subject_binary_sha256")
390
    if kind == "oak":
391
        subject_hash = subject_hash or metadata.get("oak_export_instrument_sha256")
392
    try:
393
        resolved = shutil.which(binary or kind) if subject_hash is None else None
394
        if resolved and kind == "git":
395
            # The xcrun shim execs the real Git: hash the binary that runs,
396
            # exactly as resolve_git_instrument does for the instrument.
397
            from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
398
            if is_xcrun_git_shim(resolved):
399
                real = resolve_real_git()
400
                resolved = str(real) if real else resolved
401
        if resolved:
402
            if str(Path(resolved).resolve()) == metadata.get("git_head_instrument_binary"):
403
                subject_hash = metadata.get("git_head_instrument_sha256")
404
            else:
405
                subject_hash = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
406
    except OSError:
407
        pass
408
    instrument_hash = metadata.get("git_head_instrument_sha256")
409
    evidence_class = "unverified_committed_tree"
410
    if kind == "oak":
411
        evidence_class = "subject_produced_committed_tree"
412
    elif subject_hash and instrument_hash:
413
        evidence_class = "subject_binary_committed_tree" if subject_hash == instrument_hash else "independent_committed_tree"
414
    return {"workflow_subject_binary_sha256": subject_hash, "workflow_integrity_evidence_class": evidence_class}
415
416
417
def independent_oak_evidence(scenario: str, repo: Path, python: str, evidence: dict) -> dict:
418
    """Run a separately configured read-only decoder; never invoke Oak here."""
419
    from oakbench.oak_sqlite_reader import READER_VERSION
420
    helper = Path(__file__).with_name("oak_sqlite_reader.py")
421
    binary = shutil.which(python)
422
    if binary is None:
423
        evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_python_unavailable"
424
        return evidence
425
    evidence["workflow_integrity_source"] += ";independent_oak_sqlite_read_attempt"
426
    evidence["workflow_integrity_evidence_class"] = "independent_committed_read_attempt"
427
    try:
428
        helper_hash = hashlib.sha256(helper.read_bytes()).hexdigest()
429
        evidence["oak_reader_python_sha256"] = hashlib.sha256(Path(binary).read_bytes()).hexdigest()
430
        with tempfile.TemporaryDirectory(prefix="oakbench-independent-read-") as temporary:
431
            tree = Path(temporary) / "tree"
432
            scope = committed_scope(scenario)
433
            evidence["workflow_integrity_committed_scope"] = list(scope)
434
            result = subprocess.run([binary, "-I", str(helper), str(repo), str(tree), "--scope", *scope],
435
                env=base_env(), capture_output=True, timeout=120)
436
            metadata = json.loads(result.stdout)
437
            if result.returncode == 77:
438
                evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_unavailable:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
439
                return evidence
440
            if result.returncode != 0:
441
                # A decoder failure is an instrument gap unless its contract
442
                # positively identifies invalid stored content.
443
                if result.returncode == 1 and metadata.get("corrupt") is True:
444
                    evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason="oak_stored_content_invalid:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure")))))
445
                else:
446
                    evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_failed:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
447
                return evidence
448
            if (metadata.get("reader_version") != READER_VERSION or metadata.get("reader_sha256") != helper_hash
449
                    or not re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("head", "")))
450
                    or metadata.get("scope") != list(scope)):
451
                raise ValueError("oak_reader_identity_mismatch")
452
            evidence.update({"oak_reader_version": READER_VERSION, "oak_reader_sha256": helper_hash,
453
                "oak_reader_head": metadata["head"], "oak_reader_blake3_version": metadata.get("blake3_version"),
454
                "oak_reader_zstandard_version": metadata.get("zstandard_version")})
455
            nonregular = metadata.get("nonregular_paths") or []
456
            if nonregular:
457
                # Same outcome as Git's scoped mode check (mode != 100644).
458
                reason = "committed_payload_path_type:" + str(nonregular[0])
459
            elif scenario in EXACT_SCENARIOS:
460
                reason = validate(scenario, tree)
461
            else:
462
                reason = None
463
                for relative, marker in COMMITTED_PAYLOADS[scenario].items():
464
                    path = tree / relative
465
                    # Reason strings mirror validate_git_head's so the two
466
                    # readers report the same failure for the same content.
467
                    if not path.exists() and not path.is_symlink():
468
                        reason = "committed_payload_path_set_mismatch"
469
                        break
470
                    if not _regular_file(path):
471
                        reason = "committed_payload_path_type:" + relative
472
                        break
473
                    if path.stat().st_size > PAYLOAD_LIMIT_BYTES:
474
                        reason = "committed_payload_over_limit:" + relative
475
                        break
476
                    try:
477
                        text = path.read_text(encoding="utf-8")
478
                    except UnicodeError:
479
                        reason = "committed_payload_invalid_utf8:" + relative
480
                        break
481
                    if marker not in text:
482
                        reason = "committed_payload_missing:" + relative
483
                        break
484
            evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
485
            if reason is None:
486
                evidence["workflow_integrity_evidence_class"] = "independent_committed_tree" if scenario in EXACT_SCENARIOS else "independent_committed_payload"
487
                evidence["workflow_integrity_source"] = "trusted_task_worktree;independent_oak_sqlite_reader"
488
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
489
        evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_instrument_error:" + str(exc)
490
    return evidence
491
492
493
def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None, oak_reader_python: str | None = None) -> dict:
494
    source = "trusted_exact_task_worktree"
495
    evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
496
                "workflow_integrity_evidence_class": "worktree_payload",
497
                "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
498
                "workflow_integrity_unmeasured_reason": None}
499
    reason = validate(scenario, repo)
500
    if reason == "history_answer_oracle_unavailable":
501
        evidence["workflow_integrity_evidence_class"] = None
502
        evidence["workflow_integrity_unmeasured_reason"] = reason
503
        return evidence
504
    if reason:
505
        evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
506
        return evidence
507
    configured_reader = oak_reader_python or os.environ.get("OAK_BENCH_OAK_READER_PYTHON")
508
    if kind == "oak" and configured_reader:
509
        return independent_oak_evidence(scenario, repo, configured_reader, evidence)
510
    if kind in {"git", "oak"}:
511
        git, metadata = resolve_git_instrument(git_instrument)
512
        evidence.update(metadata)
513
        if git is None:
514
            evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
515
            return evidence
516
        evidence["workflow_integrity_committed_scope"] = list(committed_scope(scenario) or ())
517
        if kind == "git":
518
            evidence["workflow_integrity_evidence_class"] = "committed_tree_attempt"
519
            evidence["workflow_integrity_source"] = source + ";git_head_read_attempt"
520
            reason = validate_git_head(scenario, repo, resolved_instrument=(git, metadata))
521
        else:
522
            oak = shutil.which(binary or "oak")
523
            if oak is None:
524
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
525
                return evidence
526
            oak = str(Path(oak).resolve())
527
            evidence["oak_export_instrument_binary"] = oak
528
            try:
529
                evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
530
                status = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
531
                before = json.loads(status.stdout) if status.returncode == 0 else {}
532
                head, branch = before.get("head"), before.get("branch")
533
                if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
534
                    raise ValueError("oak_head_identity_unreadable")
535
                evidence["oak_export_head"] = head
536
                with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
537
                    destination = Path(tmp) / "export"
538
                    evidence["workflow_integrity_evidence_class"] = "subject_produced_committed_tree_attempt"
539
                    evidence["workflow_integrity_source"] = source + ";subject_produced_oak_head_export_attempt"
540
                    exported = subprocess.run([oak, "export", "--branch", branch, str(destination)], cwd=repo,
541
                                              env=base_env(), capture_output=True, timeout=120)
542
                    if exported.returncode:
543
                        evidence["workflow_integrity_unmeasured_reason"] = "oak_head_export_unavailable"
544
                        return evidence
545
                    after = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
546
                    state = json.loads(after.stdout) if after.returncode == 0 else {}
547
                    if state.get("head") != head or state.get("branch") != branch:
548
                        raise ValueError("oak_head_changed_during_export")
549
                    reason = validate_git_head(scenario, destination, resolved_instrument=(git, metadata))
550
            except (OSError, ValueError, subprocess.TimeoutExpired) as exc:
551
                evidence["workflow_integrity_unmeasured_reason"] = str(exc)
552
                return evidence
553
    if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
554
        evidence["workflow_integrity_unmeasured_reason"] = reason
555
        return evidence
556
    if reason is None and kind in {"git", "oak"}:
557
        evidence.update(reader_evidence(kind, binary, evidence))
558
        if scenario not in EXACT_SCENARIOS:
559
            evidence["workflow_integrity_evidence_class"] = evidence["workflow_integrity_evidence_class"].replace("_tree", "_payload")
560
        suffix = "subject_produced_oak_head_export" if kind == "oak" else evidence["workflow_integrity_evidence_class"]
561
        evidence["workflow_integrity_source"] = source + ";" + suffix
562
    evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
563
    return evidence
564
565
566
def main() -> int:
567
    import argparse
568
    parser = argparse.ArgumentParser()
569
    parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
570
    parser.add_argument("--git-head", action="store_true")
571
    parser.add_argument("--oak-head", action="store_true")
572
    parser.add_argument("--binary")
573
    parser.add_argument("--evidence-json", action="store_true")
574
    parser.add_argument("--git-instrument")
575
    parser.add_argument("--oak-reader-python")
576
    args = parser.parse_args()
577
    evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument, args.oak_reader_python)
578
    reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
579
    print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
580
    passed = evidence["workflow_integrity_passed"]
581
    return 77 if passed is None else 0 if passed else 1
582
583
584
if __name__ == "__main__":
585
    raise SystemExit(main())