Viewing files at commit 2e081b95a118Commit detailsView latest

Browse files
scripts/oakbench/workflow_oracles.py 389 lines · 19.1 KB · python
1
"""Trusted exact task payload checks, outside the agent-editable checkout.
2
3
This module is stdlib-only and can run with Python -I as an external oracle.
4
It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
5
600 canonical service configs and four 16 MiB deterministic binary assets.
6
"""
7
from __future__ import annotations
8
9
from functools import lru_cache
10
import hashlib
11
import json
12
import os
13
from pathlib import Path
14
import sys
15
import shutil
16
import subprocess
17
import tempfile
18
import re
19
20
if __package__ in (None, ""):
21
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
from oakbench.environment import base_env
23
24
VERSION = "2026-09-11.2"
25
SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
26
ASSET_BYTES = 16 * 1024 * 1024
27
28
# Preserved fixture instructions, not programs executed by this oracle.
29
CONFIG_VALIDATOR = "\n".join([
30
    "from pathlib import Path", "bad = []",
31
    "for p in Path('services').glob('*/config.toml'):",
32
    "    text = p.read_text()", "    if 'timeout_ms = 2500' in text:",
33
    "        bad.append(str(p))", "print(f'bad={len(bad)}')",
34
    "raise SystemExit(1 if bad else 0)", "",
35
]).encode()
36
ASSET_VALIDATOR = "\n".join([
37
    "import json", "from pathlib import Path",
38
    "data = json.loads(Path('manifest.json').read_text())",
39
    "for item in data['assets']:", "    p = Path(item['path'])",
40
    "    assert p.exists(), p", "    assert p.stat().st_size == item['bytes'], p",
41
    "    assert item['reviewed'] is True, p", "print('manifest ok')", "",
42
]).encode()
43
44
45
def _regular_file(path: Path) -> bool:
46
    return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
47
48
49
def _tree_paths(root: Path) -> set[str]:
50
    """Return every descendant identity, rejecting links and special files."""
51
    if root.is_symlink() or not root.is_dir():
52
        raise ValueError("payload_directory_missing_or_link")
53
    result = set()
54
    for parent, dirs, files in os.walk(root, followlinks=False):
55
        for name in dirs + files:
56
            path = Path(parent) / name
57
            if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
58
                raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
59
            result.add(path.relative_to(root).as_posix())
60
    return result
61
62
63
def _config_failure(repo: Path) -> str | None:
64
    expected_paths = {f"service-{i:04d}" for i in range(600)}
65
    expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
66
    actual = _tree_paths(repo / "services")
67
    if actual != expected_paths:
68
        return "config_path_set_mismatch"
69
    for i in range(600):
70
        rel = f"services/service-{i:04d}/config.toml"
71
        expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
72
        if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
73
            return "config_payload_mismatch:" + rel
74
    return None
75
76
77
def _unique_object(pairs):
78
    result = {}
79
    for key, value in pairs:
80
        if key in result:
81
            raise ValueError("duplicate_json_key:" + key)
82
        result[key] = value
83
    return result
84
85
86
@lru_cache(maxsize=4)
87
def _expected_asset_hash(index: int) -> str:
88
    unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
89
    digest = hashlib.sha256()
90
    remaining = ASSET_BYTES
91
    while remaining:
92
        chunk = unit[:min(len(unit), remaining)]
93
        digest.update(chunk)
94
        remaining -= len(chunk)
95
    return digest.hexdigest()
96
97
98
def _asset_failure(repo: Path) -> str | None:
99
    expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
100
    if _tree_paths(repo / "assets") != expected_paths:
101
        return "asset_path_set_mismatch"
102
    manifest = repo / "manifest.json"
103
    if not _regular_file(manifest):
104
        return "manifest_missing_or_wrong_type"
105
    if manifest.stat().st_size > 64 * 1024:
106
        return "manifest_size_out_of_bounds"
107
    data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
108
    if not isinstance(data, dict) or set(data) != {"assets"}:
109
        return "manifest_object_shape"
110
    entries = data["assets"]
111
    if not isinstance(entries, list) or len(entries) != 4:
112
        return "manifest_asset_count"
113
    seen = set()
114
    for item in entries:
115
        if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
116
            return "manifest_entry_shape"
117
        path = item["path"]
118
        if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
119
            return "manifest_asset_identity"
120
        if path in seen:
121
            return "manifest_duplicate_asset:" + path
122
        seen.add(path)
123
        if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
124
            return "manifest_asset_fields:" + path
125
    for index in range(4):
126
        path = repo / "assets" / f"asset-{index:02d}.bin"
127
        if path.stat().st_size != ASSET_BYTES:
128
            return "asset_size_mismatch:" + path.name
129
        digest = hashlib.sha256()
130
        with path.open("rb") as stream:
131
            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
132
                digest.update(chunk)
133
        if digest.hexdigest() != _expected_asset_hash(index):
134
            return "asset_content_mismatch:" + path.name
135
    return None
136
137
138
def validate(scenario: str, repo: Path) -> str | None:
139
    """Return a concrete failure reason, or None on a fully verified task."""
140
    if scenario not in SCENARIOS:
141
        return "unsupported_trusted_task_oracle:" + scenario
142
    name, expected_validator = (
143
        ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
144
        else ("validate_manifest.py", ASSET_VALIDATOR)
145
    )
146
    try:
147
        path = repo / name
148
        if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
149
                or path.read_bytes() != expected_validator):
150
            return "fixture_validator_modified_or_missing:" + name
151
        return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
152
    except (OSError, ValueError, UnicodeError, RecursionError) as exc:
153
        return "trusted_payload_unreadable_or_invalid:" + str(exc)
154
155
156
def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
157
    """Resolve an explicitly configured instrument or the invoking PATH once."""
158
    requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
159
    resolved = shutil.which(requested)
160
    metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
161
                "git_head_instrument_version": None}
162
    if resolved is None:
163
        return None, metadata
164
    binary = str(Path(resolved).resolve())
165
    try:
166
        version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
167
        if version.returncode:
168
            return None, metadata
169
        metadata.update(git_head_instrument_binary=binary,
170
                        git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
171
                        git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
172
        return binary, metadata
173
    except (OSError, subprocess.TimeoutExpired):
174
        return None, metadata
175
176
177
def git_head_text(repo: Path, path: str, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> tuple[str | None, str | None]:
178
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
179
    if git is None:
180
        return None, "git_head_oracle_unavailable"
181
    try:
182
        result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
183
                                env=base_env(), capture_output=True, timeout=30)
184
        if result.returncode < 0:
185
            return None, "git_head_oracle_unavailable"
186
        return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
187
    except (OSError, subprocess.TimeoutExpired):
188
        return None, "git_head_oracle_unavailable"
189
190
191
def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> str | None:
192
    """Verify the entire task subtree from committed objects using system Git.
193
194
    Never checks out subject-controlled hooks/filters or extracts an archive.
195
    Enumerated blob paths and bounded sizes are checked before materialization.
196
    Git object replacement and inherited Git environment overrides are disabled.
197
    """
198
    git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
199
    if git is None:
200
        return "git_head_oracle_unavailable"
201
    if scenario == "wide_config_refactor":
202
        paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
203
        paths.add("validate_configs.py")
204
        scope = ["services", "validate_configs.py"]
205
        max_blob_size = 4096
206
    elif scenario == "large_asset_manifest":
207
        paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
208
        paths.update({"manifest.json", "validate_manifest.py"})
209
        scope = ["assets", "manifest.json", "validate_manifest.py"]
210
        max_blob_size = ASSET_BYTES
211
    else:
212
        return "unsupported_trusted_task_oracle:" + scenario
213
    env = base_env()
214
    command = [git, "-C", str(repo.resolve())]
215
    try:
216
        tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
217
                              env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
218
        if tree.returncode < 0:
219
            return "git_head_oracle_unavailable"
220
        if tree.returncode:
221
            return "git_head_tree_unreadable"
222
        entries = {}
223
        for raw in tree.stdout.split(b"\0"):
224
            if not raw:
225
                continue
226
            header, rel = raw.split(b"\t", 1)
227
            mode, kind, oid = header.split()
228
            path = rel.decode("utf-8")
229
            if mode != b"100644" or kind != b"blob":
230
                return "git_head_path_type:" + path
231
            if path not in paths or path in entries:
232
                return "git_head_path_set_mismatch"
233
            if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
234
                return "git_head_invalid_object_id"
235
            entries[path] = oid
236
        if set(entries) != paths:
237
            return "git_head_path_set_mismatch"
238
        ordered = sorted(entries)
239
        request = b"\n".join(entries[path] for path in ordered) + b"\n"
240
        sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
241
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
242
        if sizes.returncode < 0:
243
            return "git_head_oracle_unavailable"
244
        if sizes.returncode:
245
            return "git_head_blob_metadata_unreadable"
246
        headers = sizes.stdout.splitlines()
247
        if len(headers) != len(ordered):
248
            return "git_head_blob_metadata_incomplete"
249
        for header in headers:
250
            fields = header.split()
251
            if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
252
                return "git_head_blob_metadata_invalid"
253
            if int(fields[2]) > max_blob_size:
254
                return "git_head_blob_size_out_of_bounds"
255
        blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
256
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
257
        if blobs.returncode < 0:
258
            return "git_head_oracle_unavailable"
259
        if blobs.returncode:
260
            return "git_head_blob_unreadable"
261
        offset = 0
262
        with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
263
            root = Path(tmp)
264
            for path, expected_header in zip(ordered, headers):
265
                end = blobs.stdout.find(b"\n", offset)
266
                if end < 0 or blobs.stdout[offset:end] != expected_header:
267
                    return "git_head_blob_framing_invalid"
268
                size = int(expected_header.split()[2])
269
                offset = end + 1
270
                payload = blobs.stdout[offset:offset + size]
271
                if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
272
                    return "git_head_blob_framing_invalid"
273
                offset += size + 1
274
                destination = root / path
275
                destination.parent.mkdir(parents=True, exist_ok=True)
276
                destination.write_bytes(payload)
277
            if offset != len(blobs.stdout):
278
                return "git_head_blob_trailing_bytes"
279
            reason = validate(scenario, root)
280
            return "git_head_" + reason if reason else None
281
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
282
        return "git_head_oracle_error:" + str(exc)
283
284
285
def reader_evidence(kind: str, binary: str | None, metadata: dict) -> dict:
286
    """Classify an executed reader by observed binary bytes, never its label."""
287
    subject_hash = metadata.get("workflow_subject_binary_sha256")
288
    if kind == "oak":
289
        subject_hash = subject_hash or metadata.get("oak_export_instrument_sha256")
290
    try:
291
        resolved = shutil.which(binary or kind) if subject_hash is None else None
292
        if resolved:
293
            if str(Path(resolved).resolve()) == metadata.get("git_head_instrument_binary"):
294
                subject_hash = metadata.get("git_head_instrument_sha256")
295
            else:
296
                subject_hash = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
297
    except OSError:
298
        pass
299
    instrument_hash = metadata.get("git_head_instrument_sha256")
300
    evidence_class = "unverified_committed_tree"
301
    if kind == "oak":
302
        evidence_class = "subject_produced_committed_tree"
303
    elif subject_hash and instrument_hash:
304
        evidence_class = "subject_binary_committed_tree" if subject_hash == instrument_hash else "independent_committed_tree"
305
    return {"workflow_subject_binary_sha256": subject_hash, "workflow_integrity_evidence_class": evidence_class}
306
307
308
def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None) -> dict:
309
    source = "trusted_exact_task_worktree"
310
    evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
311
                "workflow_integrity_evidence_class": "worktree_payload",
312
                "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
313
                "workflow_integrity_unmeasured_reason": None}
314
    reason = validate(scenario, repo)
315
    if reason:
316
        evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
317
        return evidence
318
    if kind in {"git", "oak"}:
319
        git, metadata = resolve_git_instrument(git_instrument)
320
        evidence.update(metadata)
321
        if git is None:
322
            evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
323
            return evidence
324
        if kind == "git":
325
            evidence["workflow_integrity_evidence_class"] = "committed_tree_attempt"
326
            evidence["workflow_integrity_source"] = source + ";git_head_read_attempt"
327
            reason = validate_git_head(scenario, repo, resolved_instrument=(git, metadata))
328
        else:
329
            oak = shutil.which(binary or "oak")
330
            if oak is None:
331
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
332
                return evidence
333
            oak = str(Path(oak).resolve())
334
            evidence["oak_export_instrument_binary"] = oak
335
            try:
336
                evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
337
                status = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
338
                before = json.loads(status.stdout) if status.returncode == 0 else {}
339
                head, branch = before.get("head"), before.get("branch")
340
                if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
341
                    raise ValueError("oak_head_identity_unreadable")
342
                evidence["oak_export_head"] = head
343
                with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
344
                    destination = Path(tmp) / "export"
345
                    evidence["workflow_integrity_evidence_class"] = "subject_produced_committed_tree_attempt"
346
                    evidence["workflow_integrity_source"] = source + ";subject_produced_oak_head_export_attempt"
347
                    exported = subprocess.run([oak, "export", "--branch", branch, str(destination)], cwd=repo,
348
                                              env=base_env(), capture_output=True, timeout=120)
349
                    if exported.returncode:
350
                        evidence["workflow_integrity_unmeasured_reason"] = "oak_head_export_unavailable"
351
                        return evidence
352
                    after = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
353
                    state = json.loads(after.stdout) if after.returncode == 0 else {}
354
                    if state.get("head") != head or state.get("branch") != branch:
355
                        raise ValueError("oak_head_changed_during_export")
356
                    reason = validate_git_head(scenario, destination, resolved_instrument=(git, metadata))
357
            except (OSError, ValueError, subprocess.TimeoutExpired) as exc:
358
                evidence["workflow_integrity_unmeasured_reason"] = str(exc)
359
                return evidence
360
    if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
361
        evidence["workflow_integrity_unmeasured_reason"] = reason
362
        return evidence
363
    if reason is None and kind in {"git", "oak"}:
364
        evidence.update(reader_evidence(kind, binary, evidence))
365
        suffix = "subject_produced_oak_head_export" if kind == "oak" else evidence["workflow_integrity_evidence_class"]
366
        evidence["workflow_integrity_source"] = source + ";" + suffix
367
    evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
368
    return evidence
369
370
371
def main() -> int:
372
    import argparse
373
    parser = argparse.ArgumentParser()
374
    parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
375
    parser.add_argument("--git-head", action="store_true")
376
    parser.add_argument("--oak-head", action="store_true")
377
    parser.add_argument("--binary")
378
    parser.add_argument("--evidence-json", action="store_true")
379
    parser.add_argument("--git-instrument")
380
    args = parser.parse_args()
381
    evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument)
382
    reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
383
    print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
384
    passed = evidence["workflow_integrity_passed"]
385
    return 77 if passed is None else 0 if passed else 1
386
387
388
if __name__ == "__main__":
389
    raise SystemExit(main())