Viewing files at commit fcd8a785ee1aCommit detailsView latest

Browse files
scripts/oakbench/workflow_oracles.py 380 lines · 18.0 KB · python
1
"""Trusted exact task payload checks, outside the agent-editable checkout.
2
3
This module is stdlib-only and can run with Python -I as an external oracle.
4
It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
5
600 canonical service configs and four 16 MiB deterministic binary assets.
6
"""
7
from __future__ import annotations
8
9
from functools import lru_cache
10
import hashlib
11
import json
12
import os
13
from pathlib import Path
14
import sys
15
import shutil
16
import subprocess
17
import tempfile
18
import re
19
20
if __package__ in (None, ""):
21
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
from oakbench.environment import base_env
23
24
VERSION = "2026-09-11.2"
25
SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
26
ASSET_BYTES = 16 * 1024 * 1024
27
28
# Preserved fixture instructions, not programs executed by this oracle.
29
CONFIG_VALIDATOR = "\n".join([
30
    "from pathlib import Path", "bad = []",
31
    "for p in Path('services').glob('*/config.toml'):",
32
    "    text = p.read_text()", "    if 'timeout_ms = 2500' in text:",
33
    "        bad.append(str(p))", "print(f'bad={len(bad)}')",
34
    "raise SystemExit(1 if bad else 0)", "",
35
]).encode()
36
ASSET_VALIDATOR = "\n".join([
37
    "import json", "from pathlib import Path",
38
    "data = json.loads(Path('manifest.json').read_text())",
39
    "for item in data['assets']:", "    p = Path(item['path'])",
40
    "    assert p.exists(), p", "    assert p.stat().st_size == item['bytes'], p",
41
    "    assert item['reviewed'] is True, p", "print('manifest ok')", "",
42
]).encode()
43
44
45
def _regular_file(path: Path) -> bool:
46
    return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
47
48
49
def _tree_paths(root: Path) -> set[str]:
50
    """Return every descendant identity, rejecting links and special files."""
51
    if root.is_symlink() or not root.is_dir():
52
        raise ValueError("payload_directory_missing_or_link")
53
    result = set()
54
    for parent, dirs, files in os.walk(root, followlinks=False):
55
        for name in dirs + files:
56
            path = Path(parent) / name
57
            if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
58
                raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
59
            result.add(path.relative_to(root).as_posix())
60
    return result
61
62
63
def _config_failure(repo: Path) -> str | None:
64
    expected_paths = {f"service-{i:04d}" for i in range(600)}
65
    expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
66
    actual = _tree_paths(repo / "services")
67
    if actual != expected_paths:
68
        return "config_path_set_mismatch"
69
    for i in range(600):
70
        rel = f"services/service-{i:04d}/config.toml"
71
        expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
72
        if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
73
            return "config_payload_mismatch:" + rel
74
    return None
75
76
77
def _unique_object(pairs):
78
    result = {}
79
    for key, value in pairs:
80
        if key in result:
81
            raise ValueError("duplicate_json_key:" + key)
82
        result[key] = value
83
    return result
84
85
86
@lru_cache(maxsize=4)
87
def _expected_asset_hash(index: int) -> str:
88
    unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
89
    digest = hashlib.sha256()
90
    remaining = ASSET_BYTES
91
    while remaining:
92
        chunk = unit[:min(len(unit), remaining)]
93
        digest.update(chunk)
94
        remaining -= len(chunk)
95
    return digest.hexdigest()
96
97
98
def _asset_failure(repo: Path) -> str | None:
99
    expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
100
    if _tree_paths(repo / "assets") != expected_paths:
101
        return "asset_path_set_mismatch"
102
    manifest = repo / "manifest.json"
103
    if not _regular_file(manifest):
104
        return "manifest_missing_or_wrong_type"
105
    if manifest.stat().st_size > 64 * 1024:
106
        return "manifest_size_out_of_bounds"
107
    data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
108
    if not isinstance(data, dict) or set(data) != {"assets"}:
109
        return "manifest_object_shape"
110
    entries = data["assets"]
111
    if not isinstance(entries, list) or len(entries) != 4:
112
        return "manifest_asset_count"
113
    seen = set()
114
    for item in entries:
115
        if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
116
            return "manifest_entry_shape"
117
        path = item["path"]
118
        if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
119
            return "manifest_asset_identity"
120
        if path in seen:
121
            return "manifest_duplicate_asset:" + path
122
        seen.add(path)
123
        if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
124
            return "manifest_asset_fields:" + path
125
    for index in range(4):
126
        path = repo / "assets" / f"asset-{index:02d}.bin"
127
        if path.stat().st_size != ASSET_BYTES:
128
            return "asset_size_mismatch:" + path.name
129
        digest = hashlib.sha256()
130
        with path.open("rb") as stream:
131
            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
132
                digest.update(chunk)
133
        if digest.hexdigest() != _expected_asset_hash(index):
134
            return "asset_content_mismatch:" + path.name
135
    return None
136
137
138
def validate(scenario: str, repo: Path) -> str | None:
139
    """Return a concrete failure reason, or None on a fully verified task."""
140
    if scenario not in SCENARIOS:
141
        return "unsupported_trusted_task_oracle:" + scenario
142
    name, expected_validator = (
143
        ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
144
        else ("validate_manifest.py", ASSET_VALIDATOR)
145
    )
146
    try:
147
        path = repo / name
148
        if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
149
                or path.read_bytes() != expected_validator):
150
            return "fixture_validator_modified_or_missing:" + name
151
        return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
152
    except (OSError, ValueError, UnicodeError, RecursionError) as exc:
153
        return "trusted_payload_unreadable_or_invalid:" + str(exc)
154
155
156
def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
157
    """Resolve an explicitly configured instrument or the invoking PATH once."""
158
    requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
159
    resolved = shutil.which(requested)
160
    metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
161
                "git_head_instrument_version": None}
162
    if resolved is None:
163
        return None, metadata
164
    binary = str(Path(resolved).resolve())
165
    try:
166
        version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
167
        if version.returncode:
168
            return None, metadata
169
        metadata.update(git_head_instrument_binary=binary,
170
                        git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
171
                        git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
172
        return binary, metadata
173
    except (OSError, subprocess.TimeoutExpired):
174
        return None, metadata
175
176
177
def git_head_text(repo: Path, path: str, git_binary: str | None = None) -> tuple[str | None, str | None]:
178
    git, _ = resolve_git_instrument(git_binary)
179
    if git is None:
180
        return None, "git_head_oracle_unavailable"
181
    try:
182
        result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
183
                                env=base_env(), capture_output=True, timeout=30)
184
        if result.returncode < 0:
185
            return None, "git_head_oracle_unavailable"
186
        return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
187
    except (OSError, subprocess.TimeoutExpired):
188
        return None, "git_head_oracle_unavailable"
189
190
191
def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None) -> str | None:
192
    """Verify the entire task subtree from committed objects using system Git.
193
194
    Never checks out subject-controlled hooks/filters or extracts an archive.
195
    Enumerated blob paths and bounded sizes are checked before materialization.
196
    Git object replacement and inherited Git environment overrides are disabled.
197
    """
198
    git, _ = resolve_git_instrument(git_binary)
199
    if git is None:
200
        return "git_head_oracle_unavailable"
201
    if scenario == "wide_config_refactor":
202
        paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
203
        paths.add("validate_configs.py")
204
        scope = ["services", "validate_configs.py"]
205
        max_blob_size = 4096
206
    elif scenario == "large_asset_manifest":
207
        paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
208
        paths.update({"manifest.json", "validate_manifest.py"})
209
        scope = ["assets", "manifest.json", "validate_manifest.py"]
210
        max_blob_size = ASSET_BYTES
211
    else:
212
        return "unsupported_trusted_task_oracle:" + scenario
213
    env = base_env()
214
    command = [git, "-C", str(repo.resolve())]
215
    try:
216
        tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
217
                              env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
218
        if tree.returncode < 0:
219
            return "git_head_oracle_unavailable"
220
        if tree.returncode:
221
            return "git_head_tree_unreadable"
222
        entries = {}
223
        for raw in tree.stdout.split(b"\0"):
224
            if not raw:
225
                continue
226
            header, rel = raw.split(b"\t", 1)
227
            mode, kind, oid = header.split()
228
            path = rel.decode("utf-8")
229
            if mode != b"100644" or kind != b"blob":
230
                return "git_head_path_type:" + path
231
            if path not in paths or path in entries:
232
                return "git_head_path_set_mismatch"
233
            if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
234
                return "git_head_invalid_object_id"
235
            entries[path] = oid
236
        if set(entries) != paths:
237
            return "git_head_path_set_mismatch"
238
        ordered = sorted(entries)
239
        request = b"\n".join(entries[path] for path in ordered) + b"\n"
240
        sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
241
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
242
        if sizes.returncode < 0:
243
            return "git_head_oracle_unavailable"
244
        if sizes.returncode:
245
            return "git_head_blob_metadata_unreadable"
246
        headers = sizes.stdout.splitlines()
247
        if len(headers) != len(ordered):
248
            return "git_head_blob_metadata_incomplete"
249
        for header in headers:
250
            fields = header.split()
251
            if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
252
                return "git_head_blob_metadata_invalid"
253
            if int(fields[2]) > max_blob_size:
254
                return "git_head_blob_size_out_of_bounds"
255
        blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
256
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
257
        if blobs.returncode < 0:
258
            return "git_head_oracle_unavailable"
259
        if blobs.returncode:
260
            return "git_head_blob_unreadable"
261
        offset = 0
262
        with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
263
            root = Path(tmp)
264
            for path, expected_header in zip(ordered, headers):
265
                end = blobs.stdout.find(b"\n", offset)
266
                if end < 0 or blobs.stdout[offset:end] != expected_header:
267
                    return "git_head_blob_framing_invalid"
268
                size = int(expected_header.split()[2])
269
                offset = end + 1
270
                payload = blobs.stdout[offset:offset + size]
271
                if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
272
                    return "git_head_blob_framing_invalid"
273
                offset += size + 1
274
                destination = root / path
275
                destination.parent.mkdir(parents=True, exist_ok=True)
276
                destination.write_bytes(payload)
277
            if offset != len(blobs.stdout):
278
                return "git_head_blob_trailing_bytes"
279
            reason = validate(scenario, root)
280
            return "git_head_" + reason if reason else None
281
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
282
        return "git_head_oracle_error:" + str(exc)
283
284
285
def reader_evidence(kind: str, binary: str | None, metadata: dict) -> dict:
286
    """Classify an executed reader by observed binary bytes, never its label."""
287
    subject_hash = None
288
    try:
289
        resolved = shutil.which(binary or kind)
290
        if resolved:
291
            subject_hash = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
292
    except OSError:
293
        pass
294
    instrument_hash = metadata.get("git_head_instrument_sha256")
295
    evidence_class = "unverified_committed_tree"
296
    if kind == "oak":
297
        evidence_class = "subject_produced_committed_tree"
298
    elif subject_hash and instrument_hash:
299
        evidence_class = "subject_binary_committed_tree" if subject_hash == instrument_hash else "independent_committed_tree"
300
    return {"workflow_subject_binary_sha256": subject_hash, "workflow_integrity_evidence_class": evidence_class}
301
302
303
def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None) -> dict:
304
    source = "trusted_exact_task_worktree"
305
    evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
306
                "workflow_integrity_evidence_class": "worktree_payload",
307
                "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
308
                "workflow_integrity_unmeasured_reason": None}
309
    reason = validate(scenario, repo)
310
    if reason:
311
        evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
312
        return evidence
313
    if kind in {"git", "oak"}:
314
        git, metadata = resolve_git_instrument(git_instrument)
315
        evidence.update(metadata)
316
        if git is None:
317
            evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
318
            return evidence
319
        if kind == "git":
320
            reason = validate_git_head(scenario, repo, git)
321
        else:
322
            oak = shutil.which(binary or "oak")
323
            if oak is None:
324
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
325
                return evidence
326
            oak = str(Path(oak).resolve())
327
            evidence["oak_export_instrument_binary"] = oak
328
            try:
329
                evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
330
                status = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
331
                before = json.loads(status.stdout) if status.returncode == 0 else {}
332
                head, branch = before.get("head"), before.get("branch")
333
                if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
334
                    raise ValueError("oak_head_identity_unreadable")
335
                evidence["oak_export_head"] = head
336
                with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
337
                    destination = Path(tmp) / "export"
338
                    exported = subprocess.run([oak, "export", "--branch", branch, str(destination)], cwd=repo,
339
                                              env=base_env(), capture_output=True, timeout=120)
340
                    if exported.returncode:
341
                        evidence["workflow_integrity_unmeasured_reason"] = "oak_head_export_unavailable"
342
                        return evidence
343
                    after = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
344
                    state = json.loads(after.stdout) if after.returncode == 0 else {}
345
                    if state.get("head") != head or state.get("branch") != branch:
346
                        raise ValueError("oak_head_changed_during_export")
347
                    reason = validate_git_head(scenario, destination, git)
348
            except (OSError, ValueError, subprocess.TimeoutExpired) as exc:
349
                evidence["workflow_integrity_unmeasured_reason"] = str(exc)
350
                return evidence
351
    if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
352
        evidence["workflow_integrity_unmeasured_reason"] = reason
353
        return evidence
354
    if reason is None and kind in {"git", "oak"}:
355
        evidence.update(reader_evidence(kind, binary, metadata))
356
        suffix = "subject_produced_oak_head_export" if kind == "oak" else evidence["workflow_integrity_evidence_class"]
357
        evidence["workflow_integrity_source"] += ";" + suffix
358
    evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
359
    return evidence
360
361
362
def main() -> int:
363
    import argparse
364
    parser = argparse.ArgumentParser()
365
    parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
366
    parser.add_argument("--git-head", action="store_true")
367
    parser.add_argument("--oak-head", action="store_true")
368
    parser.add_argument("--binary")
369
    parser.add_argument("--evidence-json", action="store_true")
370
    parser.add_argument("--git-instrument")
371
    args = parser.parse_args()
372
    evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument)
373
    reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
374
    print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
375
    passed = evidence["workflow_integrity_passed"]
376
    return 77 if passed is None else 0 if passed else 1
377
378
379
if __name__ == "__main__":
380
    raise SystemExit(main())