Viewing files at commit 2735d7f5558eCommit detailsView latest

Browse files
scripts/oakbench/workflow_oracles.py 362 lines · 17.2 KB · python
1
"""Trusted exact task payload checks, outside the agent-editable checkout.
2
3
This module is stdlib-only and can run with Python -I as an external oracle.
4
It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
5
600 canonical service configs and four 16 MiB deterministic binary assets.
6
"""
7
from __future__ import annotations
8
9
from functools import lru_cache
10
import hashlib
11
import json
12
import os
13
from pathlib import Path
14
import sys
15
import shutil
16
import subprocess
17
import tempfile
18
import re
19
20
if __package__ in (None, ""):
21
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
from oakbench.environment import base_env
23
24
VERSION = "2026-09-11.1"
25
SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
26
ASSET_BYTES = 16 * 1024 * 1024
27
28
# Preserved fixture instructions, not programs executed by this oracle.
29
CONFIG_VALIDATOR = "\n".join([
30
    "from pathlib import Path", "bad = []",
31
    "for p in Path('services').glob('*/config.toml'):",
32
    "    text = p.read_text()", "    if 'timeout_ms = 2500' in text:",
33
    "        bad.append(str(p))", "print(f'bad={len(bad)}')",
34
    "raise SystemExit(1 if bad else 0)", "",
35
]).encode()
36
ASSET_VALIDATOR = "\n".join([
37
    "import json", "from pathlib import Path",
38
    "data = json.loads(Path('manifest.json').read_text())",
39
    "for item in data['assets']:", "    p = Path(item['path'])",
40
    "    assert p.exists(), p", "    assert p.stat().st_size == item['bytes'], p",
41
    "    assert item['reviewed'] is True, p", "print('manifest ok')", "",
42
]).encode()
43
44
45
def _regular_file(path: Path) -> bool:
46
    return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
47
48
49
def _tree_paths(root: Path) -> set[str]:
50
    """Return every descendant identity, rejecting links and special files."""
51
    if root.is_symlink() or not root.is_dir():
52
        raise ValueError("payload_directory_missing_or_link")
53
    result = set()
54
    for parent, dirs, files in os.walk(root, followlinks=False):
55
        for name in dirs + files:
56
            path = Path(parent) / name
57
            if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
58
                raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
59
            result.add(path.relative_to(root).as_posix())
60
    return result
61
62
63
def _config_failure(repo: Path) -> str | None:
64
    expected_paths = {f"service-{i:04d}" for i in range(600)}
65
    expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
66
    actual = _tree_paths(repo / "services")
67
    if actual != expected_paths:
68
        return "config_path_set_mismatch"
69
    for i in range(600):
70
        rel = f"services/service-{i:04d}/config.toml"
71
        expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
72
        if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
73
            return "config_payload_mismatch:" + rel
74
    return None
75
76
77
def _unique_object(pairs):
78
    result = {}
79
    for key, value in pairs:
80
        if key in result:
81
            raise ValueError("duplicate_json_key:" + key)
82
        result[key] = value
83
    return result
84
85
86
@lru_cache(maxsize=4)
87
def _expected_asset_hash(index: int) -> str:
88
    unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
89
    digest = hashlib.sha256()
90
    remaining = ASSET_BYTES
91
    while remaining:
92
        chunk = unit[:min(len(unit), remaining)]
93
        digest.update(chunk)
94
        remaining -= len(chunk)
95
    return digest.hexdigest()
96
97
98
def _asset_failure(repo: Path) -> str | None:
99
    expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
100
    if _tree_paths(repo / "assets") != expected_paths:
101
        return "asset_path_set_mismatch"
102
    manifest = repo / "manifest.json"
103
    if not _regular_file(manifest):
104
        return "manifest_missing_or_wrong_type"
105
    if manifest.stat().st_size > 64 * 1024:
106
        return "manifest_size_out_of_bounds"
107
    data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
108
    if not isinstance(data, dict) or set(data) != {"assets"}:
109
        return "manifest_object_shape"
110
    entries = data["assets"]
111
    if not isinstance(entries, list) or len(entries) != 4:
112
        return "manifest_asset_count"
113
    seen = set()
114
    for item in entries:
115
        if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
116
            return "manifest_entry_shape"
117
        path = item["path"]
118
        if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
119
            return "manifest_asset_identity"
120
        if path in seen:
121
            return "manifest_duplicate_asset:" + path
122
        seen.add(path)
123
        if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
124
            return "manifest_asset_fields:" + path
125
    for index in range(4):
126
        path = repo / "assets" / f"asset-{index:02d}.bin"
127
        if path.stat().st_size != ASSET_BYTES:
128
            return "asset_size_mismatch:" + path.name
129
        digest = hashlib.sha256()
130
        with path.open("rb") as stream:
131
            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
132
                digest.update(chunk)
133
        if digest.hexdigest() != _expected_asset_hash(index):
134
            return "asset_content_mismatch:" + path.name
135
    return None
136
137
138
def validate(scenario: str, repo: Path) -> str | None:
139
    """Return a concrete failure reason, or None on a fully verified task."""
140
    if scenario not in SCENARIOS:
141
        return "unsupported_trusted_task_oracle:" + scenario
142
    name, expected_validator = (
143
        ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
144
        else ("validate_manifest.py", ASSET_VALIDATOR)
145
    )
146
    try:
147
        path = repo / name
148
        if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
149
                or path.read_bytes() != expected_validator):
150
            return "fixture_validator_modified_or_missing:" + name
151
        return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
152
    except (OSError, ValueError, UnicodeError, RecursionError) as exc:
153
        return "trusted_payload_unreadable_or_invalid:" + str(exc)
154
155
156
def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
157
    """Resolve an explicitly configured instrument or the invoking PATH once."""
158
    requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
159
    resolved = shutil.which(requested)
160
    metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
161
                "git_head_instrument_version": None}
162
    if resolved is None:
163
        return None, metadata
164
    binary = str(Path(resolved).resolve())
165
    try:
166
        version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
167
        if version.returncode:
168
            return None, metadata
169
        metadata.update(git_head_instrument_binary=binary,
170
                        git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
171
                        git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
172
        return binary, metadata
173
    except (OSError, subprocess.TimeoutExpired):
174
        return None, metadata
175
176
177
def git_head_text(repo: Path, path: str, git_binary: str | None = None) -> tuple[str | None, str | None]:
178
    git, _ = resolve_git_instrument(git_binary)
179
    if git is None:
180
        return None, "git_head_oracle_unavailable"
181
    try:
182
        result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
183
                                env=base_env(), capture_output=True, timeout=30)
184
        if result.returncode < 0:
185
            return None, "git_head_oracle_unavailable"
186
        return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
187
    except (OSError, subprocess.TimeoutExpired):
188
        return None, "git_head_oracle_unavailable"
189
190
191
def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None) -> str | None:
192
    """Verify the entire task subtree from committed objects using system Git.
193
194
    Never checks out subject-controlled hooks/filters or extracts an archive.
195
    Enumerated blob paths and bounded sizes are checked before materialization.
196
    Git object replacement and inherited Git environment overrides are disabled.
197
    """
198
    git, _ = resolve_git_instrument(git_binary)
199
    if git is None:
200
        return "git_head_oracle_unavailable"
201
    if scenario == "wide_config_refactor":
202
        paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
203
        paths.add("validate_configs.py")
204
        scope = ["services", "validate_configs.py"]
205
        max_blob_size = 4096
206
    elif scenario == "large_asset_manifest":
207
        paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
208
        paths.update({"manifest.json", "validate_manifest.py"})
209
        scope = ["assets", "manifest.json", "validate_manifest.py"]
210
        max_blob_size = ASSET_BYTES
211
    else:
212
        return "unsupported_trusted_task_oracle:" + scenario
213
    env = base_env()
214
    command = [git, "-C", str(repo.resolve())]
215
    try:
216
        tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
217
                              env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
218
        if tree.returncode < 0:
219
            return "git_head_oracle_unavailable"
220
        if tree.returncode:
221
            return "git_head_tree_unreadable"
222
        entries = {}
223
        for raw in tree.stdout.split(b"\0"):
224
            if not raw:
225
                continue
226
            header, rel = raw.split(b"\t", 1)
227
            mode, kind, oid = header.split()
228
            path = rel.decode("utf-8")
229
            if mode != b"100644" or kind != b"blob":
230
                return "git_head_path_type:" + path
231
            if path not in paths or path in entries:
232
                return "git_head_path_set_mismatch"
233
            if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
234
                return "git_head_invalid_object_id"
235
            entries[path] = oid
236
        if set(entries) != paths:
237
            return "git_head_path_set_mismatch"
238
        ordered = sorted(entries)
239
        request = b"\n".join(entries[path] for path in ordered) + b"\n"
240
        sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
241
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
242
        if sizes.returncode < 0:
243
            return "git_head_oracle_unavailable"
244
        if sizes.returncode:
245
            return "git_head_blob_metadata_unreadable"
246
        headers = sizes.stdout.splitlines()
247
        if len(headers) != len(ordered):
248
            return "git_head_blob_metadata_incomplete"
249
        for header in headers:
250
            fields = header.split()
251
            if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
252
                return "git_head_blob_metadata_invalid"
253
            if int(fields[2]) > max_blob_size:
254
                return "git_head_blob_size_out_of_bounds"
255
        blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
256
                               env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
257
        if blobs.returncode < 0:
258
            return "git_head_oracle_unavailable"
259
        if blobs.returncode:
260
            return "git_head_blob_unreadable"
261
        offset = 0
262
        with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
263
            root = Path(tmp)
264
            for path, expected_header in zip(ordered, headers):
265
                end = blobs.stdout.find(b"\n", offset)
266
                if end < 0 or blobs.stdout[offset:end] != expected_header:
267
                    return "git_head_blob_framing_invalid"
268
                size = int(expected_header.split()[2])
269
                offset = end + 1
270
                payload = blobs.stdout[offset:offset + size]
271
                if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
272
                    return "git_head_blob_framing_invalid"
273
                offset += size + 1
274
                destination = root / path
275
                destination.parent.mkdir(parents=True, exist_ok=True)
276
                destination.write_bytes(payload)
277
            if offset != len(blobs.stdout):
278
                return "git_head_blob_trailing_bytes"
279
            reason = validate(scenario, root)
280
            return "git_head_" + reason if reason else None
281
    except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
282
        return "git_head_oracle_error:" + str(exc)
283
284
285
def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None) -> dict:
286
    source = "trusted_exact_task_worktree"
287
    if kind == "git":
288
        source += ";known_good_git_head_tree"
289
    elif kind == "oak":
290
        source += ";subject_produced_oak_head_export"
291
    evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
292
                "workflow_integrity_evidence_class": "independent_committed_tree" if kind == "git" else "subject_produced_committed_tree" if kind == "oak" else "worktree_payload",
293
                "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
294
                "workflow_integrity_unmeasured_reason": None}
295
    reason = validate(scenario, repo)
296
    if reason:
297
        evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
298
        return evidence
299
    if kind in {"git", "oak"}:
300
        git, metadata = resolve_git_instrument(git_instrument)
301
        evidence.update(metadata)
302
        if git is None:
303
            evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
304
            return evidence
305
        if kind == "git":
306
            reason = validate_git_head(scenario, repo, git)
307
        else:
308
            oak = shutil.which(binary or "oak")
309
            if oak is None:
310
                evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
311
                return evidence
312
            oak = str(Path(oak).resolve())
313
            evidence["oak_export_instrument_binary"] = oak
314
            try:
315
                evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
316
                status = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
317
                before = json.loads(status.stdout) if status.returncode == 0 else {}
318
                head, branch = before.get("head"), before.get("branch")
319
                if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
320
                    raise ValueError("oak_head_identity_unreadable")
321
                evidence["oak_export_head"] = head
322
                with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
323
                    destination = Path(tmp) / "export"
324
                    exported = subprocess.run([oak, "export", "--branch", branch, str(destination)], cwd=repo,
325
                                              env=base_env(), capture_output=True, timeout=120)
326
                    if exported.returncode:
327
                        evidence["workflow_integrity_unmeasured_reason"] = "oak_head_export_unavailable"
328
                        return evidence
329
                    after = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
330
                    state = json.loads(after.stdout) if after.returncode == 0 else {}
331
                    if state.get("head") != head or state.get("branch") != branch:
332
                        raise ValueError("oak_head_changed_during_export")
333
                    reason = validate_git_head(scenario, destination, git)
334
            except (OSError, ValueError, subprocess.TimeoutExpired) as exc:
335
                evidence["workflow_integrity_unmeasured_reason"] = str(exc)
336
                return evidence
337
    if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
338
        evidence["workflow_integrity_unmeasured_reason"] = reason
339
        return evidence
340
    evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
341
    return evidence
342
343
344
def main() -> int:
345
    import argparse
346
    parser = argparse.ArgumentParser()
347
    parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
348
    parser.add_argument("--git-head", action="store_true")
349
    parser.add_argument("--oak-head", action="store_true")
350
    parser.add_argument("--binary")
351
    parser.add_argument("--evidence-json", action="store_true")
352
    parser.add_argument("--git-instrument")
353
    args = parser.parse_args()
354
    evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument)
355
    reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
356
    print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
357
    passed = evidence["workflow_integrity_passed"]
358
    return 77 if passed is None else 0 if passed else 1
359
360
361
if __name__ == "__main__":
362
    raise SystemExit(main())