| 1 | """Trusted exact task payload checks, outside the agent-editable checkout.
|
| 2 |
|
| 3 | This module is stdlib-only and can run with Python -I as an external oracle.
|
| 4 | It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
|
| 5 | 600 canonical service configs and four 16 MiB deterministic binary assets.
|
| 6 | """
|
| 7 | from __future__ import annotations
|
| 8 |
|
| 9 | from functools import lru_cache
|
| 10 | import hashlib
|
| 11 | import json
|
| 12 | import os
|
| 13 | from pathlib import Path
|
| 14 | import sys
|
| 15 | import shutil
|
| 16 | import subprocess
|
| 17 | import tempfile
|
| 18 | import re
|
| 19 |
|
| 20 | if __package__ in (None, ""):
|
| 21 | sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 22 | from oakbench.environment import base_env
|
| 23 |
|
| 24 | VERSION = "2026-09-18.1"
|
| 25 | EXACT_SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
|
| 26 | COMMITTED_PAYLOADS = {
|
| 27 | "bugfix_test_loop": {"app/pricing.py": "return cents - ((cents * percent) // 100)"},
|
| 28 | "vcs_error_recovery": {"README.md": "# recovered"},
|
| 29 | "sync_push_divergence": {"work.txt": "local change", "notes.txt": "upstream change"},
|
| 30 | "sync_pull_upstream": {"notes.txt": "upstream change"},
|
| 31 | "sync_pull_dirty": {"notes.txt": "upstream change"},
|
| 32 | "sync_non_ff_after_amend": {"work.txt": "amended line"},
|
| 33 | }
|
| 34 | SCENARIOS = EXACT_SCENARIOS | frozenset(COMMITTED_PAYLOADS) | {"history_archaeology"}
|
| 35 | PAYLOAD_LIMIT_BYTES = 1024 * 1024
|
| 36 | ASSET_BYTES = 16 * 1024 * 1024
|
| 37 |
|
| 38 | # Preserved fixture instructions, not programs executed by this oracle.
|
| 39 | CONFIG_VALIDATOR = "\n".join([
|
| 40 | "from pathlib import Path", "bad = []",
|
| 41 | "for p in Path('services').glob('*/config.toml'):",
|
| 42 | " text = p.read_text()", " if 'timeout_ms = 2500' in text:",
|
| 43 | " bad.append(str(p))", "print(f'bad={len(bad)}')",
|
| 44 | "raise SystemExit(1 if bad else 0)", "",
|
| 45 | ]).encode()
|
| 46 | ASSET_VALIDATOR = "\n".join([
|
| 47 | "import json", "from pathlib import Path",
|
| 48 | "data = json.loads(Path('manifest.json').read_text())",
|
| 49 | "for item in data['assets']:", " p = Path(item['path'])",
|
| 50 | " assert p.exists(), p", " assert p.stat().st_size == item['bytes'], p",
|
| 51 | " assert item['reviewed'] is True, p", "print('manifest ok')", "",
|
| 52 | ]).encode()
|
| 53 |
|
| 54 |
|
| 55 | def _regular_file(path: Path) -> bool:
|
| 56 | return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
|
| 57 |
|
| 58 |
|
| 59 | def _tree_paths(root: Path) -> set[str]:
|
| 60 | """Return every descendant identity, rejecting links and special files."""
|
| 61 | if root.is_symlink() or not root.is_dir():
|
| 62 | raise ValueError("payload_directory_missing_or_link")
|
| 63 | result = set()
|
| 64 | for parent, dirs, files in os.walk(root, followlinks=False):
|
| 65 | for name in dirs + files:
|
| 66 | path = Path(parent) / name
|
| 67 | if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
|
| 68 | raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
|
| 69 | result.add(path.relative_to(root).as_posix())
|
| 70 | return result
|
| 71 |
|
| 72 |
|
| 73 | def _config_failure(repo: Path) -> str | None:
|
| 74 | expected_paths = {f"service-{i:04d}" for i in range(600)}
|
| 75 | expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
|
| 76 | actual = _tree_paths(repo / "services")
|
| 77 | if actual != expected_paths:
|
| 78 | return "config_path_set_mismatch"
|
| 79 | for i in range(600):
|
| 80 | rel = f"services/service-{i:04d}/config.toml"
|
| 81 | expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
|
| 82 | if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
|
| 83 | return "config_payload_mismatch:" + rel
|
| 84 | return None
|
| 85 |
|
| 86 |
|
| 87 | def _unique_object(pairs):
|
| 88 | result = {}
|
| 89 | for key, value in pairs:
|
| 90 | if key in result:
|
| 91 | raise ValueError("duplicate_json_key:" + key)
|
| 92 | result[key] = value
|
| 93 | return result
|
| 94 |
|
| 95 |
|
| 96 | @lru_cache(maxsize=4)
|
| 97 | def _expected_asset_hash(index: int) -> str:
|
| 98 | unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
|
| 99 | digest = hashlib.sha256()
|
| 100 | remaining = ASSET_BYTES
|
| 101 | while remaining:
|
| 102 | chunk = unit[:min(len(unit), remaining)]
|
| 103 | digest.update(chunk)
|
| 104 | remaining -= len(chunk)
|
| 105 | return digest.hexdigest()
|
| 106 |
|
| 107 |
|
| 108 | def _asset_failure(repo: Path) -> str | None:
|
| 109 | expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
|
| 110 | if _tree_paths(repo / "assets") != expected_paths:
|
| 111 | return "asset_path_set_mismatch"
|
| 112 | manifest = repo / "manifest.json"
|
| 113 | if not _regular_file(manifest):
|
| 114 | return "manifest_missing_or_wrong_type"
|
| 115 | if manifest.stat().st_size > 64 * 1024:
|
| 116 | return "manifest_size_out_of_bounds"
|
| 117 | data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
|
| 118 | if not isinstance(data, dict) or set(data) != {"assets"}:
|
| 119 | return "manifest_object_shape"
|
| 120 | entries = data["assets"]
|
| 121 | if not isinstance(entries, list) or len(entries) != 4:
|
| 122 | return "manifest_asset_count"
|
| 123 | seen = set()
|
| 124 | for item in entries:
|
| 125 | if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
|
| 126 | return "manifest_entry_shape"
|
| 127 | path = item["path"]
|
| 128 | if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
|
| 129 | return "manifest_asset_identity"
|
| 130 | if path in seen:
|
| 131 | return "manifest_duplicate_asset:" + path
|
| 132 | seen.add(path)
|
| 133 | if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
|
| 134 | return "manifest_asset_fields:" + path
|
| 135 | for index in range(4):
|
| 136 | path = repo / "assets" / f"asset-{index:02d}.bin"
|
| 137 | if path.stat().st_size != ASSET_BYTES:
|
| 138 | return "asset_size_mismatch:" + path.name
|
| 139 | digest = hashlib.sha256()
|
| 140 | with path.open("rb") as stream:
|
| 141 | for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
| 142 | digest.update(chunk)
|
| 143 | if digest.hexdigest() != _expected_asset_hash(index):
|
| 144 | return "asset_content_mismatch:" + path.name
|
| 145 | return None
|
| 146 |
|
| 147 |
|
| 148 | def validate(scenario: str, repo: Path) -> str | None:
|
| 149 | """Return a concrete failure reason, or None on a fully verified task."""
|
| 150 | if scenario not in SCENARIOS:
|
| 151 | return "unsupported_trusted_task_oracle:" + scenario
|
| 152 | if scenario == "history_archaeology":
|
| 153 | return "history_answer_oracle_unavailable"
|
| 154 | if scenario in COMMITTED_PAYLOADS:
|
| 155 | expected = dict(COMMITTED_PAYLOADS[scenario])
|
| 156 | if scenario == "sync_pull_dirty":
|
| 157 | expected["work.txt"] = "dirty local edit"
|
| 158 | try:
|
| 159 | for relative, marker in expected.items():
|
| 160 | path = repo / relative
|
| 161 | if (any((repo.joinpath(*Path(relative).parts[:index])).is_symlink() for index in range(1, len(Path(relative).parts) + 1))
|
| 162 | or not _regular_file(path) or path.stat().st_size > PAYLOAD_LIMIT_BYTES
|
| 163 | or marker not in path.read_text()):
|
| 164 | return "worktree_payload_missing:" + relative
|
| 165 | except (OSError, UnicodeError):
|
| 166 | return "worktree_payload_unreadable"
|
| 167 | return None
|
| 168 | name, expected_validator = (
|
| 169 | ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
|
| 170 | else ("validate_manifest.py", ASSET_VALIDATOR)
|
| 171 | )
|
| 172 | try:
|
| 173 | path = repo / name
|
| 174 | if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
|
| 175 | or path.read_bytes() != expected_validator):
|
| 176 | return "fixture_validator_modified_or_missing:" + name
|
| 177 | return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
|
| 178 | except (OSError, ValueError, UnicodeError, RecursionError) as exc:
|
| 179 | return "trusted_payload_unreadable_or_invalid:" + str(exc)
|
| 180 |
|
| 181 |
|
| 182 | def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
|
| 183 | """Resolve an explicitly configured instrument or the invoking PATH once."""
|
| 184 | requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
|
| 185 | resolved = shutil.which(requested)
|
| 186 | metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
|
| 187 | "git_head_instrument_version": None}
|
| 188 | if resolved is None:
|
| 189 | return None, metadata
|
| 190 | binary = str(Path(resolved).resolve())
|
| 191 | try:
|
| 192 | version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
|
| 193 | if version.returncode:
|
| 194 | return None, metadata
|
| 195 | metadata.update(git_head_instrument_binary=binary,
|
| 196 | git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
|
| 197 | git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
|
| 198 | return binary, metadata
|
| 199 | except (OSError, subprocess.TimeoutExpired):
|
| 200 | return None, metadata
|
| 201 |
|
| 202 |
|
| 203 | def git_head_text(repo: Path, path: str, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> tuple[str | None, str | None]:
|
| 204 | git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
|
| 205 | if git is None:
|
| 206 | return None, "git_head_oracle_unavailable"
|
| 207 | try:
|
| 208 | result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
|
| 209 | env=base_env(), capture_output=True, timeout=30)
|
| 210 | if result.returncode < 0:
|
| 211 | return None, "git_head_oracle_unavailable"
|
| 212 | return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
|
| 213 | except (OSError, subprocess.TimeoutExpired):
|
| 214 | return None, "git_head_oracle_unavailable"
|
| 215 |
|
| 216 |
|
| 217 | def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> str | None:
|
| 218 | """Verify the entire task subtree from committed objects using system Git.
|
| 219 |
|
| 220 | Never checks out subject-controlled hooks/filters or extracts an archive.
|
| 221 | Enumerated blob paths and bounded sizes are checked before materialization.
|
| 222 | Git object replacement and inherited Git environment overrides are disabled.
|
| 223 | """
|
| 224 | git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
|
| 225 | if git is None:
|
| 226 | return "git_head_oracle_unavailable"
|
| 227 | if scenario in COMMITTED_PAYLOADS:
|
| 228 | try:
|
| 229 | expected = COMMITTED_PAYLOADS[scenario]
|
| 230 | command = [git, "-C", str(repo.resolve())]
|
| 231 | # One tree read binds every required path to the same committed tree.
|
| 232 | tree = subprocess.run([*command, "ls-tree", "-r", "-z", "HEAD", "--", *expected],
|
| 233 | env=base_env(), capture_output=True, timeout=30)
|
| 234 | if tree.returncode < 0:
|
| 235 | return "git_head_oracle_unavailable"
|
| 236 | if tree.returncode:
|
| 237 | return "git_head_tree_unreadable"
|
| 238 | entries = {}
|
| 239 | for raw in tree.stdout.split(b"\0"):
|
| 240 | if not raw:
|
| 241 | continue
|
| 242 | header, relative = raw.split(b"\t", 1)
|
| 243 | mode, kind, oid = header.split()
|
| 244 | path = relative.decode("utf-8")
|
| 245 | if mode != b"100644" or kind != b"blob":
|
| 246 | return "committed_payload_path_type:" + path
|
| 247 | if path not in expected or path in entries:
|
| 248 | return "committed_payload_path_set_mismatch"
|
| 249 | if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
|
| 250 | return "git_head_invalid_object_id"
|
| 251 | entries[path] = oid.decode("ascii")
|
| 252 | if set(entries) != set(expected):
|
| 253 | return "committed_payload_path_set_mismatch"
|
| 254 | for path, marker in expected.items():
|
| 255 | oid = entries[path]
|
| 256 | size = subprocess.run([*command, "cat-file", "-s", oid],
|
| 257 | env=base_env(), capture_output=True, timeout=30)
|
| 258 | if size.returncode < 0:
|
| 259 | return "git_head_oracle_unavailable"
|
| 260 | if size.returncode or not size.stdout.strip().isdigit():
|
| 261 | return "committed_payload_missing:" + path
|
| 262 | length = int(size.stdout.strip())
|
| 263 | if length > PAYLOAD_LIMIT_BYTES:
|
| 264 | return "committed_payload_over_limit:" + path
|
| 265 | blob = subprocess.run([*command, "cat-file", "blob", oid],
|
| 266 | env=base_env(), capture_output=True, timeout=30)
|
| 267 | if blob.returncode < 0:
|
| 268 | return "git_head_oracle_unavailable"
|
| 269 | if blob.returncode or len(blob.stdout) != length:
|
| 270 | return "git_head_blob_unreadable"
|
| 271 | try:
|
| 272 | text = blob.stdout.decode("utf-8")
|
| 273 | except UnicodeError:
|
| 274 | return "committed_payload_invalid_utf8:" + path
|
| 275 | if marker not in text:
|
| 276 | return "committed_payload_missing:" + path
|
| 277 | return None
|
| 278 | except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired):
|
| 279 | return "git_head_oracle_unavailable"
|
| 280 | if scenario == "wide_config_refactor":
|
| 281 | paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
|
| 282 | paths.add("validate_configs.py")
|
| 283 | scope = ["services", "validate_configs.py"]
|
| 284 | max_blob_size = 4096
|
| 285 | elif scenario == "large_asset_manifest":
|
| 286 | paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
|
| 287 | paths.update({"manifest.json", "validate_manifest.py"})
|
| 288 | scope = ["assets", "manifest.json", "validate_manifest.py"]
|
| 289 | max_blob_size = ASSET_BYTES
|
| 290 | else:
|
| 291 | return "unsupported_trusted_task_oracle:" + scenario
|
| 292 | env = base_env()
|
| 293 | command = [git, "-C", str(repo.resolve())]
|
| 294 | try:
|
| 295 | tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
|
| 296 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
|
| 297 | if tree.returncode < 0:
|
| 298 | return "git_head_oracle_unavailable"
|
| 299 | if tree.returncode:
|
| 300 | return "git_head_tree_unreadable"
|
| 301 | entries = {}
|
| 302 | for raw in tree.stdout.split(b"\0"):
|
| 303 | if not raw:
|
| 304 | continue
|
| 305 | header, rel = raw.split(b"\t", 1)
|
| 306 | mode, kind, oid = header.split()
|
| 307 | path = rel.decode("utf-8")
|
| 308 | if mode != b"100644" or kind != b"blob":
|
| 309 | return "git_head_path_type:" + path
|
| 310 | if path not in paths or path in entries:
|
| 311 | return "git_head_path_set_mismatch"
|
| 312 | if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
|
| 313 | return "git_head_invalid_object_id"
|
| 314 | entries[path] = oid
|
| 315 | if set(entries) != paths:
|
| 316 | return "git_head_path_set_mismatch"
|
| 317 | ordered = sorted(entries)
|
| 318 | request = b"\n".join(entries[path] for path in ordered) + b"\n"
|
| 319 | sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
|
| 320 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
|
| 321 | if sizes.returncode < 0:
|
| 322 | return "git_head_oracle_unavailable"
|
| 323 | if sizes.returncode:
|
| 324 | return "git_head_blob_metadata_unreadable"
|
| 325 | headers = sizes.stdout.splitlines()
|
| 326 | if len(headers) != len(ordered):
|
| 327 | return "git_head_blob_metadata_incomplete"
|
| 328 | for header in headers:
|
| 329 | fields = header.split()
|
| 330 | if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
|
| 331 | return "git_head_blob_metadata_invalid"
|
| 332 | if int(fields[2]) > max_blob_size:
|
| 333 | return "git_head_blob_size_out_of_bounds"
|
| 334 | blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
|
| 335 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
|
| 336 | if blobs.returncode < 0:
|
| 337 | return "git_head_oracle_unavailable"
|
| 338 | if blobs.returncode:
|
| 339 | return "git_head_blob_unreadable"
|
| 340 | offset = 0
|
| 341 | with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
|
| 342 | root = Path(tmp)
|
| 343 | for path, expected_header in zip(ordered, headers):
|
| 344 | end = blobs.stdout.find(b"\n", offset)
|
| 345 | if end < 0 or blobs.stdout[offset:end] != expected_header:
|
| 346 | return "git_head_blob_framing_invalid"
|
| 347 | size = int(expected_header.split()[2])
|
| 348 | offset = end + 1
|
| 349 | payload = blobs.stdout[offset:offset + size]
|
| 350 | if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
|
| 351 | return "git_head_blob_framing_invalid"
|
| 352 | offset += size + 1
|
| 353 | destination = root / path
|
| 354 | destination.parent.mkdir(parents=True, exist_ok=True)
|
| 355 | destination.write_bytes(payload)
|
| 356 | if offset != len(blobs.stdout):
|
| 357 | return "git_head_blob_trailing_bytes"
|
| 358 | reason = validate(scenario, root)
|
| 359 | return "git_head_" + reason if reason else None
|
| 360 | except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
| 361 | return "git_head_oracle_error:" + str(exc)
|
| 362 |
|
| 363 |
|
| 364 | def reader_evidence(kind: str, binary: str | None, metadata: dict) -> dict:
|
| 365 | """Classify an executed reader by observed binary bytes, never its label."""
|
| 366 | subject_hash = metadata.get("workflow_subject_binary_sha256")
|
| 367 | if kind == "oak":
|
| 368 | subject_hash = subject_hash or metadata.get("oak_export_instrument_sha256")
|
| 369 | try:
|
| 370 | resolved = shutil.which(binary or kind) if subject_hash is None else None
|
| 371 | if resolved:
|
| 372 | if str(Path(resolved).resolve()) == metadata.get("git_head_instrument_binary"):
|
| 373 | subject_hash = metadata.get("git_head_instrument_sha256")
|
| 374 | else:
|
| 375 | subject_hash = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
|
| 376 | except OSError:
|
| 377 | pass
|
| 378 | instrument_hash = metadata.get("git_head_instrument_sha256")
|
| 379 | evidence_class = "unverified_committed_tree"
|
| 380 | if kind == "oak":
|
| 381 | evidence_class = "subject_produced_committed_tree"
|
| 382 | elif subject_hash and instrument_hash:
|
| 383 | evidence_class = "subject_binary_committed_tree" if subject_hash == instrument_hash else "independent_committed_tree"
|
| 384 | return {"workflow_subject_binary_sha256": subject_hash, "workflow_integrity_evidence_class": evidence_class}
|
| 385 |
|
| 386 |
|
| 387 | def independent_oak_evidence(scenario: str, repo: Path, python: str, evidence: dict) -> dict:
|
| 388 | """Run a separately configured read-only decoder; never invoke Oak here."""
|
| 389 | from oakbench.oak_sqlite_reader import READER_VERSION
|
| 390 | helper = Path(__file__).with_name("oak_sqlite_reader.py")
|
| 391 | binary = shutil.which(python)
|
| 392 | if binary is None:
|
| 393 | evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_python_unavailable"
|
| 394 | return evidence
|
| 395 | evidence["workflow_integrity_source"] += ";independent_oak_sqlite_read_attempt"
|
| 396 | evidence["workflow_integrity_evidence_class"] = "independent_committed_read_attempt"
|
| 397 | try:
|
| 398 | helper_hash = hashlib.sha256(helper.read_bytes()).hexdigest()
|
| 399 | evidence["oak_reader_python_sha256"] = hashlib.sha256(Path(binary).read_bytes()).hexdigest()
|
| 400 | with tempfile.TemporaryDirectory(prefix="oakbench-independent-read-") as temporary:
|
| 401 | tree = Path(temporary) / "tree"
|
| 402 | result = subprocess.run([binary, "-I", str(helper), str(repo), str(tree)],
|
| 403 | env=base_env(), capture_output=True, timeout=120)
|
| 404 | metadata = json.loads(result.stdout)
|
| 405 | if result.returncode == 77:
|
| 406 | evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_unavailable:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
|
| 407 | return evidence
|
| 408 | if result.returncode != 0:
|
| 409 | # A decoder failure is an instrument gap unless its contract
|
| 410 | # positively identifies invalid stored content.
|
| 411 | if result.returncode == 1 and metadata.get("corrupt") is True:
|
| 412 | evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason="oak_stored_content_invalid:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure")))))
|
| 413 | else:
|
| 414 | evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_failed:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
|
| 415 | return evidence
|
| 416 | if (metadata.get("reader_version") != READER_VERSION or metadata.get("reader_sha256") != helper_hash
|
| 417 | or not re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("head", "")))):
|
| 418 | raise ValueError("oak_reader_identity_mismatch")
|
| 419 | evidence.update({"oak_reader_version": READER_VERSION, "oak_reader_sha256": helper_hash,
|
| 420 | "oak_reader_head": metadata["head"], "oak_reader_blake3_version": metadata.get("blake3_version"),
|
| 421 | "oak_reader_zstandard_version": metadata.get("zstandard_version")})
|
| 422 | if scenario in EXACT_SCENARIOS:
|
| 423 | reason = validate(scenario, tree)
|
| 424 | else:
|
| 425 | reason = None
|
| 426 | for relative, marker in COMMITTED_PAYLOADS[scenario].items():
|
| 427 | path = tree / relative
|
| 428 | if not _regular_file(path) or path.stat().st_size > PAYLOAD_LIMIT_BYTES:
|
| 429 | reason = "committed_payload_missing:" + relative
|
| 430 | break
|
| 431 | try:
|
| 432 | text = path.read_text(encoding="utf-8")
|
| 433 | except UnicodeError:
|
| 434 | reason = "committed_payload_invalid_utf8:" + relative
|
| 435 | break
|
| 436 | if marker not in text:
|
| 437 | reason = "committed_payload_missing:" + relative
|
| 438 | break
|
| 439 | evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
|
| 440 | if reason is None:
|
| 441 | evidence["workflow_integrity_evidence_class"] = "independent_committed_tree" if scenario in EXACT_SCENARIOS else "independent_committed_payload"
|
| 442 | evidence["workflow_integrity_source"] = "trusted_task_worktree;independent_oak_sqlite_reader"
|
| 443 | except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
| 444 | evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_instrument_error:" + str(exc)
|
| 445 | return evidence
|
| 446 |
|
| 447 |
|
| 448 | def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None, oak_reader_python: str | None = None) -> dict:
|
| 449 | source = "trusted_exact_task_worktree"
|
| 450 | evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
|
| 451 | "workflow_integrity_evidence_class": "worktree_payload",
|
| 452 | "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
|
| 453 | "workflow_integrity_unmeasured_reason": None}
|
| 454 | reason = validate(scenario, repo)
|
| 455 | if reason == "history_answer_oracle_unavailable":
|
| 456 | evidence["workflow_integrity_evidence_class"] = None
|
| 457 | evidence["workflow_integrity_unmeasured_reason"] = reason
|
| 458 | return evidence
|
| 459 | if reason:
|
| 460 | evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
|
| 461 | return evidence
|
| 462 | configured_reader = oak_reader_python or os.environ.get("OAK_BENCH_OAK_READER_PYTHON")
|
| 463 | if kind == "oak" and configured_reader:
|
| 464 | return independent_oak_evidence(scenario, repo, configured_reader, evidence)
|
| 465 | if kind in {"git", "oak"}:
|
| 466 | git, metadata = resolve_git_instrument(git_instrument)
|
| 467 | evidence.update(metadata)
|
| 468 | if git is None:
|
| 469 | evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
|
| 470 | return evidence
|
| 471 | if kind == "git":
|
| 472 | evidence["workflow_integrity_evidence_class"] = "committed_tree_attempt"
|
| 473 | evidence["workflow_integrity_source"] = source + ";git_head_read_attempt"
|
| 474 | reason = validate_git_head(scenario, repo, resolved_instrument=(git, metadata))
|
| 475 | else:
|
| 476 | oak = shutil.which(binary or "oak")
|
| 477 | if oak is None:
|
| 478 | evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
|
| 479 | return evidence
|
| 480 | oak = str(Path(oak).resolve())
|
| 481 | evidence["oak_export_instrument_binary"] = oak
|
| 482 | try:
|
| 483 | evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
|
| 484 | status = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
|
| 485 | before = json.loads(status.stdout) if status.returncode == 0 else {}
|
| 486 | head, branch = before.get("head"), before.get("branch")
|
| 487 | if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
|
| 488 | raise ValueError("oak_head_identity_unreadable")
|
| 489 | evidence["oak_export_head"] = head
|
| 490 | with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
|
| 491 | destination = Path(tmp) / "export"
|
| 492 | evidence["workflow_integrity_evidence_class"] = "subject_produced_committed_tree_attempt"
|
| 493 | evidence["workflow_integrity_source"] = source + ";subject_produced_oak_head_export_attempt"
|
| 494 | exported = subprocess.run([oak, "export", "--branch", branch, str(destination)], cwd=repo,
|
| 495 | env=base_env(), capture_output=True, timeout=120)
|
| 496 | if exported.returncode:
|
| 497 | evidence["workflow_integrity_unmeasured_reason"] = "oak_head_export_unavailable"
|
| 498 | return evidence
|
| 499 | after = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
|
| 500 | state = json.loads(after.stdout) if after.returncode == 0 else {}
|
| 501 | if state.get("head") != head or state.get("branch") != branch:
|
| 502 | raise ValueError("oak_head_changed_during_export")
|
| 503 | reason = validate_git_head(scenario, destination, resolved_instrument=(git, metadata))
|
| 504 | except (OSError, ValueError, subprocess.TimeoutExpired) as exc:
|
| 505 | evidence["workflow_integrity_unmeasured_reason"] = str(exc)
|
| 506 | return evidence
|
| 507 | if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
|
| 508 | evidence["workflow_integrity_unmeasured_reason"] = reason
|
| 509 | return evidence
|
| 510 | if reason is None and kind in {"git", "oak"}:
|
| 511 | evidence.update(reader_evidence(kind, binary, evidence))
|
| 512 | if scenario not in EXACT_SCENARIOS:
|
| 513 | evidence["workflow_integrity_evidence_class"] = evidence["workflow_integrity_evidence_class"].replace("_tree", "_payload")
|
| 514 | suffix = "subject_produced_oak_head_export" if kind == "oak" else evidence["workflow_integrity_evidence_class"]
|
| 515 | evidence["workflow_integrity_source"] = source + ";" + suffix
|
| 516 | evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
|
| 517 | return evidence
|
| 518 |
|
| 519 |
|
| 520 | def main() -> int:
|
| 521 | import argparse
|
| 522 | parser = argparse.ArgumentParser()
|
| 523 | parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
|
| 524 | parser.add_argument("--git-head", action="store_true")
|
| 525 | parser.add_argument("--oak-head", action="store_true")
|
| 526 | parser.add_argument("--binary")
|
| 527 | parser.add_argument("--evidence-json", action="store_true")
|
| 528 | parser.add_argument("--git-instrument")
|
| 529 | parser.add_argument("--oak-reader-python")
|
| 530 | args = parser.parse_args()
|
| 531 | evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument, args.oak_reader_python)
|
| 532 | reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
|
| 533 | print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
|
| 534 | passed = evidence["workflow_integrity_passed"]
|
| 535 | return 77 if passed is None else 0 if passed else 1
|
| 536 |
|
| 537 |
|
| 538 | if __name__ == "__main__":
|
| 539 | raise SystemExit(main())
|