| 1 | """Trusted exact task payload checks, outside the agent-editable checkout.
|
| 2 |
|
| 3 | This module is stdlib-only and can run with Python -I as an external oracle.
|
| 4 | It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
|
| 5 | 600 canonical service configs and four 16 MiB deterministic binary assets.
|
| 6 | """
|
| 7 | from __future__ import annotations
|
| 8 |
|
| 9 | from functools import lru_cache
|
| 10 | import hashlib
|
| 11 | import json
|
| 12 | import os
|
| 13 | from pathlib import Path
|
| 14 | import sys
|
| 15 | import shutil
|
| 16 | import subprocess
|
| 17 | import tempfile
|
| 18 | import re
|
| 19 |
|
| 20 | if __package__ in (None, ""):
|
| 21 | sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 22 | from oakbench.environment import base_env
|
| 23 |
|
| 24 | VERSION = "2026-09-27.1"
|
| 25 | EXACT_SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
|
| 26 | COMMITTED_PAYLOADS = {
|
| 27 | "bugfix_test_loop": {"app/pricing.py": "return cents - ((cents * percent) // 100)"},
|
| 28 | "vcs_error_recovery": {"README.md": "# recovered"},
|
| 29 | "sync_push_divergence": {"work.txt": "local change", "notes.txt": "upstream change"},
|
| 30 | "sync_pull_upstream": {"notes.txt": "upstream change"},
|
| 31 | "sync_pull_dirty": {"notes.txt": "upstream change"},
|
| 32 | "sync_non_ff_after_amend": {"work.txt": "amended line"},
|
| 33 | }
|
| 34 | SCENARIOS = EXACT_SCENARIOS | frozenset(COMMITTED_PAYLOADS) | {"history_archaeology"}
|
| 35 | PAYLOAD_LIMIT_BYTES = 1024 * 1024
|
| 36 | ASSET_BYTES = 16 * 1024 * 1024
|
| 37 |
|
| 38 | # Preserved fixture instructions, not programs executed by this oracle.
|
| 39 | CONFIG_VALIDATOR = "\n".join([
|
| 40 | "from pathlib import Path", "bad = []",
|
| 41 | "for p in Path('services').glob('*/config.toml'):",
|
| 42 | " text = p.read_text()", " if 'timeout_ms = 2500' in text:",
|
| 43 | " bad.append(str(p))", "print(f'bad={len(bad)}')",
|
| 44 | "raise SystemExit(1 if bad else 0)", "",
|
| 45 | ]).encode()
|
| 46 | ASSET_VALIDATOR = "\n".join([
|
| 47 | "import json", "from pathlib import Path",
|
| 48 | "data = json.loads(Path('manifest.json').read_text())",
|
| 49 | "for item in data['assets']:", " p = Path(item['path'])",
|
| 50 | " assert p.exists(), p", " assert p.stat().st_size == item['bytes'], p",
|
| 51 | " assert item['reviewed'] is True, p", "print('manifest ok')", "",
|
| 52 | ]).encode()
|
| 53 |
|
| 54 |
|
| 55 | EXACT_SCOPES = {
|
| 56 | "wide_config_refactor": ("services", "validate_configs.py"),
|
| 57 | "large_asset_manifest": ("assets", "manifest.json", "validate_manifest.py"),
|
| 58 | }
|
| 59 |
|
| 60 |
|
| 61 | def committed_blob_limit(scenario: str) -> int:
|
| 62 | """Largest committed blob the Git check accepts for a scenario."""
|
| 63 | return {"wide_config_refactor": 4096, "large_asset_manifest": ASSET_BYTES}.get(scenario, PAYLOAD_LIMIT_BYTES)
|
| 64 |
|
| 65 |
|
| 66 | def committed_scope(scenario: str) -> tuple[str, ...] | None:
|
| 67 | """The committed paths every reader checks for a scenario, Git or Oak.
|
| 68 |
|
| 69 | One definition, so the Git ls-tree check and the independent Oak reader
|
| 70 | read the SAME scope at the same strength and may share an evidence class.
|
| 71 | """
|
| 72 | if scenario in COMMITTED_PAYLOADS:
|
| 73 | return tuple(sorted(COMMITTED_PAYLOADS[scenario]))
|
| 74 | return EXACT_SCOPES.get(scenario)
|
| 75 |
|
| 76 |
|
| 77 | class _InstrumentUnavailable(Exception):
|
| 78 | """The instrument could not run to completion (signal); never a subject verdict."""
|
| 79 |
|
| 80 |
|
| 81 | def _regular_file(path: Path) -> bool:
|
| 82 | return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
|
| 83 |
|
| 84 |
|
| 85 | def _tree_paths(root: Path) -> set[str]:
|
| 86 | """Return every descendant identity, rejecting links and special files."""
|
| 87 | if root.is_symlink() or not root.is_dir():
|
| 88 | raise ValueError("payload_directory_missing_or_link")
|
| 89 | result = set()
|
| 90 | for parent, dirs, files in os.walk(root, followlinks=False):
|
| 91 | for name in dirs + files:
|
| 92 | path = Path(parent) / name
|
| 93 | if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
|
| 94 | raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
|
| 95 | result.add(path.relative_to(root).as_posix())
|
| 96 | return result
|
| 97 |
|
| 98 |
|
| 99 | def _config_failure(repo: Path) -> str | None:
|
| 100 | expected_paths = {f"service-{i:04d}" for i in range(600)}
|
| 101 | expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
|
| 102 | actual = _tree_paths(repo / "services")
|
| 103 | if actual != expected_paths:
|
| 104 | return "config_path_set_mismatch"
|
| 105 | for i in range(600):
|
| 106 | rel = f"services/service-{i:04d}/config.toml"
|
| 107 | expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
|
| 108 | if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
|
| 109 | return "config_payload_mismatch:" + rel
|
| 110 | return None
|
| 111 |
|
| 112 |
|
| 113 | def _unique_object(pairs):
|
| 114 | result = {}
|
| 115 | for key, value in pairs:
|
| 116 | if key in result:
|
| 117 | raise ValueError("duplicate_json_key:" + key)
|
| 118 | result[key] = value
|
| 119 | return result
|
| 120 |
|
| 121 |
|
| 122 | @lru_cache(maxsize=4)
|
| 123 | def _expected_asset_hash(index: int) -> str:
|
| 124 | unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
|
| 125 | digest = hashlib.sha256()
|
| 126 | remaining = ASSET_BYTES
|
| 127 | while remaining:
|
| 128 | chunk = unit[:min(len(unit), remaining)]
|
| 129 | digest.update(chunk)
|
| 130 | remaining -= len(chunk)
|
| 131 | return digest.hexdigest()
|
| 132 |
|
| 133 |
|
| 134 | def _asset_failure(repo: Path) -> str | None:
|
| 135 | expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
|
| 136 | if _tree_paths(repo / "assets") != expected_paths:
|
| 137 | return "asset_path_set_mismatch"
|
| 138 | manifest = repo / "manifest.json"
|
| 139 | if not _regular_file(manifest):
|
| 140 | return "manifest_missing_or_wrong_type"
|
| 141 | if manifest.stat().st_size > 64 * 1024:
|
| 142 | return "manifest_size_out_of_bounds"
|
| 143 | data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
|
| 144 | if not isinstance(data, dict) or set(data) != {"assets"}:
|
| 145 | return "manifest_object_shape"
|
| 146 | entries = data["assets"]
|
| 147 | if not isinstance(entries, list) or len(entries) != 4:
|
| 148 | return "manifest_asset_count"
|
| 149 | seen = set()
|
| 150 | for item in entries:
|
| 151 | if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
|
| 152 | return "manifest_entry_shape"
|
| 153 | path = item["path"]
|
| 154 | if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
|
| 155 | return "manifest_asset_identity"
|
| 156 | if path in seen:
|
| 157 | return "manifest_duplicate_asset:" + path
|
| 158 | seen.add(path)
|
| 159 | if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
|
| 160 | return "manifest_asset_fields:" + path
|
| 161 | for index in range(4):
|
| 162 | path = repo / "assets" / f"asset-{index:02d}.bin"
|
| 163 | if path.stat().st_size != ASSET_BYTES:
|
| 164 | return "asset_size_mismatch:" + path.name
|
| 165 | digest = hashlib.sha256()
|
| 166 | with path.open("rb") as stream:
|
| 167 | for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
| 168 | digest.update(chunk)
|
| 169 | if digest.hexdigest() != _expected_asset_hash(index):
|
| 170 | return "asset_content_mismatch:" + path.name
|
| 171 | return None
|
| 172 |
|
| 173 |
|
| 174 | def validate(scenario: str, repo: Path) -> str | None:
|
| 175 | """Return a concrete failure reason, or None on a fully verified task."""
|
| 176 | if scenario not in SCENARIOS:
|
| 177 | return "unsupported_trusted_task_oracle:" + scenario
|
| 178 | if scenario == "history_archaeology":
|
| 179 | return "history_answer_oracle_unavailable"
|
| 180 | if scenario in COMMITTED_PAYLOADS:
|
| 181 | expected = dict(COMMITTED_PAYLOADS[scenario])
|
| 182 | if scenario == "sync_pull_dirty":
|
| 183 | expected["work.txt"] = "dirty local edit"
|
| 184 | try:
|
| 185 | for relative, marker in expected.items():
|
| 186 | path = repo / relative
|
| 187 | if (any((repo.joinpath(*Path(relative).parts[:index])).is_symlink() for index in range(1, len(Path(relative).parts) + 1))
|
| 188 | or not _regular_file(path) or path.stat().st_size > PAYLOAD_LIMIT_BYTES
|
| 189 | or marker not in path.read_text()):
|
| 190 | return "worktree_payload_missing:" + relative
|
| 191 | except (OSError, UnicodeError):
|
| 192 | return "worktree_payload_unreadable"
|
| 193 | return None
|
| 194 | name, expected_validator = (
|
| 195 | ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
|
| 196 | else ("validate_manifest.py", ASSET_VALIDATOR)
|
| 197 | )
|
| 198 | try:
|
| 199 | path = repo / name
|
| 200 | if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
|
| 201 | or path.read_bytes() != expected_validator):
|
| 202 | return "fixture_validator_modified_or_missing:" + name
|
| 203 | return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
|
| 204 | except (OSError, ValueError, UnicodeError, RecursionError) as exc:
|
| 205 | return "trusted_payload_unreadable_or_invalid:" + str(exc)
|
| 206 |
|
| 207 |
|
| 208 | def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
|
| 209 | """Resolve an explicitly configured instrument or the invoking PATH once."""
|
| 210 | requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
|
| 211 | resolved = shutil.which(requested)
|
| 212 | # The xcrun trampoline execs the real Git; identify the binary that runs
|
| 213 | # so evidence classes compare the same bytes the Git subject executes.
|
| 214 | from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
|
| 215 | if resolved and is_xcrun_git_shim(resolved):
|
| 216 | real = resolve_real_git()
|
| 217 | resolved = str(real) if real else resolved
|
| 218 | metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
|
| 219 | "git_head_instrument_version": None}
|
| 220 | if resolved is None:
|
| 221 | return None, metadata
|
| 222 | binary = str(Path(resolved).resolve())
|
| 223 | try:
|
| 224 | version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
|
| 225 | if version.returncode:
|
| 226 | return None, metadata
|
| 227 | metadata.update(git_head_instrument_binary=binary,
|
| 228 | git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
|
| 229 | git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
|
| 230 | return binary, metadata
|
| 231 | except (OSError, subprocess.TimeoutExpired):
|
| 232 | return None, metadata
|
| 233 |
|
| 234 |
|
| 235 | def git_head_text(repo: Path, path: str, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> tuple[str | None, str | None]:
|
| 236 | git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
|
| 237 | if git is None:
|
| 238 | return None, "git_head_oracle_unavailable"
|
| 239 | try:
|
| 240 | result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
|
| 241 | env=base_env(), capture_output=True, timeout=30)
|
| 242 | if result.returncode < 0:
|
| 243 | return None, "git_head_oracle_unavailable"
|
| 244 | return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
|
| 245 | except (OSError, subprocess.TimeoutExpired):
|
| 246 | return None, "git_head_oracle_unavailable"
|
| 247 |
|
| 248 |
|
| 249 | def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None, *, resolved_instrument: tuple[str | None, dict] | None = None) -> str | None:
|
| 250 | """Verify the entire task subtree from committed objects using system Git.
|
| 251 |
|
| 252 | Never checks out subject-controlled hooks/filters or extracts an archive.
|
| 253 | Enumerated blob paths and bounded sizes are checked before materialization.
|
| 254 | Git object replacement and inherited Git environment overrides are disabled.
|
| 255 | """
|
| 256 | git, _ = resolved_instrument if resolved_instrument is not None else resolve_git_instrument(git_binary)
|
| 257 | if git is None:
|
| 258 | return "git_head_oracle_unavailable"
|
| 259 | if scenario in COMMITTED_PAYLOADS:
|
| 260 | try:
|
| 261 | expected = COMMITTED_PAYLOADS[scenario]
|
| 262 | command = [git, "-C", str(repo.resolve())]
|
| 263 | # One tree read binds every required path to the same committed tree.
|
| 264 | tree = subprocess.run([*command, "ls-tree", "-r", "-z", "HEAD", "--", *committed_scope(scenario)],
|
| 265 | env=base_env(), capture_output=True, timeout=30)
|
| 266 | if tree.returncode < 0:
|
| 267 | return "git_head_oracle_unavailable"
|
| 268 | if tree.returncode:
|
| 269 | return "git_head_tree_unreadable"
|
| 270 | entries = {}
|
| 271 | for raw in tree.stdout.split(b"\0"):
|
| 272 | if not raw:
|
| 273 | continue
|
| 274 | header, relative = raw.split(b"\t", 1)
|
| 275 | mode, kind, oid = header.split()
|
| 276 | path = relative.decode("utf-8")
|
| 277 | if mode != b"100644" or kind != b"blob":
|
| 278 | return "committed_payload_path_type:" + path
|
| 279 | if path not in expected or path in entries:
|
| 280 | return "committed_payload_path_set_mismatch"
|
| 281 | if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
|
| 282 | return "git_head_invalid_object_id"
|
| 283 | entries[path] = oid.decode("ascii")
|
| 284 | if set(entries) != set(expected):
|
| 285 | return "committed_payload_path_set_mismatch"
|
| 286 | for path, marker in expected.items():
|
| 287 | oid = entries[path]
|
| 288 | size = subprocess.run([*command, "cat-file", "-s", oid],
|
| 289 | env=base_env(), capture_output=True, timeout=30)
|
| 290 | if size.returncode < 0:
|
| 291 | return "git_head_oracle_unavailable"
|
| 292 | if size.returncode or not size.stdout.strip().isdigit():
|
| 293 | return "committed_payload_missing:" + path
|
| 294 | length = int(size.stdout.strip())
|
| 295 | if length > PAYLOAD_LIMIT_BYTES:
|
| 296 | return "committed_payload_over_limit:" + path
|
| 297 | blob = subprocess.run([*command, "cat-file", "blob", oid],
|
| 298 | env=base_env(), capture_output=True, timeout=30)
|
| 299 | if blob.returncode < 0:
|
| 300 | return "git_head_oracle_unavailable"
|
| 301 | if blob.returncode or len(blob.stdout) != length:
|
| 302 | return "git_head_blob_unreadable"
|
| 303 | try:
|
| 304 | text = blob.stdout.decode("utf-8")
|
| 305 | except UnicodeError:
|
| 306 | return "committed_payload_invalid_utf8:" + path
|
| 307 | if marker not in text:
|
| 308 | return "committed_payload_missing:" + path
|
| 309 | return None
|
| 310 | except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired):
|
| 311 | return "git_head_oracle_unavailable"
|
| 312 | if scenario == "wide_config_refactor":
|
| 313 | paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
|
| 314 | paths.add("validate_configs.py")
|
| 315 | scope = list(committed_scope(scenario))
|
| 316 | max_blob_size = 4096
|
| 317 | elif scenario == "large_asset_manifest":
|
| 318 | paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
|
| 319 | paths.update({"manifest.json", "validate_manifest.py"})
|
| 320 | scope = list(committed_scope(scenario))
|
| 321 | max_blob_size = ASSET_BYTES
|
| 322 | else:
|
| 323 | return "unsupported_trusted_task_oracle:" + scenario
|
| 324 | env = base_env()
|
| 325 | command = [git, "-C", str(repo.resolve())]
|
| 326 | try:
|
| 327 | tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
|
| 328 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
|
| 329 | if tree.returncode < 0:
|
| 330 | return "git_head_oracle_unavailable"
|
| 331 | if tree.returncode:
|
| 332 | return "git_head_tree_unreadable"
|
| 333 | entries = {}
|
| 334 | for raw in tree.stdout.split(b"\0"):
|
| 335 | if not raw:
|
| 336 | continue
|
| 337 | header, rel = raw.split(b"\t", 1)
|
| 338 | mode, kind, oid = header.split()
|
| 339 | path = rel.decode("utf-8")
|
| 340 | if mode != b"100644" or kind != b"blob":
|
| 341 | return "git_head_path_type:" + path
|
| 342 | if path not in paths or path in entries:
|
| 343 | return "git_head_path_set_mismatch"
|
| 344 | if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
|
| 345 | return "git_head_invalid_object_id"
|
| 346 | entries[path] = oid
|
| 347 | if set(entries) != paths:
|
| 348 | return "git_head_path_set_mismatch"
|
| 349 | ordered = sorted(entries)
|
| 350 | request = b"\n".join(entries[path] for path in ordered) + b"\n"
|
| 351 | sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
|
| 352 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
|
| 353 | if sizes.returncode < 0:
|
| 354 | return "git_head_oracle_unavailable"
|
| 355 | if sizes.returncode:
|
| 356 | return "git_head_blob_metadata_unreadable"
|
| 357 | headers = sizes.stdout.splitlines()
|
| 358 | if len(headers) != len(ordered):
|
| 359 | return "git_head_blob_metadata_incomplete"
|
| 360 | for header in headers:
|
| 361 | fields = header.split()
|
| 362 | if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
|
| 363 | return "git_head_blob_metadata_invalid"
|
| 364 | if int(fields[2]) > max_blob_size:
|
| 365 | return "git_head_blob_size_out_of_bounds"
|
| 366 | blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
|
| 367 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
|
| 368 | if blobs.returncode < 0:
|
| 369 | return "git_head_oracle_unavailable"
|
| 370 | if blobs.returncode:
|
| 371 | return "git_head_blob_unreadable"
|
| 372 | offset = 0
|
| 373 | with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
|
| 374 | root = Path(tmp)
|
| 375 | for path, expected_header in zip(ordered, headers):
|
| 376 | end = blobs.stdout.find(b"\n", offset)
|
| 377 | if end < 0 or blobs.stdout[offset:end] != expected_header:
|
| 378 | return "git_head_blob_framing_invalid"
|
| 379 | size = int(expected_header.split()[2])
|
| 380 | offset = end + 1
|
| 381 | payload = blobs.stdout[offset:offset + size]
|
| 382 | if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
|
| 383 | return "git_head_blob_framing_invalid"
|
| 384 | offset += size + 1
|
| 385 | destination = root / path
|
| 386 | destination.parent.mkdir(parents=True, exist_ok=True)
|
| 387 | destination.write_bytes(payload)
|
| 388 | if offset != len(blobs.stdout):
|
| 389 | return "git_head_blob_trailing_bytes"
|
| 390 | reason = validate(scenario, root)
|
| 391 | return "git_head_" + reason if reason else None
|
| 392 | except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
| 393 | return "git_head_oracle_error:" + str(exc)
|
| 394 |
|
| 395 |
|
| 396 | def reader_evidence(kind: str, binary: str | None, metadata: dict) -> dict:
|
| 397 | """Classify an executed reader by observed binary bytes, never its label."""
|
| 398 | subject_hash = metadata.get("workflow_subject_binary_sha256")
|
| 399 | if kind == "oak":
|
| 400 | subject_hash = subject_hash or metadata.get("oak_export_instrument_sha256")
|
| 401 | try:
|
| 402 | resolved = shutil.which(binary or kind) if subject_hash is None else None
|
| 403 | if resolved and kind == "git":
|
| 404 | # The xcrun shim execs the real Git: hash the binary that runs,
|
| 405 | # exactly as resolve_git_instrument does for the instrument.
|
| 406 | from oakbench.subjects import is_xcrun_git_shim, resolve_real_git
|
| 407 | if is_xcrun_git_shim(resolved):
|
| 408 | real = resolve_real_git()
|
| 409 | resolved = str(real) if real else resolved
|
| 410 | if resolved:
|
| 411 | if str(Path(resolved).resolve()) == metadata.get("git_head_instrument_binary"):
|
| 412 | subject_hash = metadata.get("git_head_instrument_sha256")
|
| 413 | else:
|
| 414 | subject_hash = hashlib.sha256(Path(resolved).read_bytes()).hexdigest()
|
| 415 | except OSError:
|
| 416 | pass
|
| 417 | instrument_hash = metadata.get("git_head_instrument_sha256")
|
| 418 | evidence_class = "unverified_committed_tree"
|
| 419 | if kind == "oak":
|
| 420 | evidence_class = "subject_produced_committed_tree"
|
| 421 | elif subject_hash and instrument_hash:
|
| 422 | evidence_class = "subject_binary_committed_tree" if subject_hash == instrument_hash else "independent_committed_tree"
|
| 423 | return {"workflow_subject_binary_sha256": subject_hash, "workflow_integrity_evidence_class": evidence_class}
|
| 424 |
|
| 425 |
|
| 426 | def independent_oak_evidence(scenario: str, repo: Path, python: str, evidence: dict) -> dict:
|
| 427 | """Run a separately configured read-only decoder; never invoke Oak here."""
|
| 428 | from oakbench.oak_sqlite_reader import READER_VERSION
|
| 429 | helper = Path(__file__).with_name("oak_sqlite_reader.py")
|
| 430 | binary = shutil.which(python)
|
| 431 | if binary is None:
|
| 432 | evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_python_unavailable"
|
| 433 | return evidence
|
| 434 | evidence["workflow_integrity_source"] += ";independent_oak_sqlite_read_attempt"
|
| 435 | evidence["workflow_integrity_evidence_class"] = "independent_committed_read_attempt"
|
| 436 | try:
|
| 437 | helper_hash = hashlib.sha256(helper.read_bytes()).hexdigest()
|
| 438 | evidence["oak_reader_python_sha256"] = hashlib.sha256(Path(binary).read_bytes()).hexdigest()
|
| 439 | with tempfile.TemporaryDirectory(prefix="oakbench-independent-read-") as temporary:
|
| 440 | tree = Path(temporary) / "tree"
|
| 441 | scope = committed_scope(scenario)
|
| 442 | evidence["workflow_integrity_committed_scope"] = list(scope)
|
| 443 | result = subprocess.run([binary, "-I", str(helper), str(repo), str(tree),
|
| 444 | "--max-blob-bytes", str(committed_blob_limit(scenario)), "--scope", *scope],
|
| 445 | env=base_env(), capture_output=True, timeout=120)
|
| 446 | metadata = json.loads(result.stdout)
|
| 447 | if result.returncode == 77:
|
| 448 | evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_unavailable:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
|
| 449 | return evidence
|
| 450 | if result.returncode != 0:
|
| 451 | # A decoder failure is an instrument gap unless its contract
|
| 452 | # positively identifies invalid stored content.
|
| 453 | if result.returncode == 1 and metadata.get("corrupt") is True:
|
| 454 | evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason="oak_stored_content_invalid:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure")))))
|
| 455 | else:
|
| 456 | evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_failed:" + str(metadata.get("reason", metadata.get("unavailable", metadata.get("failure"))))
|
| 457 | return evidence
|
| 458 | if (metadata.get("reader_version") != READER_VERSION or metadata.get("reader_sha256") != helper_hash
|
| 459 | or not re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("head", "")))
|
| 460 | or metadata.get("scope") != list(scope)):
|
| 461 | raise ValueError("oak_reader_identity_mismatch")
|
| 462 | evidence.update({"oak_reader_version": READER_VERSION, "oak_reader_sha256": helper_hash,
|
| 463 | "oak_reader_head": metadata["head"], "oak_reader_blake3_version": metadata.get("blake3_version"),
|
| 464 | "oak_reader_zstandard_version": metadata.get("zstandard_version")})
|
| 465 | nonregular = metadata.get("nonregular_paths") or []
|
| 466 | oversized = metadata.get("oversized_paths") or []
|
| 467 | collisions = metadata.get("collision_paths") or []
|
| 468 | exact = scenario in EXACT_SCENARIOS
|
| 469 | if nonregular:
|
| 470 | # Same outcome as Git's scoped mode check (mode != 100644).
|
| 471 | reason = ("git_head_path_type:" if exact else "committed_payload_path_type:") + str(nonregular[0])
|
| 472 | elif collisions:
|
| 473 | reason = "git_head_path_set_mismatch" if exact else "committed_payload_path_set_mismatch"
|
| 474 | elif oversized:
|
| 475 | reason = "git_head_blob_size_out_of_bounds" if exact else "committed_payload_over_limit:" + str(oversized[0])
|
| 476 | elif scenario in EXACT_SCENARIOS:
|
| 477 | reason = validate(scenario, tree)
|
| 478 | else:
|
| 479 | reason = None
|
| 480 | for relative, marker in COMMITTED_PAYLOADS[scenario].items():
|
| 481 | path = tree / relative
|
| 482 | # Reason strings mirror validate_git_head's so the two
|
| 483 | # readers report the same failure for the same content.
|
| 484 | if not path.exists() and not path.is_symlink():
|
| 485 | reason = "committed_payload_path_set_mismatch"
|
| 486 | break
|
| 487 | if not _regular_file(path):
|
| 488 | reason = "committed_payload_path_type:" + relative
|
| 489 | break
|
| 490 | if path.stat().st_size > PAYLOAD_LIMIT_BYTES:
|
| 491 | reason = "committed_payload_over_limit:" + relative
|
| 492 | break
|
| 493 | try:
|
| 494 | text = path.read_text(encoding="utf-8")
|
| 495 | except UnicodeError:
|
| 496 | reason = "committed_payload_invalid_utf8:" + relative
|
| 497 | break
|
| 498 | if marker not in text:
|
| 499 | reason = "committed_payload_missing:" + relative
|
| 500 | break
|
| 501 | evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
|
| 502 | if reason is None:
|
| 503 | evidence["workflow_integrity_evidence_class"] = "independent_committed_tree" if scenario in EXACT_SCENARIOS else "independent_committed_payload"
|
| 504 | evidence["workflow_integrity_source"] = "trusted_task_worktree;independent_oak_sqlite_reader"
|
| 505 | except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
| 506 | evidence["workflow_integrity_unmeasured_reason"] = "oak_reader_instrument_error:" + str(exc)
|
| 507 | return evidence
|
| 508 |
|
| 509 |
|
| 510 | def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None, oak_reader_python: str | None = None) -> dict:
|
| 511 | source = "trusted_exact_task_worktree"
|
| 512 | evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
|
| 513 | "workflow_integrity_evidence_class": "worktree_payload",
|
| 514 | "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
|
| 515 | "workflow_integrity_unmeasured_reason": None}
|
| 516 | reason = validate(scenario, repo)
|
| 517 | if reason == "history_answer_oracle_unavailable":
|
| 518 | evidence["workflow_integrity_evidence_class"] = None
|
| 519 | evidence["workflow_integrity_unmeasured_reason"] = reason
|
| 520 | return evidence
|
| 521 | if reason:
|
| 522 | evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
|
| 523 | return evidence
|
| 524 | configured_reader = oak_reader_python or os.environ.get("OAK_BENCH_OAK_READER_PYTHON")
|
| 525 | if kind == "oak" and configured_reader:
|
| 526 | return independent_oak_evidence(scenario, repo, configured_reader, evidence)
|
| 527 | if kind in {"git", "oak"}:
|
| 528 | git, metadata = resolve_git_instrument(git_instrument)
|
| 529 | evidence.update(metadata)
|
| 530 | if git is None:
|
| 531 | evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
|
| 532 | return evidence
|
| 533 | evidence["workflow_integrity_committed_scope"] = list(committed_scope(scenario) or ())
|
| 534 | if kind == "git":
|
| 535 | evidence["workflow_integrity_evidence_class"] = "committed_tree_attempt"
|
| 536 | evidence["workflow_integrity_source"] = source + ";git_head_read_attempt"
|
| 537 | reason = validate_git_head(scenario, repo, resolved_instrument=(git, metadata))
|
| 538 | else:
|
| 539 | oak = shutil.which(binary or "oak")
|
| 540 | if oak is None:
|
| 541 | evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
|
| 542 | return evidence
|
| 543 | oak = str(Path(oak).resolve())
|
| 544 | evidence["oak_export_instrument_binary"] = oak
|
| 545 | # Same classification as the Git arm (validate_git_head): a
|
| 546 | # positive nonzero exit or unusable output from the subject, or
|
| 547 | # subject-controlled repository state, is a MEASURED failure; only
|
| 548 | # an instrument that cannot run (launch error, signal, timeout) is
|
| 549 | # unmeasured. A broken `oak status`/`oak export` must never turn an
|
| 550 | # Oak failure into a skip.
|
| 551 | reason = None
|
| 552 | try:
|
| 553 | evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
|
| 554 |
|
| 555 | def run(command, timeout):
|
| 556 | result = subprocess.run(command, cwd=repo, env=base_env(), capture_output=True, timeout=timeout)
|
| 557 | if result.returncode < 0:
|
| 558 | raise _InstrumentUnavailable("oak_instrument_signalled")
|
| 559 | return result
|
| 560 |
|
| 561 | def identity(result):
|
| 562 | if result.returncode:
|
| 563 | return None, None
|
| 564 | try:
|
| 565 | data = json.loads(result.stdout)
|
| 566 | except ValueError:
|
| 567 | return None, None
|
| 568 | if not isinstance(data, dict):
|
| 569 | return None, None
|
| 570 | return data.get("head"), data.get("branch")
|
| 571 |
|
| 572 | head, branch = identity(run([oak, "status", "--json"], 30))
|
| 573 | if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
|
| 574 | reason = "oak_head_identity_unreadable"
|
| 575 | else:
|
| 576 | evidence["oak_export_head"] = head
|
| 577 | with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
|
| 578 | destination = Path(tmp) / "export"
|
| 579 | evidence["workflow_integrity_evidence_class"] = "subject_produced_committed_tree_attempt"
|
| 580 | evidence["workflow_integrity_source"] = source + ";subject_produced_oak_head_export_attempt"
|
| 581 | exported = run([oak, "export", "--branch", branch, str(destination)], 120)
|
| 582 | if exported.returncode:
|
| 583 | reason = "oak_head_export_failed"
|
| 584 | elif identity(run([oak, "status", "--json"], 30)) != (head, branch):
|
| 585 | reason = "oak_head_changed_during_export"
|
| 586 | else:
|
| 587 | reason = validate_git_head(scenario, destination, resolved_instrument=(git, metadata))
|
| 588 | except (OSError, subprocess.TimeoutExpired, _InstrumentUnavailable) as exc:
|
| 589 | evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_error:" + str(exc)
|
| 590 | return evidence
|
| 591 | if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
|
| 592 | evidence["workflow_integrity_unmeasured_reason"] = reason
|
| 593 | return evidence
|
| 594 | if reason is None and kind in {"git", "oak"}:
|
| 595 | evidence.update(reader_evidence(kind, binary, evidence))
|
| 596 | if scenario not in EXACT_SCENARIOS:
|
| 597 | evidence["workflow_integrity_evidence_class"] = evidence["workflow_integrity_evidence_class"].replace("_tree", "_payload")
|
| 598 | suffix = "subject_produced_oak_head_export" if kind == "oak" else evidence["workflow_integrity_evidence_class"]
|
| 599 | evidence["workflow_integrity_source"] = source + ";" + suffix
|
| 600 | evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
|
| 601 | return evidence
|
| 602 |
|
| 603 |
|
| 604 | def main() -> int:
|
| 605 | import argparse
|
| 606 | parser = argparse.ArgumentParser()
|
| 607 | parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
|
| 608 | parser.add_argument("--git-head", action="store_true")
|
| 609 | parser.add_argument("--oak-head", action="store_true")
|
| 610 | parser.add_argument("--binary")
|
| 611 | parser.add_argument("--evidence-json", action="store_true")
|
| 612 | parser.add_argument("--git-instrument")
|
| 613 | parser.add_argument("--oak-reader-python")
|
| 614 | args = parser.parse_args()
|
| 615 | evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument, args.oak_reader_python)
|
| 616 | reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
|
| 617 | print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
|
| 618 | passed = evidence["workflow_integrity_passed"]
|
| 619 | return 77 if passed is None else 0 if passed else 1
|
| 620 |
|
| 621 |
|
| 622 | if __name__ == "__main__":
|
| 623 | raise SystemExit(main())
|