| 1 | """Trusted exact task payload checks, outside the agent-editable checkout.
|
| 2 |
|
| 3 | This module is stdlib-only and can run with Python -I as an external oracle.
|
| 4 | It never imports or executes fixture code. Fixture version 2026-06-09.2 defines
|
| 5 | 600 canonical service configs and four 16 MiB deterministic binary assets.
|
| 6 | """
|
| 7 | from __future__ import annotations
|
| 8 |
|
| 9 | from functools import lru_cache
|
| 10 | import hashlib
|
| 11 | import json
|
| 12 | import os
|
| 13 | from pathlib import Path
|
| 14 | import sys
|
| 15 | import shutil
|
| 16 | import subprocess
|
| 17 | import tempfile
|
| 18 | import re
|
| 19 |
|
| 20 | if __package__ in (None, ""):
|
| 21 | sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 22 | from oakbench.environment import base_env
|
| 23 |
|
| 24 | VERSION = "2026-09-10.2"
|
| 25 | SCENARIOS = frozenset({"wide_config_refactor", "large_asset_manifest"})
|
| 26 | ASSET_BYTES = 16 * 1024 * 1024
|
| 27 |
|
| 28 | # Preserved fixture instructions, not programs executed by this oracle.
|
| 29 | CONFIG_VALIDATOR = "\n".join([
|
| 30 | "from pathlib import Path", "bad = []",
|
| 31 | "for p in Path('services').glob('*/config.toml'):",
|
| 32 | " text = p.read_text()", " if 'timeout_ms = 2500' in text:",
|
| 33 | " bad.append(str(p))", "print(f'bad={len(bad)}')",
|
| 34 | "raise SystemExit(1 if bad else 0)", "",
|
| 35 | ]).encode()
|
| 36 | ASSET_VALIDATOR = "\n".join([
|
| 37 | "import json", "from pathlib import Path",
|
| 38 | "data = json.loads(Path('manifest.json').read_text())",
|
| 39 | "for item in data['assets']:", " p = Path(item['path'])",
|
| 40 | " assert p.exists(), p", " assert p.stat().st_size == item['bytes'], p",
|
| 41 | " assert item['reviewed'] is True, p", "print('manifest ok')", "",
|
| 42 | ]).encode()
|
| 43 |
|
| 44 |
|
| 45 | def _regular_file(path: Path) -> bool:
|
| 46 | return path.is_file() and not path.is_symlink() and not (path.stat().st_mode & 0o111)
|
| 47 |
|
| 48 |
|
| 49 | def _tree_paths(root: Path) -> set[str]:
|
| 50 | """Return every descendant identity, rejecting links and special files."""
|
| 51 | if root.is_symlink() or not root.is_dir():
|
| 52 | raise ValueError("payload_directory_missing_or_link")
|
| 53 | result = set()
|
| 54 | for parent, dirs, files in os.walk(root, followlinks=False):
|
| 55 | for name in dirs + files:
|
| 56 | path = Path(parent) / name
|
| 57 | if path.is_symlink() or (not path.is_dir() and not _regular_file(path)):
|
| 58 | raise ValueError("payload_path_type:" + path.relative_to(root).as_posix())
|
| 59 | result.add(path.relative_to(root).as_posix())
|
| 60 | return result
|
| 61 |
|
| 62 |
|
| 63 | def _config_failure(repo: Path) -> str | None:
|
| 64 | expected_paths = {f"service-{i:04d}" for i in range(600)}
|
| 65 | expected_paths.update(f"service-{i:04d}/config.toml" for i in range(600))
|
| 66 | actual = _tree_paths(repo / "services")
|
| 67 | if actual != expected_paths:
|
| 68 | return "config_path_set_mismatch"
|
| 69 | for i in range(600):
|
| 70 | rel = f"services/service-{i:04d}/config.toml"
|
| 71 | expected = f'name = "service-{i:04d}"\ntimeout_ms = {3000 if i % 3 == 0 else 1800}\nretries = 2\n'.encode()
|
| 72 | if (repo / rel).stat().st_size != len(expected) or (repo / rel).read_bytes() != expected:
|
| 73 | return "config_payload_mismatch:" + rel
|
| 74 | return None
|
| 75 |
|
| 76 |
|
| 77 | def _unique_object(pairs):
|
| 78 | result = {}
|
| 79 | for key, value in pairs:
|
| 80 | if key in result:
|
| 81 | raise ValueError("duplicate_json_key:" + key)
|
| 82 | result[key] = value
|
| 83 | return result
|
| 84 |
|
| 85 |
|
| 86 | @lru_cache(maxsize=4)
|
| 87 | def _expected_asset_hash(index: int) -> str:
|
| 88 | unit = (f"oak-workflow:asset-{index}:".encode() * 4096)[:64 * 1024]
|
| 89 | digest = hashlib.sha256()
|
| 90 | remaining = ASSET_BYTES
|
| 91 | while remaining:
|
| 92 | chunk = unit[:min(len(unit), remaining)]
|
| 93 | digest.update(chunk)
|
| 94 | remaining -= len(chunk)
|
| 95 | return digest.hexdigest()
|
| 96 |
|
| 97 |
|
| 98 | def _asset_failure(repo: Path) -> str | None:
|
| 99 | expected_paths = {f"asset-{i:02d}.bin" for i in range(4)}
|
| 100 | if _tree_paths(repo / "assets") != expected_paths:
|
| 101 | return "asset_path_set_mismatch"
|
| 102 | manifest = repo / "manifest.json"
|
| 103 | if not _regular_file(manifest):
|
| 104 | return "manifest_missing_or_wrong_type"
|
| 105 | if manifest.stat().st_size > 64 * 1024:
|
| 106 | return "manifest_size_out_of_bounds"
|
| 107 | data = json.loads(manifest.read_text(), object_pairs_hook=_unique_object)
|
| 108 | if not isinstance(data, dict) or set(data) != {"assets"}:
|
| 109 | return "manifest_object_shape"
|
| 110 | entries = data["assets"]
|
| 111 | if not isinstance(entries, list) or len(entries) != 4:
|
| 112 | return "manifest_asset_count"
|
| 113 | seen = set()
|
| 114 | for item in entries:
|
| 115 | if not isinstance(item, dict) or set(item) != {"path", "bytes", "reviewed"}:
|
| 116 | return "manifest_entry_shape"
|
| 117 | path = item["path"]
|
| 118 | if not isinstance(path, str) or path not in {"assets/" + p for p in expected_paths}:
|
| 119 | return "manifest_asset_identity"
|
| 120 | if path in seen:
|
| 121 | return "manifest_duplicate_asset:" + path
|
| 122 | seen.add(path)
|
| 123 | if type(item["bytes"]) is not int or item["bytes"] != ASSET_BYTES or item["reviewed"] is not True:
|
| 124 | return "manifest_asset_fields:" + path
|
| 125 | for index in range(4):
|
| 126 | path = repo / "assets" / f"asset-{index:02d}.bin"
|
| 127 | if path.stat().st_size != ASSET_BYTES:
|
| 128 | return "asset_size_mismatch:" + path.name
|
| 129 | digest = hashlib.sha256()
|
| 130 | with path.open("rb") as stream:
|
| 131 | for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
| 132 | digest.update(chunk)
|
| 133 | if digest.hexdigest() != _expected_asset_hash(index):
|
| 134 | return "asset_content_mismatch:" + path.name
|
| 135 | return None
|
| 136 |
|
| 137 |
|
| 138 | def validate(scenario: str, repo: Path) -> str | None:
|
| 139 | """Return a concrete failure reason, or None on a fully verified task."""
|
| 140 | if scenario not in SCENARIOS:
|
| 141 | return "unsupported_trusted_task_oracle:" + scenario
|
| 142 | name, expected_validator = (
|
| 143 | ("validate_configs.py", CONFIG_VALIDATOR) if scenario == "wide_config_refactor"
|
| 144 | else ("validate_manifest.py", ASSET_VALIDATOR)
|
| 145 | )
|
| 146 | try:
|
| 147 | path = repo / name
|
| 148 | if (not _regular_file(path) or path.stat().st_size != len(expected_validator)
|
| 149 | or path.read_bytes() != expected_validator):
|
| 150 | return "fixture_validator_modified_or_missing:" + name
|
| 151 | return _config_failure(repo) if scenario == "wide_config_refactor" else _asset_failure(repo)
|
| 152 | except (OSError, ValueError, UnicodeError, RecursionError) as exc:
|
| 153 | return "trusted_payload_unreadable_or_invalid:" + str(exc)
|
| 154 |
|
| 155 |
|
| 156 | def resolve_git_instrument(explicit: str | None = None) -> tuple[str | None, dict]:
|
| 157 | """Resolve an explicitly configured instrument or the invoking PATH once."""
|
| 158 | requested = explicit or os.environ.get("OAK_BENCH_GIT_INSTRUMENT") or "git"
|
| 159 | resolved = shutil.which(requested)
|
| 160 | metadata = {"git_head_instrument_binary": None, "git_head_instrument_sha256": None,
|
| 161 | "git_head_instrument_version": None}
|
| 162 | if resolved is None:
|
| 163 | return None, metadata
|
| 164 | binary = str(Path(resolved).resolve())
|
| 165 | try:
|
| 166 | version = subprocess.run([binary, "--version"], env=base_env(), capture_output=True, timeout=10)
|
| 167 | if version.returncode:
|
| 168 | return None, metadata
|
| 169 | metadata.update(git_head_instrument_binary=binary,
|
| 170 | git_head_instrument_sha256=hashlib.sha256(Path(binary).read_bytes()).hexdigest(),
|
| 171 | git_head_instrument_version=version.stdout.decode("utf-8", "replace").strip())
|
| 172 | return binary, metadata
|
| 173 | except (OSError, subprocess.TimeoutExpired):
|
| 174 | return None, metadata
|
| 175 |
|
| 176 |
|
| 177 | def git_head_text(repo: Path, path: str, git_binary: str | None = None) -> tuple[str | None, str | None]:
|
| 178 | git, _ = resolve_git_instrument(git_binary)
|
| 179 | if git is None:
|
| 180 | return None, "git_head_oracle_unavailable"
|
| 181 | try:
|
| 182 | result = subprocess.run([git, "-C", str(repo.resolve()), "show", f"HEAD:{path}"],
|
| 183 | env=base_env(), capture_output=True, timeout=30)
|
| 184 | if result.returncode < 0:
|
| 185 | return None, "git_head_oracle_unavailable"
|
| 186 | return (result.stdout.decode("utf-8", "replace"), None) if result.returncode == 0 else (None, "git_head_tree_unreadable")
|
| 187 | except (OSError, subprocess.TimeoutExpired):
|
| 188 | return None, "git_head_oracle_unavailable"
|
| 189 |
|
| 190 |
|
| 191 | def validate_git_head(scenario: str, repo: Path, git_binary: str | None = None) -> str | None:
|
| 192 | """Verify the entire task subtree from committed objects using system Git.
|
| 193 |
|
| 194 | Never checks out subject-controlled hooks/filters or extracts an archive.
|
| 195 | Enumerated blob paths and bounded sizes are checked before materialization.
|
| 196 | Git object replacement and inherited Git environment overrides are disabled.
|
| 197 | """
|
| 198 | git, _ = resolve_git_instrument(git_binary)
|
| 199 | if git is None:
|
| 200 | return "git_head_oracle_unavailable"
|
| 201 | if scenario == "wide_config_refactor":
|
| 202 | paths = {f"services/service-{i:04d}/config.toml" for i in range(600)}
|
| 203 | paths.add("validate_configs.py")
|
| 204 | scope = ["services", "validate_configs.py"]
|
| 205 | max_blob_size = 4096
|
| 206 | elif scenario == "large_asset_manifest":
|
| 207 | paths = {f"assets/asset-{i:02d}.bin" for i in range(4)}
|
| 208 | paths.update({"manifest.json", "validate_manifest.py"})
|
| 209 | scope = ["assets", "manifest.json", "validate_manifest.py"]
|
| 210 | max_blob_size = ASSET_BYTES
|
| 211 | else:
|
| 212 | return "unsupported_trusted_task_oracle:" + scenario
|
| 213 | env = base_env()
|
| 214 | command = [git, "-C", str(repo.resolve())]
|
| 215 | try:
|
| 216 | tree = subprocess.run([*command, "ls-tree", "-r", "-z", "--full-tree", "HEAD", "--", *scope],
|
| 217 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
|
| 218 | if tree.returncode < 0:
|
| 219 | return "git_head_oracle_unavailable"
|
| 220 | if tree.returncode:
|
| 221 | return "git_head_tree_unreadable"
|
| 222 | entries = {}
|
| 223 | for raw in tree.stdout.split(b"\0"):
|
| 224 | if not raw:
|
| 225 | continue
|
| 226 | header, rel = raw.split(b"\t", 1)
|
| 227 | mode, kind, oid = header.split()
|
| 228 | path = rel.decode("utf-8")
|
| 229 | if mode != b"100644" or kind != b"blob":
|
| 230 | return "git_head_path_type:" + path
|
| 231 | if path not in paths or path in entries:
|
| 232 | return "git_head_path_set_mismatch"
|
| 233 | if len(oid) not in (40, 64) or any(c not in b"0123456789abcdef" for c in oid):
|
| 234 | return "git_head_invalid_object_id"
|
| 235 | entries[path] = oid
|
| 236 | if set(entries) != paths:
|
| 237 | return "git_head_path_set_mismatch"
|
| 238 | ordered = sorted(entries)
|
| 239 | request = b"\n".join(entries[path] for path in ordered) + b"\n"
|
| 240 | sizes = subprocess.run([*command, "cat-file", "--batch-check"], input=request,
|
| 241 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
|
| 242 | if sizes.returncode < 0:
|
| 243 | return "git_head_oracle_unavailable"
|
| 244 | if sizes.returncode:
|
| 245 | return "git_head_blob_metadata_unreadable"
|
| 246 | headers = sizes.stdout.splitlines()
|
| 247 | if len(headers) != len(ordered):
|
| 248 | return "git_head_blob_metadata_incomplete"
|
| 249 | for header in headers:
|
| 250 | fields = header.split()
|
| 251 | if len(fields) != 3 or fields[1] != b"blob" or not fields[2].isdigit():
|
| 252 | return "git_head_blob_metadata_invalid"
|
| 253 | if int(fields[2]) > max_blob_size:
|
| 254 | return "git_head_blob_size_out_of_bounds"
|
| 255 | blobs = subprocess.run([*command, "cat-file", "--batch"], input=request,
|
| 256 | env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
|
| 257 | if blobs.returncode < 0:
|
| 258 | return "git_head_oracle_unavailable"
|
| 259 | if blobs.returncode:
|
| 260 | return "git_head_blob_unreadable"
|
| 261 | offset = 0
|
| 262 | with tempfile.TemporaryDirectory(prefix="oakbench-trusted-head-") as tmp:
|
| 263 | root = Path(tmp)
|
| 264 | for path, expected_header in zip(ordered, headers):
|
| 265 | end = blobs.stdout.find(b"\n", offset)
|
| 266 | if end < 0 or blobs.stdout[offset:end] != expected_header:
|
| 267 | return "git_head_blob_framing_invalid"
|
| 268 | size = int(expected_header.split()[2])
|
| 269 | offset = end + 1
|
| 270 | payload = blobs.stdout[offset:offset + size]
|
| 271 | if len(payload) != size or blobs.stdout[offset + size:offset + size + 1] != b"\n":
|
| 272 | return "git_head_blob_framing_invalid"
|
| 273 | offset += size + 1
|
| 274 | destination = root / path
|
| 275 | destination.parent.mkdir(parents=True, exist_ok=True)
|
| 276 | destination.write_bytes(payload)
|
| 277 | if offset != len(blobs.stdout):
|
| 278 | return "git_head_blob_trailing_bytes"
|
| 279 | reason = validate(scenario, root)
|
| 280 | return "git_head_" + reason if reason else None
|
| 281 | except (OSError, ValueError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
| 282 | return "git_head_oracle_error:" + str(exc)
|
| 283 |
|
| 284 |
|
| 285 | def task_evidence(scenario: str, repo: Path, kind: str | None = None, binary: str | None = None, git_instrument: str | None = None) -> dict:
|
| 286 | source = "trusted_exact_task_worktree"
|
| 287 | if kind == "git":
|
| 288 | source += ";known_good_git_head_tree"
|
| 289 | elif kind == "oak":
|
| 290 | source += ";subject_produced_oak_head_export"
|
| 291 | evidence = {"oracle_version": VERSION, "workflow_integrity_source": source,
|
| 292 | "workflow_integrity_passed": None, "workflow_integrity_failure_reason": None,
|
| 293 | "workflow_integrity_unmeasured_reason": None}
|
| 294 | reason = validate(scenario, repo)
|
| 295 | if reason:
|
| 296 | evidence.update(workflow_integrity_passed=False, workflow_integrity_failure_reason=reason)
|
| 297 | return evidence
|
| 298 | if kind in {"git", "oak"}:
|
| 299 | git, metadata = resolve_git_instrument(git_instrument or (binary if kind == "git" else None))
|
| 300 | evidence.update(metadata)
|
| 301 | if git is None:
|
| 302 | evidence["workflow_integrity_unmeasured_reason"] = "git_head_oracle_unavailable"
|
| 303 | return evidence
|
| 304 | if kind == "git":
|
| 305 | reason = validate_git_head(scenario, repo, git)
|
| 306 | else:
|
| 307 | oak = shutil.which(binary or "oak")
|
| 308 | if oak is None:
|
| 309 | evidence["workflow_integrity_unmeasured_reason"] = "oak_export_instrument_unavailable"
|
| 310 | return evidence
|
| 311 | oak = str(Path(oak).resolve())
|
| 312 | evidence["oak_export_instrument_binary"] = oak
|
| 313 | try:
|
| 314 | evidence["oak_export_instrument_sha256"] = hashlib.sha256(Path(oak).read_bytes()).hexdigest()
|
| 315 | status = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
|
| 316 | before = json.loads(status.stdout) if status.returncode == 0 else {}
|
| 317 | head, branch = before.get("head"), before.get("branch")
|
| 318 | if not isinstance(head, str) or not re.fullmatch(r"[0-9a-f]{64}", head) or not isinstance(branch, str) or not branch:
|
| 319 | raise ValueError("oak_head_identity_unreadable")
|
| 320 | evidence["oak_export_head"] = head
|
| 321 | with tempfile.TemporaryDirectory(prefix="oakbench-oak-head-") as tmp:
|
| 322 | destination = Path(tmp) / "export"
|
| 323 | exported = subprocess.run([oak, "export", "--branch", branch, str(destination)], cwd=repo,
|
| 324 | env=base_env(), capture_output=True, timeout=120)
|
| 325 | if exported.returncode:
|
| 326 | evidence["workflow_integrity_unmeasured_reason"] = "oak_head_export_unavailable"
|
| 327 | return evidence
|
| 328 | after = subprocess.run([oak, "status", "--json"], cwd=repo, env=base_env(), capture_output=True, timeout=30)
|
| 329 | state = json.loads(after.stdout) if after.returncode == 0 else {}
|
| 330 | if state.get("head") != head or state.get("branch") != branch:
|
| 331 | raise ValueError("oak_head_changed_during_export")
|
| 332 | reason = validate_git_head(scenario, destination, git)
|
| 333 | except (OSError, ValueError, subprocess.TimeoutExpired) as exc:
|
| 334 | evidence["workflow_integrity_unmeasured_reason"] = str(exc)
|
| 335 | return evidence
|
| 336 | if reason == "git_head_oracle_unavailable" or (reason and reason.startswith("git_head_oracle_error:")):
|
| 337 | evidence["workflow_integrity_unmeasured_reason"] = reason
|
| 338 | return evidence
|
| 339 | evidence.update(workflow_integrity_passed=reason is None, workflow_integrity_failure_reason=reason)
|
| 340 | return evidence
|
| 341 |
|
| 342 |
|
| 343 | def main() -> int:
|
| 344 | import argparse
|
| 345 | parser = argparse.ArgumentParser()
|
| 346 | parser.add_argument("scenario"); parser.add_argument("repo", type=Path)
|
| 347 | parser.add_argument("--git-head", action="store_true")
|
| 348 | parser.add_argument("--oak-head", action="store_true")
|
| 349 | parser.add_argument("--binary")
|
| 350 | parser.add_argument("--evidence-json", action="store_true")
|
| 351 | parser.add_argument("--git-instrument")
|
| 352 | args = parser.parse_args()
|
| 353 | evidence = task_evidence(args.scenario, args.repo, "git" if args.git_head else "oak" if args.oak_head else None, args.binary, args.git_instrument)
|
| 354 | reason = evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"]
|
| 355 | print(json.dumps(evidence) if args.evidence_json else reason or "trusted_task_payload_verified:" + args.scenario)
|
| 356 | passed = evidence["workflow_integrity_passed"]
|
| 357 | return 77 if passed is None else 0 if passed else 1
|
| 358 |
|
| 359 |
|
| 360 | if __name__ == "__main__":
|
| 361 | raise SystemExit(main())
|