| 1 | """Exact output identities and actual patch application, with false-pass controls."""
|
| 2 | import os
|
| 3 | from pathlib import Path
|
| 4 | import shutil
|
| 5 | import subprocess
|
| 6 | import sys
|
| 7 | import tempfile
|
| 8 | import unittest
|
| 9 | from unittest import mock
|
| 10 |
|
| 11 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 12 | import bench
|
| 13 | from oakbench.conformance import Repository, tree_manifest
|
| 14 | from oakbench.environment import base_env
|
| 15 | from oakbench.execution import TimedCapture, run_timed
|
| 16 | from oakbench.output_semantics import (information_recall_fields, reported_paths,
|
| 17 | patch_application_fields, unified_diff_compat_fields, _patch_instrument_metadata)
|
| 18 | from oakbench.public_output import output_evidence_failures
|
| 19 | from oakbench.command_semantics import CommandSemantics
|
| 20 | import json
|
| 21 | from oakbench.subjects import Subject
|
| 22 |
|
| 23 |
|
| 24 | class ExactPathTest(unittest.TestCase):
|
| 25 | def test_core_capture_cap_is_null_evidence_not_subject_failure(self):
|
| 26 | with tempfile.TemporaryDirectory() as tmp:
|
| 27 | root = Path(tmp)
|
| 28 | capture = TimedCapture(1, 0, "", bench.PATCH_PROBE_MAX_BYTES + 1, True,
|
| 29 | "", 0, False, stdout_raw=None)
|
| 30 | subject = Subject(name="git", kind="git", label="Git", bin=Path("/bin/echo"))
|
| 31 | with mock.patch.object(bench, "run_timed", return_value=capture):
|
| 32 | row = bench.run_information_probe(subject, bench.Scenario("small", 1, 10, 1, False, 1),
|
| 33 | "diff.full", root, ["/bin/echo"], 0, {}, ["a"], root)
|
| 34 | self.assertEqual(row["returncode"], 0)
|
| 35 | self.assertIsNone(row["patch_apply_ok"])
|
| 36 | self.assertIsNone(row["patch_tree_matches"])
|
| 37 | self.assertEqual(row["patch_skip_reason"], "patch_output_exceeds_capture_limit")
|
| 38 |
|
| 39 | def test_parser_dispatch_uses_config_even_without_cli_flags(self):
|
| 40 | with tempfile.TemporaryDirectory() as tmp:
|
| 41 | root = Path(tmp)
|
| 42 | config = json.loads((bench.ROOT / "config/command_semantics.json").read_text())
|
| 43 | config["output_formats"]["oak"]["status"]["agent-default"] = "oak-json"
|
| 44 | path = root / "semantics.json"
|
| 45 | path.write_text(json.dumps(config))
|
| 46 | capture = TimedCapture(1, 0, '{"changes":[{"path":"exact"}]}', 30, False, "", 0, False)
|
| 47 | subject = Subject(name="oak", kind="oak", label="Oak", bin=Path("/bin/echo"))
|
| 48 | with mock.patch.object(bench, "semantics", return_value=CommandSemantics(path)), mock.patch.object(bench, "run_timed", return_value=capture):
|
| 49 | row = bench.run_information_probe(subject, bench.Scenario("small", 1, 10, 1, False, 1),
|
| 50 | "status.dirty", root, ["/bin/echo"], 0, {}, ["exact"])
|
| 51 | self.assertEqual(row["output_format"], "oak-json")
|
| 52 | self.assertEqual(row["information_recall"], 1)
|
| 53 |
|
| 54 | def test_substrings_and_diff_bodies_cannot_supply_missing_names(self):
|
| 55 | output = ('diff --git a/prefix-a.txt b/prefix-a.txt\n'
|
| 56 | '--- a/prefix-a.txt\n+++ b/prefix-a.txt\n@@ -1 +1 @@\n'
|
| 57 | '-old\n+mentions a.txt and ghost.txt\n')
|
| 58 | fields = information_recall_fields(output, ["a.txt", "ghost.txt"], len(output), False)
|
| 59 | self.assertEqual(fields["information_recall"], 0)
|
| 60 | self.assertEqual(fields["information_precision"], 0)
|
| 61 | self.assertEqual(fields["changed_files_reported"], 1)
|
| 62 |
|
| 63 | def test_c_quoted_unicode_controls_and_exact_root_names(self):
|
| 64 | output = ' M "space name"\n M "tab\\tname"\n M "line\\nbreak"\n M "\\316\\273"\n M -dash\n M a/root\n'
|
| 65 | expected = ["space name", "tab\tname", "line\nbreak", "λ", "-dash", "a/root"]
|
| 66 | self.assertEqual(reported_paths(output, "status"), sorted(expected))
|
| 67 | self.assertEqual(information_recall_fields(output, expected + expected, len(output), False)["information_recall"], 1)
|
| 68 |
|
| 69 | def test_nul_rename_names_are_not_split_or_normalized(self):
|
| 70 | output = 'R destination -> literal\0source\nname\0 M leading\0'
|
| 71 | self.assertEqual(reported_paths(output, "porcelain-z"),
|
| 72 | sorted(["destination -> literal", "source\nname", " leading"]))
|
| 73 | self.assertEqual(reported_paths('XX arbitrary\0', "porcelain-z"), [])
|
| 74 | self.assertEqual(reported_paths('R "old -> name" -> new\n', "status"), ["new", "old -> name"])
|
| 75 | self.assertEqual(reported_paths('Binary files a/one and two and b/one and two differ\n', "diff"), ["one and two"])
|
| 76 |
|
| 77 | def test_oak_json_only_reads_typed_change_records(self):
|
| 78 | self.assertEqual(reported_paths('{"changes":[{"path":"a.txt"}],"note":"ghost.txt"}', "oak-json"), ["a.txt"])
|
| 79 | self.assertEqual(reported_paths('{"changes":"ghost.txt"}', "oak-json"), [])
|
| 80 |
|
| 81 | def test_multi_file_oak_hunks_reset_on_oak_headers(self):
|
| 82 | output = ('diff --oak a/first b/first\n--- a/first\n+++ b/first\n@@ -1 +1 @@\n-old\n+new\n'
|
| 83 | 'diff --oak a/space name b/space name\n--- a/space name\n+++ b/space name\n@@ -1 +1 @@\n-old\n+new\n')
|
| 84 | self.assertEqual(reported_paths(output, "diff"), ["first", "space name"])
|
| 85 |
|
| 86 | def test_metadata_grammar_preserves_paths_and_excludes_bare_prose(self):
|
| 87 | self.assertEqual(reported_paths(" M leading\n M trailing \n", "status"), [" leading", "trailing "])
|
| 88 | self.assertEqual(reported_paths("a.txt mentioned in prose\n", "status"), [])
|
| 89 | self.assertEqual(reported_paths(' M "bad\\qname"\n', "status"), [])
|
| 90 | self.assertEqual(reported_paths(" src/a.txt | 2 +-\n", "stat"), ["src/a.txt"])
|
| 91 | self.assertEqual(reported_paths("M path with space +2 -1\n", "stat"), ["path with space"])
|
| 92 | self.assertEqual(reported_paths(" src/{old => new}/a | 2 +-\n", "stat"), ["src/new/a", "src/old/a"])
|
| 93 |
|
| 94 |
|
| 95 | @unittest.skipUnless(shutil.which("git"), "git apply correctness instrument required")
|
| 96 | class PatchApplicationTest(unittest.TestCase):
|
| 97 | def setUp(self):
|
| 98 | self.tmp = tempfile.TemporaryDirectory()
|
| 99 | self.root = Path(self.tmp.name)
|
| 100 | self.pristine = self.root / "pristine"
|
| 101 | self.pristine.mkdir()
|
| 102 | (self.pristine / "alpha").write_bytes(b"before\n")
|
| 103 | (self.pristine / "space name").write_bytes(b"keep\n")
|
| 104 | self.expected = self.root / "expected"
|
| 105 | shutil.copytree(self.pristine, self.expected)
|
| 106 | (self.expected / "alpha").write_bytes(b"after\n")
|
| 107 | (self.expected / "space name").write_bytes(b"changed\n")
|
| 108 | self.patch = (b"diff --git a/alpha b/alpha\n--- a/alpha\n+++ b/alpha\n@@ -1 +1 @@\n-before\n+after\n"
|
| 109 | b"diff --git a/space name b/space name\n--- a/space name\n+++ b/space name\n@@ -1 +1 @@\n-keep\n+changed\n")
|
| 110 |
|
| 111 | def tearDown(self):
|
| 112 | self.tmp.cleanup()
|
| 113 |
|
| 114 | def check(self, patch, **kwargs):
|
| 115 | return patch_application_fields(patch, self.pristine, tree_manifest(self.expected), self.root, **kwargs)
|
| 116 |
|
| 117 | def test_patch_and_mutation_fingerprints_share_manifest_canonicalization(self):
|
| 118 | from oakbench.output_semantics import manifest_sha256, finalize_patch_evidence
|
| 119 | expected = tree_manifest(self.expected)
|
| 120 | reordered = dict(reversed(list(expected.items())))
|
| 121 | self.assertEqual(manifest_sha256(expected), manifest_sha256(reordered))
|
| 122 | fields = self.check(self.patch)
|
| 123 | self.assertEqual(fields["patch_expected_tree_sha256"], manifest_sha256(expected))
|
| 124 | (self.expected / "extra").write_bytes(b"unexpected")
|
| 125 | changed = tree_manifest(self.expected)
|
| 126 | finalized = finalize_patch_evidence(fields, expected, changed, application_required=True)
|
| 127 | self.assertEqual(finalized["patch_subject_tree_after_sha256"], manifest_sha256(changed))
|
| 128 | self.assertEqual(finalized["returncode"], 1)
|
| 129 | self.assertTrue(fields["patch_apply_ok"])
|
| 130 |
|
| 131 | def test_exact_tree_application_preserves_pristine(self):
|
| 132 | before = tree_manifest(self.pristine)
|
| 133 | fields = self.check(self.patch)
|
| 134 | self.assertTrue(fields["patch_apply_ok"], fields)
|
| 135 | self.assertTrue(fields["patch_tree_matches"])
|
| 136 | self.assertEqual(tree_manifest(self.pristine), before)
|
| 137 |
|
| 138 | def test_truncated_malformed_missing_sections_and_binary_notice_fail(self):
|
| 139 | invalid = [self.patch[:-5], self.patch.replace(b"@@ -1 +1 @@", b"@@ -1,9 +1,9 @@", 1),
|
| 140 | self.patch.split(b"diff --git a/space name")[0],
|
| 141 | b"Binary files a/alpha and b/alpha differ\n", b""]
|
| 142 | for patch in invalid:
|
| 143 | with self.subTest(patch=patch):
|
| 144 | self.assertFalse(self.check(patch)["patch_apply_ok"])
|
| 145 | self.assertTrue(unified_diff_compat_fields(invalid[3].decode())["pipe_compatible_unified"])
|
| 146 | self.assertFalse(self.check(self.patch, output_truncated=True)["patch_apply_ok"])
|
| 147 |
|
| 148 | def test_wrong_content_mode_or_extra_path_cannot_pass(self):
|
| 149 | for mutation in (lambda: (self.expected / "alpha").write_bytes(b"wrong\n"),
|
| 150 | lambda: (self.expected / "extra").write_bytes(b""),
|
| 151 | lambda: (self.expected / "space name").chmod(0o755)):
|
| 152 | mutation()
|
| 153 | self.assertFalse(self.check(self.patch)["patch_apply_ok"])
|
| 154 |
|
| 155 | def test_missing_no_final_newline_marker_is_not_applicable(self):
|
| 156 | (self.pristine / "alpha").write_bytes(b"before")
|
| 157 | patch = self.patch.replace(b"-before\n", b"-before\n\\ No newline at end of file\n")
|
| 158 | self.assertTrue(self.check(patch)["patch_apply_ok"])
|
| 159 | self.assertFalse(self.check(self.patch)["patch_apply_ok"])
|
| 160 |
|
| 161 | def test_actual_core_fixture_marker_is_not_subject_content(self):
|
| 162 | scenario = bench.Scenario("marker_patch_fixture_v1", 3, 1024, 2, False, 1)
|
| 163 | fixture = bench.ensure_fixture(self.root / "fixtures", scenario)
|
| 164 | self.assertTrue((fixture / ".fixture-ready").exists())
|
| 165 | subject = Subject(name="git", kind="git", label="Git", bin=Path(shutil.which("git")))
|
| 166 | rows = bench.run_subject_scenario(subject, scenario, fixture, 0, self.root / "runs",
|
| 167 | {}, False, 20000, "core-equivalent", skip_remote=True)
|
| 168 | proof = next(row for row in rows if row["operation"] == "diff.full.inforecall")
|
| 169 | self.assertTrue(proof["patch_apply_ok"], proof)
|
| 170 | self.assertEqual(proof["information_recall"], 1)
|
| 171 | dirty = next(row for row in rows if row["operation"] == "diff.dirty.inforecall")
|
| 172 | self.assertEqual(dirty["information_recall"], 1)
|
| 173 | self.assertTrue(dirty["pipe_compatible_unified"])
|
| 174 | self.assertEqual(output_evidence_failures(dirty), [])
|
| 175 | self.assertEqual(output_evidence_failures(proof), [])
|
| 176 |
|
| 177 | def test_instrument_errors_are_unmeasured(self):
|
| 178 | for error in (subprocess.TimeoutExpired("git", 30), subprocess.CalledProcessError(1, "git")):
|
| 179 | _patch_instrument_metadata.cache_clear()
|
| 180 | with self.subTest(error=error), mock.patch("subprocess.run", side_effect=error):
|
| 181 | fields = self.check(self.patch)
|
| 182 | self.assertIsNone(fields["patch_apply_ok"])
|
| 183 | self.assertIsNone(fields["patch_tree_matches"])
|
| 184 | self.assertIn("instrument", fields["patch_skip_reason"])
|
| 185 | self.assertNotIn("patch_failure_reason", fields)
|
| 186 |
|
| 187 | def test_apply_timeout_after_valid_identity_is_unmeasured(self):
|
| 188 | self.assertTrue(self.check(self.patch)["patch_apply_ok"])
|
| 189 | with mock.patch("subprocess.run", side_effect=subprocess.TimeoutExpired("git apply", 30)):
|
| 190 | fields = self.check(self.patch)
|
| 191 | self.assertIsNone(fields["patch_apply_ok"])
|
| 192 | self.assertEqual(fields["patch_skip_reason"], "git_apply_instrument_timeout")
|
| 193 | self.assertIn("patch_instrument_sha256", fields)
|
| 194 |
|
| 195 | def test_fixture_copy_failure_is_unmeasured(self):
|
| 196 | with mock.patch("shutil.copytree", side_effect=OSError("disk unavailable")):
|
| 197 | fields = self.check(self.patch)
|
| 198 | self.assertIsNone(fields["patch_apply_ok"])
|
| 199 | self.assertEqual(fields["patch_skip_reason"], "git_apply_instrument_error")
|
| 200 |
|
| 201 | @unittest.skipUnless(shutil.which("oak"), "Oak producer control required")
|
| 202 | def test_real_same_oak_binary_produces_comparable_failure_fingerprints(self):
|
| 203 | import devloop
|
| 204 | scenario = bench.Scenario("identical_patch_inputs_v1", 3, 1024, 2, False, 1)
|
| 205 | fixture = bench.ensure_fixture(self.root / "paired-fixtures", scenario)
|
| 206 | proofs = []
|
| 207 | for name in ("oak_installed", "oak_local"):
|
| 208 | subject = Subject(name=name, kind="oak", label=name, bin=Path(shutil.which("oak")))
|
| 209 | rows = bench.run_subject_scenario(subject, scenario, fixture, 0, self.root / "paired-runs",
|
| 210 | {}, False, 20000, "core-equivalent", skip_remote=True)
|
| 211 | proofs.append(next(row for row in rows if row["operation"] == "diff.full.inforecall"))
|
| 212 | self.assertEqual(proofs[0]["patch_input_sha256"], proofs[1]["patch_input_sha256"])
|
| 213 | self.assertEqual(proofs[0]["patch_expected_tree_sha256"], proofs[1]["patch_expected_tree_sha256"])
|
| 214 | self.assertEqual(proofs[0]["patch_apply_ok"], proofs[1]["patch_apply_ok"])
|
| 215 | diagnostics = []
|
| 216 | failures, _ = devloop.compare_subjects(proofs, "oak_local", "oak_installed", "same binary", 100, 100,
|
| 217 | efficiency_gates=False, output_bytes_threshold_pct=100, diagnostics=diagnostics)
|
| 218 | self.assertEqual(failures, [])
|
| 219 | if proofs[0]["patch_apply_ok"] is False:
|
| 220 | self.assertTrue(any("PREEXISTING" in note for note in diagnostics))
|
| 221 |
|
| 222 | def test_subject_mutation_has_failure_evidence_without_instrument_skip(self):
|
| 223 | import devloop
|
| 224 | subject = Subject(name="oak_local", kind="oak", label="Oak", bin=Path("/bin/echo"))
|
| 225 | def mutating_capture(*args, **kwargs):
|
| 226 | (self.expected / "alpha").write_bytes(b"collateral mutation\n")
|
| 227 | return TimedCapture(1, 0, "", 0, False, "", 0, False, stdout_raw=b"")
|
| 228 | original = patch_application_fields
|
| 229 | def missing_instrument(*args, **kwargs):
|
| 230 | return original(*args, **kwargs, git_binary="/missing/instrument")
|
| 231 | with mock.patch.object(bench, "run_timed", side_effect=mutating_capture), mock.patch.object(bench.oakbench_semantics, "patch_application_fields", side_effect=missing_instrument):
|
| 232 | row = bench.run_information_probe(subject, bench.Scenario("small", 2, 10, 2, False, 1),
|
| 233 | "diff.full", self.expected, ["/bin/echo"], 0, {}, ["alpha"], self.pristine)
|
| 234 | self.assertEqual(row["returncode"], 1)
|
| 235 | self.assertIs(row["patch_apply_ok"], False)
|
| 236 | self.assertNotIn("patch_skip_reason", row)
|
| 237 | self.assertEqual(row["patch_failure_reason"], "diff_mutated_subject_tree")
|
| 238 | self.assertIsNone(row["patch_apply_returncode"])
|
| 239 | self.assertTrue(devloop.same_patch_failure(row, dict(row)))
|
| 240 | self.assertFalse(devloop.same_patch_failure(row, {**row, "patch_subject_tree_after_sha256": "f" * 64}))
|
| 241 |
|
| 242 | @unittest.skipUnless(shutil.which("oak"), "Oak producer control required")
|
| 243 | def test_default_core_binary_probe_is_optional_for_cli_verdict(self):
|
| 244 | import contextlib
|
| 245 | import io
|
| 246 | import devloop
|
| 247 | scenario = next(item for item in bench.PROFILES["smoke"] if item.binary)
|
| 248 | fixture = bench.ensure_fixture(self.root / "default-binary-fixtures", scenario)
|
| 249 | proofs = []
|
| 250 | for name in ("oak_installed", "oak_local"):
|
| 251 | subject = Subject(name=name, kind="oak", label=name, bin=Path(shutil.which("oak")))
|
| 252 | rows = bench.run_subject_scenario(subject, scenario, fixture, 0, self.root / "binary-runs",
|
| 253 | {}, False, 20000, "agent-default", skip_remote=True)
|
| 254 | proofs.append(next(row for row in rows if row["operation"] == "diff.full.inforecall"))
|
| 255 | self.assertTrue(all(row["patch_application_required"] is False for row in proofs))
|
| 256 | argv = ["devloop.py", "--lanes", "core", "--skip-noise-floor", "--skip-build",
|
| 257 | "--oak-baseline-bin", "/bin/echo", "--oak-local-bin", "/bin/echo", "--results", str(self.root / "verdict")]
|
| 258 | output = io.StringIO()
|
| 259 | with mock.patch.object(sys, "argv", argv), mock.patch.object(devloop, "run_lane"), mock.patch.object(devloop, "load_jsonl", return_value=proofs), mock.patch.object(devloop.oakbench_remotes, "print_remote_preflight_warnings"), contextlib.redirect_stdout(output):
|
| 260 | self.assertEqual(devloop.main(), 0, output.getvalue())
|
| 261 | self.assertIn("**PASS**", output.getvalue())
|
| 262 | if any(row["patch_apply_ok"] is None for row in proofs):
|
| 263 | self.assertIn("INFO PATCH-APPLY", output.getvalue())
|
| 264 |
|
| 265 | def test_missing_instrument_is_null_not_false_success(self):
|
| 266 | fields = self.check(self.patch, git_binary="/missing/git-apply")
|
| 267 | self.assertIsNone(fields["patch_apply_ok"])
|
| 268 | self.assertTrue(fields["patch_skip_reason"])
|
| 269 |
|
| 270 | def test_patch_cannot_escape_disposable_tree(self):
|
| 271 | outside = self.root / "outside"
|
| 272 | outside.write_bytes(b"sentinel\n")
|
| 273 | patch = b"--- a/../../outside\n+++ b/../../outside\n@@ -1 +1 @@\n-sentinel\n+corrupt\n"
|
| 274 | self.assertFalse(self.check(patch)["patch_apply_ok"])
|
| 275 | self.assertEqual(outside.read_bytes(), b"sentinel\n")
|
| 276 |
|
| 277 | def test_raw_capture_does_not_reencode_invalid_utf8(self):
|
| 278 | command = [sys.executable, "-c", "import sys; sys.stdout.buffer.write(bytes([255,13,10,0]))"]
|
| 279 | capture = run_timed(command, self.root, base_env(), 20, raw_output_bytes=4)
|
| 280 | self.assertEqual(capture.stdout_raw, b"\xff\r\n\0")
|
| 281 | self.assertIsNone(run_timed(command, self.root, base_env(), 20, raw_output_bytes=3).stdout_raw)
|
| 282 |
|
| 283 | def test_real_git_and_available_oak_emit_applicable_multifile_patch(self):
|
| 284 | for kind in ("git", "oak"):
|
| 285 | binary = shutil.which(kind)
|
| 286 | if binary is None:
|
| 287 | continue
|
| 288 | with self.subTest(subject=kind):
|
| 289 | root = self.root / kind
|
| 290 | shutil.copytree(self.pristine, root)
|
| 291 | repo = Repository(root, kind, binary, self.root / (kind + "-home"))
|
| 292 | repo.initialize(); repo.commit()
|
| 293 | for path in self.expected.iterdir():
|
| 294 | shutil.copy2(path, root / path.name)
|
| 295 | subject = Subject(name=kind, kind=kind, label=kind, bin=Path(binary))
|
| 296 | if kind == "git":
|
| 297 | stat_text = repo.run("diff", "--stat").stdout.decode()
|
| 298 | self.assertEqual(reported_paths(stat_text, "stat"), ["alpha", "space name"])
|
| 299 | command = [binary, "diff", "--no-ext-diff" if kind == "git" else "--print"]
|
| 300 | row = bench.run_information_probe(subject, bench.Scenario("independent_patch_fixture_v1", 2, 10, 2, False, 1),
|
| 301 | "diff.full", root, command, 0, {}, ["alpha", "space name"], self.pristine)
|
| 302 | self.assertEqual(row["returncode"], 0, row)
|
| 303 | self.assertEqual(row["information_recall"], 1)
|
| 304 | self.assertEqual(row["information_precision"], 1)
|
| 305 | self.assertTrue(row["patch_apply_ok"], row)
|
| 306 |
|
| 307 |
|
| 308 | if __name__ == "__main__":
|
| 309 | unittest.main()
|