Viewing files at commit 1c9b819ab86bCommit detailsView latest

Browse files
tests/test_workflow_oracles.py 447 lines · 26.6 KB · python
1
"""Negative task-oracle tests: fixture validators are not trusted evidence."""
2
from __future__ import annotations
3
4
import json
5
import subprocess
6
from pathlib import Path
7
import shutil
8
import sys
9
import tempfile
10
import unittest
11
from unittest import mock
12
13
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
14
import workflow_ab
15
import agent_workflow
16
from oakbench import workflow_oracles
17
18
19
class ExistingOracleFalsePassTest(unittest.TestCase):
20
    def test_oracle_version_is_pinned(self):
21
        self.assertEqual(workflow_oracles.VERSION, "2026-09-27.1")
22
23
    def test_deleting_rewritten_configs_fails(self):
24
        with tempfile.TemporaryDirectory() as tmp:
25
            repo = Path(tmp)
26
            workflow_ab.make_wide_config_fixture(repo)
27
            for path in (repo / "services").glob("*/config.toml"):
28
                path.write_text(path.read_text().replace("timeout_ms = 2500", "timeout_ms = 3000"))
29
            (repo / "services/service-0003/config.toml").unlink()
30
            self.assertIsNotNone(workflow_ab.workflow_worktree_payload_failure(
31
                workflow_ab.WORKFLOWS["wide_config_refactor"], repo
32
            ))
33
34
    def test_empty_manifest_fails(self):
35
        with tempfile.TemporaryDirectory() as tmp:
36
            repo = Path(tmp)
37
            (repo / "manifest.json").write_text('{"assets": []}')
38
            self.assertIsNotNone(workflow_ab.workflow_worktree_payload_failure(
39
                workflow_ab.WORKFLOWS["large_asset_manifest"], repo
40
            ))
41
42
43
44
def finish_configs(repo):
45
    for path in (repo / "services").glob("*/config.toml"):
46
        path.write_text(path.read_text().replace("timeout_ms = 2500", "timeout_ms = 3000"))
47
48
49
def finish_assets(repo):
50
    path = repo / "manifest.json"
51
    data = json.loads(path.read_text())
52
    for entry in data["assets"]:
53
        entry["reviewed"] = True
54
    path.write_text(json.dumps(data))
55
56
57
class TrustedWorkflowOracleTest(unittest.TestCase):
58
    def assert_both_lanes_reject(self, name, repo, reason):
59
        workflow = workflow_ab.WORKFLOWS[name]
60
        failure = workflow_ab.workflow_worktree_payload_failure(workflow, repo)
61
        self.assertIsNotNone(failure)
62
        self.assertIn(reason, failure)
63
        # This is the exact external command run by real-agent run_oracle.
64
        proc = agent_workflow.run_command(agent_workflow.test_command_for(workflow), repo)
65
        self.assertEqual(proc.returncode, 1, proc.stderr)
66
        self.assertIn(reason, proc.stdout)
67
68
    def test_exact_configs_reject_loss_extras_wrong_values_and_types(self):
69
        with tempfile.TemporaryDirectory() as tmp:
70
            repo = Path(tmp)
71
            workflow_ab.make_wide_config_fixture(repo)
72
            finish_configs(repo)
73
            self.assertIsNone(workflow_oracles.validate("wide_config_refactor", repo))
74
            path = repo / "services/service-0003/config.toml"
75
            original = path.read_bytes()
76
            for wrong in (b"", original.replace(b"3000", b"1800"), original + b"retries = 99\n"):
77
                with self.subTest(wrong=wrong):
78
                    path.write_bytes(wrong)
79
                    self.assert_both_lanes_reject("wide_config_refactor", repo, "config_payload_mismatch")
80
            path.unlink()
81
            self.assert_both_lanes_reject("wide_config_refactor", repo, "config_path_set_mismatch")
82
            path.symlink_to("../../service-0000/config.toml")
83
            self.assert_both_lanes_reject("wide_config_refactor", repo, "payload_path_type")
84
            path.unlink()
85
            path.write_bytes(original)
86
            extra = repo / "services/extra"
87
            extra.write_bytes(b"unexpected")
88
            self.assert_both_lanes_reject("wide_config_refactor", repo, "config_path_set_mismatch")
89
            extra.unlink()
90
            untouched = repo / "services/service-0001/config.toml"
91
            untouched.write_bytes(untouched.read_bytes().replace(b"1800", b"3000"))
92
            self.assert_both_lanes_reject("wide_config_refactor", repo, "config_payload_mismatch")
93
94
    def test_assets_reject_manifest_lies_and_content_corruption(self):
95
        with tempfile.TemporaryDirectory() as tmp:
96
            repo = Path(tmp)
97
            workflow_ab.make_large_asset_fixture(repo)
98
            finish_assets(repo)
99
            self.assertIsNone(workflow_oracles.validate("large_asset_manifest", repo))
100
            path = repo / "manifest.json"
101
            valid = json.loads(path.read_text())
102
            variants = [
103
                ({"assets": []}, "manifest_asset_count"),
104
                ({"assets": valid["assets"][:3]}, "manifest_asset_count"),
105
                ({"assets": valid["assets"] + [valid["assets"][0]]}, "manifest_asset_count"),
106
                ({"assets": [valid["assets"][0]] * 4}, "manifest_duplicate_asset"),
107
                ({"assets": [{**item, "bytes": True} for item in valid["assets"]]}, "manifest_asset_fields"),
108
                ({"assets": [{**item, "reviewed": 1} for item in valid["assets"]]}, "manifest_asset_fields"),
109
                ({"assets": [{**item, "path": "../outside"} for item in valid["assets"]]}, "manifest_asset_identity"),
110
                ({"assets": None}, "manifest_asset_count"),
111
                ({}, "manifest_object_shape"),
112
            ]
113
            for data, reason in variants:
114
                with self.subTest(data=data):
115
                    path.write_text(json.dumps(data))
116
                    self.assert_both_lanes_reject("large_asset_manifest", repo, reason)
117
            path.write_text('{"assets": [], "assets": []}')
118
            self.assert_both_lanes_reject("large_asset_manifest", repo, "duplicate_json_key")
119
            path.write_text(json.dumps(valid))
120
            extra = repo / "assets/asset-04.bin"
121
            extra.write_bytes(b"extra")
122
            self.assert_both_lanes_reject("large_asset_manifest", repo, "asset_path_set_mismatch")
123
            extra.unlink()
124
            asset = repo / "assets/asset-03.bin"
125
            with asset.open("r+b") as stream:
126
                stream.write(b"X")
127
            self.assert_both_lanes_reject("large_asset_manifest", repo, "asset_content_mismatch")
128
            asset.unlink()
129
            self.assert_both_lanes_reject("large_asset_manifest", repo, "asset_path_set_mismatch")
130
131
    def test_deleted_or_noop_fixture_validator_cannot_bypass_oracle(self):
132
        for name, filename, maker, finish in (
133
            ("wide_config_refactor", "validate_configs.py", workflow_ab.make_wide_config_fixture, finish_configs),
134
            ("large_asset_manifest", "validate_manifest.py", workflow_ab.make_large_asset_fixture, finish_assets),
135
        ):
136
            with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
137
                repo = Path(tmp) / "repo"
138
                repo.mkdir()
139
                artifacts = Path(tmp) / "artifacts"
140
                artifacts.mkdir()
141
                maker(repo)
142
                before = agent_workflow.workspace_manifest(repo)
143
                finish(repo)
144
                validator = repo / filename
145
                for replacement in (b"raise SystemExit(0)\n", None):
146
                    if replacement is None:
147
                        validator.unlink()
148
                    else:
149
                        validator.write_bytes(replacement)
150
                    self.assert_both_lanes_reject(name, repo, "fixture_validator_modified_or_missing")
151
                    subject = workflow_ab.Subject(name="git", kind="git", label="git", bin=Path("/git"))
152
                    with mock.patch.object(agent_workflow, "vcs_status", return_value=("", 0, 0.0)), \
153
                         mock.patch.object(agent_workflow, "commit_count", return_value=2):
154
                        result = agent_workflow.run_oracle(
155
                            subject, workflow_ab.WORKFLOWS[name], repo, before, 1, artifacts,
156
                        )
157
                    self.assertFalse(result.success)
158
                    self.assertFalse(result.checks["required_checks_passed"])
159
                    self.assertIn("fixture_validator_modified_or_missing", (artifacts / "oracle.test.log").read_text())
160
161
    def test_invalid_trusted_oracle_envelope_is_unmeasured(self):
162
        with tempfile.TemporaryDirectory() as tmp:
163
            repo = Path(tmp)
164
            workflow_ab.make_wide_config_fixture(repo)
165
            before = agent_workflow.workspace_manifest(repo)
166
            finish_configs(repo)
167
            artifacts = repo / "logs"
168
            artifacts.mkdir()
169
            subject = workflow_ab.Subject("git", "git", "Git", Path("/git"))
170
            for output, code in (("", 1), ("{}", 0), ("not json", 0),
171
                                 (json.dumps({"oracle_version": workflow_oracles.VERSION,
172
                                              "workflow_integrity_source": "trusted", "workflow_integrity_passed": None}), 77)):
173
                with mock.patch.object(agent_workflow, "run_command", return_value=subprocess.CompletedProcess([], code, output, "")), \
174
                     mock.patch.object(agent_workflow, "vcs_status", return_value=("", 0, 0)), \
175
                     mock.patch.object(agent_workflow, "commit_count", return_value=2):
176
                    result = agent_workflow.run_oracle(subject, workflow_ab.WORKFLOWS["wide_config_refactor"], repo, before, 1, artifacts)
177
                self.assertIsNone(result.evidence["workflow_integrity_passed"])
178
                self.assertEqual(result.evidence["workflow_integrity_unmeasured_reason"], "trusted_oracle_evidence_invalid_or_missing")
179
180
    def test_unavailable_required_check_does_not_hide_measured_failure(self):
181
        with tempfile.TemporaryDirectory() as tmp:
182
            repo = Path(tmp)
183
            workflow_ab.make_wide_config_fixture(repo)
184
            before = agent_workflow.workspace_manifest(repo)
185
            finish_configs(repo)
186
            evidence = {"oracle_version": workflow_oracles.VERSION, "workflow_integrity_source": "trusted",
187
                "workflow_integrity_passed": None, "workflow_integrity_unmeasured_reason": "missing"}
188
            with mock.patch.object(agent_workflow, "run_command", return_value=subprocess.CompletedProcess([], 77, json.dumps(evidence), "")), \
189
                 mock.patch.object(agent_workflow, "vcs_status", return_value=("dirty", 1, 0)), \
190
                 mock.patch.object(agent_workflow, "expected_change_ok", return_value=True), \
191
                 mock.patch.object(agent_workflow, "commit_count", return_value=2):
192
                result = agent_workflow.run_oracle(workflow_ab.Subject("git", "git", "Git", Path("/git")),
193
                    workflow_ab.WORKFLOWS["wide_config_refactor"], repo, before, 1, repo)
194
            self.assertIsNone(result.checks["required_checks_passed"])
195
            self.assertEqual(result.failure_reason, "final_vcs_clean")
196
197
    def _bugfix_oracle(self, repo, behavior):
198
        trusted = {"oracle_version": workflow_oracles.VERSION, "workflow_integrity_source": "trusted",
199
                   "workflow_integrity_evidence_class": "independent_committed_payload",
200
                   "workflow_integrity_passed": True, "workflow_integrity_failure_reason": None,
201
                   "workflow_integrity_unmeasured_reason": None}
202
        real_run = agent_workflow.run_command
203
        calls = []
204
        def fake(command, cwd, *args, **kwargs):
205
            calls.append(command)
206
            if len(calls) == 1:
207
                return subprocess.CompletedProcess(command, 0, json.dumps(trusted), "")
208
            return behavior(real_run, command, cwd, *args, **kwargs)
209
        with mock.patch.object(agent_workflow, "run_command", side_effect=fake), \
210
             mock.patch.object(agent_workflow, "vcs_status", return_value=("", 0, 0)), \
211
             mock.patch.object(agent_workflow, "expected_change_ok", return_value=True), \
212
             mock.patch.object(agent_workflow, "commit_count", return_value=2):
213
            return agent_workflow.run_oracle(workflow_ab.Subject("git", "git", "Git", Path("/git")),
214
                workflow_ab.WORKFLOWS["bugfix_test_loop"], repo, {}, 1, repo)
215
216
    def test_subject_controlled_exit_77_is_a_measured_failure_not_unmeasured(self):
217
        # Real producer: agent-edited test module exits 77 at import time.
218
        with tempfile.TemporaryDirectory() as tmp:
219
            repo = Path(tmp)
220
            (repo / "tests").mkdir()
221
            (repo / "tests" / "test_sentinel.py").write_text("import os\nos._exit(77)\n")
222
            result = self._bugfix_oracle(repo, lambda real, command, cwd, *a, **k: real(command, cwd, *a, **k))
223
            self.assertIs(result.checks["behavior_tests_passed"], False)
224
            self.assertEqual(result.failure_reason, "behavior_tests_passed")
225
            self.assertIs(result.evidence["workflow_integrity_passed"], True)
226
            self.assertIsNone(result.evidence["workflow_integrity_unmeasured_reason"])
227
228
    def test_behavior_interpreter_launch_failure_is_harness_unmeasured(self):
229
        def missing(real, command, cwd, *a, **k):
230
            raise FileNotFoundError("python3")
231
        with tempfile.TemporaryDirectory() as tmp:
232
            result = self._bugfix_oracle(Path(tmp), missing)
233
            self.assertIsNone(result.checks["behavior_tests_passed"])
234
            self.assertIsNone(result.evidence["workflow_integrity_passed"])
235
            self.assertEqual(result.evidence["workflow_integrity_unmeasured_reason"], "behavior_test_instrument_unavailable")
236
237
    def test_fixture_module_shadowing_is_not_executed(self):
238
        with tempfile.TemporaryDirectory() as tmp:
239
            repo = Path(tmp)
240
            workflow_ab.make_wide_config_fixture(repo)
241
            finish_configs(repo)
242
            (repo / "json.py").write_text("raise RuntimeError('untrusted import executed')")
243
            proc = agent_workflow.run_command(
244
                agent_workflow.test_command_for(workflow_ab.WORKFLOWS["wide_config_refactor"]), repo,
245
            )
246
            self.assertEqual(proc.returncode, 0, proc.stderr)
247
248
    def test_scripted_unavailable_instrument_row_persists(self):
249
        from oakbench.rows import validate_rows
250
        from oakbench.results import ResultsStore
251
        with tempfile.TemporaryDirectory() as tmp:
252
            repo = Path(tmp)
253
            workflow_ab.make_wide_config_fixture(repo)
254
            finish_configs(repo)
255
            subject = workflow_ab.Subject("git", "git", "Git", Path("/subject/not-reader"))
256
            with mock.patch.object(workflow_oracles, "resolve_git_instrument", return_value=(None, {})):
257
                evidence = workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS["wide_config_refactor"], repo, [])
258
            row = workflow_ab.sum_rows([], {"bench_id": "test", "profile": "test"}, subject,
259
                workflow_ab.WORKFLOWS["wide_config_refactor"], 0, "workflow", "workflow.total", evidence)
260
            self.assertEqual(row["returncode"], 77)
261
            self.assertEqual(validate_rows([row], "workflow"), [])
262
            ResultsStore(repo / "results", "workflow").write("test", [row])
263
264
    def test_workflow_summary_fails_when_trusted_payload_fails(self):
265
        with tempfile.TemporaryDirectory() as tmp:
266
            repo = Path(tmp)
267
            workflow_ab.make_wide_config_fixture(repo)
268
            finish_configs(repo)
269
            (repo / "services/service-0003/config.toml").unlink()
270
            subject = workflow_ab.Subject(name="oak", kind="oak", label="oak", bin=Path("/oak"))
271
            fields = workflow_ab.workflow_integrity_fields(
272
                subject, workflow_ab.WORKFLOWS["wide_config_refactor"], repo, [],
273
            )
274
            self.assertFalse(fields["correctness_passed"])
275
            self.assertIn("config_path_set_mismatch", fields["correctness_failure_reason"])
276
            self.assertEqual(fields["oracle_version"], workflow_oracles.VERSION)
277
278
279
@unittest.skipIf(shutil.which("git") is None, "git not installed")
280
class CommittedWorkflowOracleTest(unittest.TestCase):
281
    def test_subject_binary_never_becomes_committed_tree_reader(self):
282
        with tempfile.TemporaryDirectory() as tmp:
283
            repo = Path(tmp)
284
            workflow_ab.make_wide_config_fixture(repo)
285
            finish_configs(repo)
286
            subject = workflow_ab.Subject("git", "git", "Git candidate", Path("/candidate/git"))
287
            with mock.patch.object(workflow_oracles, "resolve_git_instrument", return_value=(None, {})) as resolve:
288
                workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS["wide_config_refactor"], repo, [])
289
                resolve.assert_called_once_with(None)
290
            command = agent_workflow.test_command_for(workflow_ab.WORKFLOWS["wide_config_refactor"], subject)
291
            self.assertEqual(command[command.index("--binary") + 1], "/candidate/git")
292
            self.assertNotIn("--git-instrument", command)
293
294
    def test_nontrusted_git_records_actual_scoped_head_evidence(self):
295
        with tempfile.TemporaryDirectory() as tmp:
296
            repo = Path(tmp)
297
            workflow_ab.make_bugfix_fixture(repo)
298
            path = repo / "app/pricing.py"
299
            path.write_text(path.read_text().replace("return cents - percent", "return cents - ((cents * percent) // 100)"))
300
            self.git(repo, "init"); self.git(repo, "add", "."); self.git(repo, "commit", "-m", "fixed")
301
            fields = workflow_ab.workflow_integrity_fields(workflow_ab.Subject("git", "git", "Git", Path(shutil.which("git"))),
302
                workflow_ab.WORKFLOWS["bugfix_test_loop"], repo, [])
303
            self.assertTrue(fields["workflow_integrity_passed"])
304
            self.assertEqual(fields["workflow_integrity_evidence_class"], "subject_binary_committed_payload")
305
            self.assertEqual(fields["workflow_subject_binary_sha256"], fields["git_head_instrument_sha256"])
306
307
    def test_task_evidence_resolves_instrument_once(self):
308
        with tempfile.TemporaryDirectory() as tmp:
309
            repo = Path(tmp)
310
            workflow_ab.make_wide_config_fixture(repo)
311
            finish_configs(repo)
312
            actual = workflow_oracles.resolve_git_instrument
313
            with mock.patch.object(workflow_oracles, "resolve_git_instrument", wraps=actual) as resolver:
314
                evidence = workflow_oracles.task_evidence("wide_config_refactor", repo, "git", shutil.which("git"))
315
            self.assertEqual(resolver.call_count, 1)
316
            self.assertFalse(evidence["workflow_integrity_passed"])
317
318
    def test_bound_subject_hashes_are_reused_without_binary_reads(self):
319
        for kind, metadata, expected in (
320
            ("oak", {"oak_export_instrument_sha256": "a" * 64}, "subject_produced_committed_tree"),
321
            ("git", {"workflow_subject_binary_sha256": "a" * 64, "git_head_instrument_sha256": "a" * 64}, "subject_binary_committed_tree"),
322
        ):
323
            with mock.patch.object(Path, "read_bytes", side_effect=AssertionError("redundant binary read")), \
324
                 mock.patch.object(workflow_oracles.shutil, "which", side_effect=AssertionError("redundant subject lookup")):
325
                result = workflow_oracles.reader_evidence(kind, "/subject", metadata)
326
            self.assertEqual(result["workflow_integrity_evidence_class"], expected)
327
328
    def test_killed_git_instrument_is_unmeasured(self):
329
        with mock.patch.object(workflow_oracles, "resolve_git_instrument", return_value=("/git", {})), \
330
             mock.patch.object(workflow_oracles.subprocess, "run", return_value=subprocess.CompletedProcess([], -9, b"", b"")):
331
            self.assertEqual(workflow_oracles.validate_git_head("wide_config_refactor", Path("/tmp")), "git_head_oracle_unavailable")
332
            self.assertEqual(workflow_oracles.git_head_text(Path("/tmp"), "x"), (None, "git_head_oracle_unavailable"))
333
334
    def test_path_instrument_and_missing_instrument_are_not_subject_failures(self):
335
        real_git = shutil.which("git")
336
        with tempfile.TemporaryDirectory() as tmp:
337
            repo = Path(tmp)
338
            workflow_ab.make_wide_config_fixture(repo)
339
            finish_configs(repo)
340
            with mock.patch.object(workflow_oracles.shutil, "which", return_value=None) as lookup:
341
                evidence = workflow_oracles.task_evidence("wide_config_refactor", repo, "git")
342
                lookup.assert_called_with("git")
343
                self.assertIsNone(evidence["workflow_integrity_passed"])
344
                self.assertEqual(evidence["workflow_integrity_unmeasured_reason"], "git_head_oracle_unavailable")
345
                self.assertIsNone(evidence["git_head_instrument_sha256"])
346
                self.assertEqual(evidence["workflow_integrity_evidence_class"], "worktree_payload")
347
            binary, metadata = workflow_oracles.resolve_git_instrument(real_git)
348
            self.assertTrue(Path(binary).is_absolute())
349
            self.assertEqual(len(metadata["git_head_instrument_sha256"]), 64)
350
            self.assertTrue(metadata["git_head_instrument_version"])
351
352
    @unittest.skipUnless(shutil.which("oak"), "Oak installed binary required")
353
    def test_correct_oak_worktree_cannot_hide_partial_checkpoint(self):
354
        binary = shutil.which("oak")
355
        with tempfile.TemporaryDirectory() as tmp:
356
            repo = Path(tmp)
357
            workflow_ab.make_wide_config_fixture(repo)
358
            def oak(*args):
359
                process = subprocess.run([binary, *args], cwd=repo, env=workflow_ab.base_env(), capture_output=True, timeout=60)
360
                self.assertEqual(process.returncode, 0, process.stderr)
361
            oak("init"); oak("commit", "--no-verify")
362
            finish_configs(repo)
363
            oak("commit", "--no-verify", "services/service-0000/config.toml")
364
            self.assertIsNone(workflow_oracles.validate("wide_config_refactor", repo))
365
            evidence = workflow_oracles.task_evidence("wide_config_refactor", repo, "oak", binary)
366
            self.assertIs(evidence["workflow_integrity_passed"], False, evidence)
367
            self.assertEqual(evidence["workflow_integrity_evidence_class"], "subject_produced_committed_tree_attempt")
368
            self.assertIn("git_head_config_payload_mismatch", evidence["workflow_integrity_failure_reason"])
369
            oak("commit", "--no-verify")
370
            verified = workflow_oracles.task_evidence("wide_config_refactor", repo, "oak", binary)
371
            self.assertIs(verified["workflow_integrity_passed"], True, verified)
372
            self.assertNotEqual(evidence["oak_export_head"], verified["oak_export_head"])
373
374
    def git(self, repo, *args):
375
        result = subprocess.run(
376
            [shutil.which("git"), *args], cwd=repo, env=workflow_ab.base_env(),
377
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
378
        )
379
        self.assertEqual(result.returncode, 0, result.stderr)
380
381
    def test_correct_worktree_cannot_hide_partial_commit(self):
382
        for name, maker, finish, staged in (
383
            ("wide_config_refactor", workflow_ab.make_wide_config_fixture, finish_configs,
384
             "services/service-0000/config.toml"),
385
            ("large_asset_manifest", workflow_ab.make_large_asset_fixture, finish_assets, "manifest.json"),
386
        ):
387
            with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
388
                repo = Path(tmp)
389
                maker(repo)
390
                self.git(repo, "init")
391
                self.git(repo, "add", ".")
392
                if name == "large_asset_manifest":
393
                    self.git(repo, "rm", "--cached", "assets/asset-03.bin")
394
                self.git(repo, "commit", "-m", "initial")
395
                finish(repo)
396
                self.git(repo, "add", staged)
397
                self.git(repo, "commit", "-m", "partial task")
398
                self.assertIsNone(workflow_oracles.validate(name, repo))
399
                self.assertIsNotNone(workflow_oracles.validate_git_head(name, repo))
400
                subject = workflow_ab.Subject(name="git", kind="git", label="Git", bin=Path(shutil.which("git")))
401
                fields = workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS[name], repo, [])
402
                self.assertFalse(fields["correctness_passed"])
403
                proc = agent_workflow.run_command(agent_workflow.test_command_for(workflow_ab.WORKFLOWS[name], subject), repo)
404
                self.assertEqual(proc.returncode, 1, proc.stderr)
405
                self.assertIn("git_head_", proc.stdout)
406
                self.git(repo, "add", ".")
407
                self.git(repo, "commit", "-m", "complete task")
408
                self.assertIsNone(workflow_oracles.validate_git_head(name, repo))
409
                proc = agent_workflow.run_command(agent_workflow.test_command_for(workflow_ab.WORKFLOWS[name], subject), repo)
410
                self.assertEqual(proc.returncode, 0, proc.stderr)
411
                observed = json.loads(proc.stdout)
412
                self.assertEqual(observed["workflow_integrity_evidence_class"], "subject_binary_committed_tree")
413
                self.assertEqual(observed["workflow_subject_binary_sha256"], observed["git_head_instrument_sha256"])
414
415
416
class RealSubjectWorkflowOracleTest(unittest.TestCase):
417
    def test_mock_agent_config_and_asset_tasks_on_real_vcs(self):
418
        for kind in ("git", "oak"):
419
            binary = shutil.which(kind)
420
            if binary is None:
421
                with self.subTest(kind=kind):
422
                    self.skipTest(f"{kind} not installed")
423
                continue
424
            for name in ("wide_config_refactor", "large_asset_manifest"):
425
                with self.subTest(kind=kind, name=name), tempfile.TemporaryDirectory() as tmp:
426
                    root = Path(tmp)
427
                    subject = workflow_ab.Subject(name=kind, kind=kind, label=kind, bin=Path(binary))
428
                    workflow = workflow_ab.WORKFLOWS[name]
429
                    fixture = workflow_ab.prepare_fixture(root, workflow)
430
                    row = agent_workflow.run_one(
431
                        agent=agent_workflow.AgentConfig(name="mock", adapter="deterministic_mock", provider="local",
432
                                                        bin=None, model=None, version="test"),
433
                        subject=subject, workflow=workflow, fixture=fixture, run_index=0,
434
                        run_root=root / "runs", artifacts_root=root / "artifacts", timeout_seconds=300,
435
                        track="agent-default", agent_environment="minimal", instruction_level="full-docs",
436
                        bench_id="oracle-integration", timestamp_utc="2026-09-10T00:00:00Z", source={}, host_meta={},
437
                    )
438
                    self.assertTrue(row["success"], row.get("failure_reason"))
439
                    self.assertEqual(agent_workflow.validate_row(row), [])
440
                    self.assertIn(workflow_oracles.VERSION, row["scenario_version"])
441
                    self.assertEqual(row["oracle_version"], workflow_oracles.VERSION)
442
                    self.assertTrue(row["workflow_integrity_passed"])
443
                    self.assertIn("git_head_instrument_sha256", row)
444
                    from oakbench.populations import comparison_identity
445
                    identity = comparison_identity(row)
446
                    self.assertEqual(identity["oracle_version"], workflow_oracles.VERSION)
447
                    self.assertEqual(identity["env_isolation_version"], agent_workflow.ENV_ISOLATION_VERSION)