Viewing files at commit d3edd8c5e361Commit detailsView latest

Browse files
tests/test_agent_campaign.py 515 lines · 27.9 KB · python
1
"""Campaign contracts and transaction boundaries; no paid agent invocations."""
2
from __future__ import annotations
3
import copy
4
import json
5
import os
6
from pathlib import Path
7
import sys
8
import tempfile
9
import unittest
10
from unittest import mock
11
12
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
13
from oakbench import agent_campaign as campaign
14
15
16
class CampaignTest(unittest.TestCase):
17
    def setUp(self):
18
        self.tmp = tempfile.TemporaryDirectory()
19
        self.root = Path(self.tmp.name).resolve()
20
        self.manifest_path = self.root / "manifest.json"
21
        self.raw = json.loads((campaign.ROOT / "config/agent-campaign-smoke.json").read_text())
22
        self.raw.update(agents_config=str(campaign.ROOT / "config/agents.toml"),
23
                        subjects_config=str(campaign.ROOT / "config/subjects.toml"))
24
        self.write_manifest()
25
26
    def tearDown(self):
27
        self.tmp.cleanup()
28
29
    def write_manifest(self):
30
        self.manifest_path.write_text(json.dumps(self.raw))
31
32
    def manifest(self):
33
        self.write_manifest()
34
        return campaign.load_manifest(self.manifest_path)
35
36
    def plan(self):
37
        with mock.patch.object(campaign, "resolve_runtime", return_value=({}, {}, {})):
38
            return campaign.create_plan(self.manifest_path, self.root / "campaign")
39
40
    def test_schedule_determinism_balance_and_identity(self):
41
        first = campaign.schedule(self.manifest())
42
        self.assertEqual(first, campaign.schedule(self.manifest()))
43
        self.assertEqual(len({trial["trial_id"] for trial in first}), len(first))
44
        self.assertEqual([x["subject"] for x in first[:2]], list(reversed([x["subject"] for x in first[2:]])))
45
        self.assertEqual(first[0]["block_id"], first[1]["block_id"])
46
        self.assertNotEqual(first[0]["block_id"], first[2]["block_id"])
47
        self.raw["instruction_levels"] = ["full-docs", "zero-shot"]
48
        self.raw["max_trials"] = 8
49
        self.assertEqual(len(campaign.schedule(self.manifest())), 8)
50
51
    def test_invalid_limits_unknown_fields_duplicates_and_paths(self):
52
        for key, value in (("schema_version", True), ("repetitions", True), ("repetitions", 0),
53
                           ("max_trials", -1), ("max_wall_seconds", float("nan")),
54
                           ("trial_timeout_seconds", float("inf")), ("seed", True),
55
                           ("subjects", ["git", "git"]), ("subjects", ["../escape"]),
56
                           ("workflows", ["history_archaeology"]), ("typo", 1)):
57
            with self.subTest(key=key, value=value):
58
                raw = dict(self.raw)
59
                raw[key] = value
60
                self.manifest_path.write_text(json.dumps(raw))
61
                with self.assertRaises(campaign.CampaignError):
62
                    campaign.load_manifest(self.manifest_path)
63
        self.manifest_path.write_text('{"schema_version":1,"schema_version":1}')
64
        with self.assertRaises(campaign.CampaignError):
65
            campaign.load_manifest(self.manifest_path)
66
67
    def test_huge_schedule_rejected_before_materialization(self):
68
        self.raw["repetitions"] = 10 ** 30
69
        with self.assertRaises(campaign.CampaignError):
70
            campaign.schedule(self.manifest())
71
72
    def test_freeze_collision_and_tamper_rejected(self):
73
        plan = self.plan()
74
        directory = self.root / "campaign"
75
        self.assertEqual(campaign.load_plan(directory), plan)
76
        with mock.patch.object(campaign, "resolve_runtime", return_value=({}, {}, {})):
77
            with self.assertRaises(campaign.CampaignError):
78
                campaign.create_plan(self.manifest_path, directory)
79
        plan["schedule"][0]["subject"] = "wrong"
80
        (directory / "plan.json").write_text(json.dumps(plan))
81
        with self.assertRaises(campaign.CampaignError):
82
            campaign.load_plan(directory)
83
84
    def test_interrupted_trial_is_never_pending_and_reserves_budget(self):
85
        plan = self.plan()
86
        directory = self.root / "campaign"
87
        trial = plan["schedule"][0]["trial_id"]
88
        campaign.append_event(directory, plan, {"event": "started", "trial_id": trial, "timeout_seconds": 60})
89
        status = campaign.campaign_status(plan, campaign.load_events(directory, plan))
90
        self.assertEqual(status["interrupted_trial_ids"], [trial])
91
        self.assertNotIn(trial, status["pending_trial_ids"])
92
        self.assertEqual(status["charged_episode_seconds"], 60)
93
94
    def test_torn_duplicate_unknown_nonfinite_and_bad_duration_journal_fail_closed(self):
95
        plan = self.plan()
96
        directory = self.root / "campaign"
97
        trial = plan["schedule"][0]["trial_id"]
98
        start = {"event": "started", "trial_id": trial, "plan_sha256": plan["plan_sha256"], "timeout_seconds": 60}
99
        variants = [b'{"event":', b'[]\n', (json.dumps(start) + '\n') .encode() * 2,
100
                    (json.dumps({**start, "trial_id": "other"}) + '\n').encode(),
101
                    (json.dumps({**start, "timeout_seconds": float("nan")}) + '\n').encode()]
102
        for wall in (True, -1, float("inf")):
103
            finished = {"event": "finished", "trial_id": trial, "plan_sha256": plan["plan_sha256"],
104
                        "outcome": "error", "error": "test", "wall_seconds": wall}
105
            variants.append((json.dumps(start) + '\n' + json.dumps(finished) + '\n').encode())
106
        for content in variants:
107
            with self.subTest(content=content):
108
                (directory / "events.jsonl").write_bytes(content)
109
                with self.assertRaises(campaign.CampaignError):
110
                    campaign.load_events(directory, plan)
111
                self.assertEqual((directory / "events.jsonl").read_bytes(), content)
112
113
    def test_torn_new_start_after_old_interruption_reserves_next_trial(self):
114
        plan = self.plan()
115
        directory = self.root / "campaign"
116
        first, second, third = plan["schedule"][:3]
117
        for trial in (first, second):
118
            campaign.append_event(directory, plan, {"event": "started", "trial_id": trial["trial_id"], "timeout_seconds": 60})
119
        campaign.append_event(directory, plan, {"event": "finished", "trial_id": second["trial_id"],
120
            "outcome": "error", "wall_seconds": 1})
121
        with (directory / "events.jsonl").open("ab") as stream:
122
            stream.write(b'{"event":"started"')
123
        events = campaign.load_events(directory, plan, tolerate_torn=True)
124
        self.assertEqual(events[-2]["trial_id"], third["trial_id"])
125
        self.assertTrue(events[-2]["uncertain_torn_start"])
126
        state = campaign.campaign_status(plan, events)
127
        self.assertNotIn(third["trial_id"], state["pending_trial_ids"])
128
129
    def test_system_tmp_alias_is_accepted_but_arbitrary_symlinks_are_not(self):
130
        self.assertEqual(campaign.safe_path(Path("/tmp") / "campaign-safe-path-check"),
131
                         (Path("/tmp") / "campaign-safe-path-check").resolve())
132
133
    def test_terminal_row_identity_cannot_be_rebound(self):
134
        from agent_metrics_schema import example_row
135
        plan = self.plan()
136
        directory = self.root / "campaign"
137
        trial = plan["schedule"][0]
138
        row = example_row()
139
        row.update(task_id=trial["trial_id"], bench_id=plan["campaign_id"], subject=trial["subject"],
140
                   scenario=trial["workflow"], run=trial["repetition"], outcome="pass")
141
        row["agent"].update(name=trial["agent"], model=trial["model"],
142
                             instruction_level=trial["instruction_level"], environment="minimal")
143
        campaign.validate_trial_row(row, trial, plan)
144
        for key, value in (("subject", "wrong"), ("scenario", "wrong"), ("run", 999), ("bench_id", "other")):
145
            wrong = copy.deepcopy(row)
146
            wrong[key] = value
147
            with self.subTest(key=key), self.assertRaises(campaign.CampaignError):
148
                campaign.validate_trial_row(wrong, trial, plan)
149
        for key in ("name", "model", "instruction_level"):
150
            wrong = copy.deepcopy(row)
151
            wrong["agent"][key] = "wrong"
152
            with self.subTest(agent_key=key), self.assertRaises(campaign.CampaignError):
153
                campaign.validate_trial_row(wrong, trial, plan)
154
        campaign.append_event(directory, plan, {"event": "started", "trial_id": trial["trial_id"], "timeout_seconds": 60})
155
        row["subject"] = "wrong"
156
        campaign.append_event(directory, plan, {"event": "finished", "trial_id": trial["trial_id"],
157
            "outcome": "pass", "row": row, "wall_seconds": 1})
158
        with self.assertRaises(campaign.CampaignError):
159
            campaign.load_events(directory, plan)
160
161
    def test_symlink_destinations_and_lock_rejected(self):
162
        alias = self.root / "alias"
163
        alias.symlink_to(self.root, target_is_directory=True)
164
        with self.assertRaises(campaign.CampaignError):
165
            campaign.safe_path(alias / "campaign")
166
        plan = self.plan()
167
        directory = self.root / "campaign"
168
        for name in ("events.jsonl", ".run.lock", "trials"):
169
            path = directory / name
170
            path.symlink_to(self.root / "outside")
171
            with self.assertRaises(campaign.CampaignError):
172
                campaign.safe_path(path)
173
            path.unlink()
174
175
    def test_exclusive_runner_ownership(self):
176
        self.plan()
177
        with campaign.campaign_lock(self.root / "campaign"):
178
            with self.assertRaises(campaign.CampaignError):
179
                with campaign.campaign_lock(self.root / "campaign"):
180
                    pass
181
182
    def test_preflight_drift_real_optin_and_attestation(self):
183
        plan = self.plan()
184
        directory = self.root / "campaign"
185
        with mock.patch.object(campaign, "resolve_runtime", return_value=({}, {}, {"changed": True})):
186
            with self.assertRaises(campaign.CampaignError):
187
                campaign.preflight(plan, directory)
188
        from agent_workflow import AgentConfig
189
        real = AgentConfig("real", "codex_exec_json", "openai", Path("/binary"), "pinned", "1")
190
        with mock.patch.object(campaign, "resolve_runtime", return_value=({"real": real}, {}, {})):
191
            with self.assertRaises(campaign.CampaignError):
192
                campaign.preflight(plan, directory)
193
            with self.assertRaises(campaign.CampaignError):
194
                campaign.preflight(plan, directory, enable_real=True)
195
            plan["manifest"]["isolation_attestation"] = {
196
                "worker_root": str(self.root), "unrestricted_cli_acknowledged": True}
197
            self.assertEqual(campaign.preflight(plan, directory, enable_real=True)[0]["real"], real)
198
199
    def test_lock_provenance_drift_names_the_key(self):
200
        plan = self.plan()
201
        plan["provenance"] = {"measurement_lock_path": "/old/lock"}
202
        with mock.patch.object(campaign, "resolve_runtime", return_value=({}, {}, {"measurement_lock_path": "/new/lock"})):
203
            with self.assertRaisesRegex(campaign.CampaignError, "measurement_lock_path"):
204
                campaign.preflight(plan, self.root / "campaign")
205
206
    def test_fixture_cache_reuses_generation_and_rejects_mutated_payload(self):
207
        import workflow_ab
208
        from dataclasses import replace
209
        counter = mock.Mock(side_effect=workflow_ab.WORKFLOWS["wide_config_refactor"].make_fixture)
210
        workflow = replace(workflow_ab.WORKFLOWS["wide_config_refactor"], make_fixture=counter)
211
        plan = self.plan()
212
        directory = self.root / "campaign"
213
        fixture = campaign.cached_fixture(directory, plan, workflow)
214
        self.assertEqual(fixture, directory / "fixtures" / workflow.name)
215
        self.assertEqual(campaign.cached_fixture(directory, plan, workflow), fixture)
216
        self.assertEqual(counter.call_count, 1)
217
        payload = next(fixture.glob("services/*/config.toml"))
218
        payload.chmod(0o600)
219
        payload.write_text("corrupt fixture")
220
        with self.assertRaisesRegex(campaign.CampaignError, "fixture.*drift"):
221
            campaign.cached_fixture(directory, plan, workflow)
222
        self.assertEqual(counter.call_count, 1)
223
224
    def test_cache_permissions_allow_rebuild_and_plain_rmtree(self):
225
        import workflow_ab
226
        import shutil
227
        plan = self.plan()
228
        directory = self.root / "campaign"
229
        workflow = workflow_ab.WORKFLOWS["wide_config_refactor"]
230
        fixture = campaign.cached_fixture(directory, plan, workflow)
231
        self.assertTrue(fixture.stat().st_mode & 0o200)
232
        self.assertTrue(all(path.stat().st_mode & 0o200 for path in fixture.rglob("*") if path.is_dir()))
233
        self.assertTrue(all(not path.stat().st_mode & 0o222 for path in fixture.rglob("*") if path.is_file()))
234
        marker = fixture / ".fixture-ready"
235
        marker.chmod(0o600)
236
        marker.write_text("old fixture version")
237
        rebuilt = workflow_ab.prepare_fixture(directory, workflow)
238
        self.assertEqual((rebuilt / ".fixture-ready").read_text().strip(), workflow_ab.FIXTURE_VERSION)
239
        campaign.fixture_permissions(rebuilt, writable=False)
240
        shutil.rmtree(directory)
241
        self.assertFalse(directory.exists())
242
243
    def test_permissions_do_not_require_chmod_nofollow_support(self):
244
        fixture = self.root / "permission-fixture"
245
        fixture.mkdir()
246
        (fixture / "file").write_text("content")
247
        chmod = os.chmod
248
        def portable(path, mode, **kwargs):
249
            if kwargs.get("follow_symlinks") is False:
250
                raise NotImplementedError("chmod nofollow unavailable on this platform")
251
            return chmod(path, mode, **kwargs)
252
        with mock.patch.object(os, "chmod", side_effect=portable):
253
            campaign.fixture_permissions(fixture, writable=False)
254
            campaign.fixture_permissions(fixture, writable=True)
255
256
    def test_fifo_cache_is_a_named_campaign_error(self):
257
        import workflow_ab
258
        plan = self.plan()
259
        directory = self.root / "campaign"
260
        fixture = campaign.cached_fixture(directory, plan, workflow_ab.WORKFLOWS["wide_config_refactor"])
261
        fixture.chmod(0o700)
262
        os.mkfifo(fixture / "unexpected-pipe")
263
        with self.assertRaisesRegex(campaign.CampaignError, "wide_config_refactor"):
264
            campaign.cached_fixture(directory, plan, workflow_ab.WORKFLOWS["wide_config_refactor"])
265
266
    def test_corrupted_cache_aborts_before_any_trial_admission(self):
267
        import workflow_ab
268
        plan = self.plan()
269
        directory = self.root / "campaign"
270
        fixture = campaign.cached_fixture(directory, plan, workflow_ab.WORKFLOWS["wide_config_refactor"])
271
        payload = next(fixture.glob("services/*/config.toml"))
272
        payload.chmod(0o600)
273
        payload.write_text("bad cached baseline")
274
        with mock.patch.object(campaign, "preflight", return_value=({}, {})), mock.patch("agent_workflow.run_one") as run:
275
            with self.assertRaisesRegex(campaign.CampaignError, "fixture.*drift"):
276
                campaign.run_campaign(directory, resume=True)
277
        run.assert_not_called()
278
        self.assertFalse((directory / "events.jsonl").exists())
279
        self.assertEqual(len(campaign.campaign_status(plan, [])['pending_trial_ids']), 4)
280
281
    def test_generation_crash_and_incomplete_cache_preserve_all_pending_trials(self):
282
        import workflow_ab
283
        plan = self.plan()
284
        directory = self.root / "campaign"
285
        def crash(root, workflow):
286
            path = root / "fixtures" / workflow.name
287
            path.mkdir(parents=True)
288
            (path / "partial").write_text("interrupted generation")
289
            raise KeyboardInterrupt("crash before fixture integrity record")
290
        with mock.patch.object(campaign, "preflight", return_value=({}, {})), mock.patch.object(workflow_ab, "prepare_fixture", side_effect=crash):
291
            with self.assertRaises(KeyboardInterrupt):
292
                campaign.run_campaign(directory)
293
        self.assertFalse((directory / "events.jsonl").exists())
294
        with mock.patch.object(campaign, "preflight", return_value=({}, {})):
295
            with self.assertRaisesRegex(campaign.CampaignError, "fixture cache is incomplete"):
296
                campaign.run_campaign(directory, resume=True)
297
        self.assertFalse((directory / "events.jsonl").exists())
298
        self.assertEqual(len(campaign.campaign_status(plan, [])['pending_trial_ids']), 4)
299
300
    def test_real_git_mock_cache_poisoning_invalidates_attempt_and_preserves_next(self):
301
        def mutate(directory):
302
            payload = next((directory / "fixtures" / "wide_config_refactor").glob("services/*/config.toml"))
303
            payload.chmod(0o600)  # Same-uid processes can defeat read-only files.
304
            payload.write_text("poisoned shared baseline")
305
        self.assert_midtrial_fixture_invalidation(mutate)
306
307
    def test_real_git_mock_deleted_cache_record_invalidates_attempt(self):
308
        self.assert_midtrial_fixture_invalidation(lambda directory:
309
            (directory / "fixture-integrity/wide_config_refactor.json").unlink())
310
311
    def test_real_git_mock_corrupt_cache_record_invalidates_attempt(self):
312
        self.assert_midtrial_fixture_invalidation(lambda directory:
313
            (directory / "fixture-integrity/wide_config_refactor.json").write_text("invalid JSON"))
314
315
    def assert_midtrial_fixture_invalidation(self, mutate):
316
        import agent_workflow
317
        import workflow_ab
318
        import shutil
319
        if not shutil.which("git"):
320
            self.skipTest("Git required")
321
        self.raw.update(subjects=["git"], repetitions=2)
322
        self.write_manifest()
323
        plan = self.plan()
324
        directory = self.root / "campaign"
325
        agent = agent_workflow.AgentConfig("mock", "deterministic_mock", "local", None, None, "1")
326
        subject = workflow_ab.Subject("git", "git", "Git", Path(shutil.which("git")))
327
        original = agent_workflow.run_mock_agent
328
        def poison(*args, **kwargs):
329
            execution = original(*args, **kwargs)
330
            mutate(directory)
331
            return execution
332
        with mock.patch.object(campaign, "preflight", return_value=({"mock": agent}, {"git": subject})), \
333
             mock.patch.object(agent_workflow, "run_mock_agent", side_effect=poison) as execute:
334
            state = campaign.run_campaign(directory)
335
        self.assertEqual(execute.call_count, 1)
336
        self.assertEqual(len(state["pending_trial_ids"]), 1)
337
        self.assertEqual(state["stopped_reason"], "fixture_integrity_drift")
338
        events = campaign.load_events(directory, plan)
339
        self.assertEqual(len(events), 2)
340
        self.assertTrue(events[-1]["measurement_invalidated"])
341
        self.assertTrue(events[-1]["rejected_row"]["success"])
342
        self.assertNotIn("row", events[-1])
343
        self.assertEqual((directory / "rows.jsonl").read_bytes(), b"")
344
        from oakbench.campaign_report import report_campaign
345
        report = report_campaign(plan, events)
346
        self.assertEqual(report["counts"]["skipped"], 1)
347
        self.assertEqual(report["counts"]["unstarted"], 1)
348
349
    def test_copied_fixture_race_is_rejected_before_real_agent_launch(self):
350
        import agent_workflow
351
        import workflow_ab
352
        import shutil
353
        if not shutil.which("git"):
354
            self.skipTest("Git required")
355
        self.raw.update(subjects=["git"], repetitions=2)
356
        self.write_manifest()
357
        plan = self.plan()
358
        directory = self.root / "campaign"
359
        agent = agent_workflow.AgentConfig("mock", "deterministic_mock", "local", None, None, "1")
360
        subject = workflow_ab.Subject("git", "git", "Git", Path(shutil.which("git")))
361
        original = agent_workflow.copy_fixture
362
        def corrupt_copy(src, dest):
363
            original(src, dest)
364
            payload = next(dest.glob("services/*/config.toml"))
365
            payload.chmod(0o600)
366
            payload.write_text("changed during copy")
367
        with mock.patch.object(campaign, "preflight", return_value=({"mock": agent}, {"git": subject})), \
368
             mock.patch.object(agent_workflow, "copy_fixture", side_effect=corrupt_copy), \
369
             mock.patch.object(agent_workflow, "run_mock_agent") as execute:
370
            with self.assertRaisesRegex(campaign.CampaignError, "copied fixture identity drift"):
371
                campaign.run_campaign(directory)
372
        self.assertEqual(execute.call_count, 0)
373
        self.assertFalse((directory / "events.jsonl").exists())
374
        self.assertEqual(len(campaign.campaign_status(plan, [])["pending_trial_ids"]), 2)
375
        with mock.patch.object(campaign, "preflight", return_value=({"mock": agent}, {"git": subject})):
376
            state = campaign.run_campaign(directory, resume=True)
377
        self.assertEqual(state["finished"], 2)
378
        self.assertEqual(state["pending_trial_ids"], [])
379
        self.assertTrue(all(event["row"]["success"] for event in campaign.load_events(directory, plan) if event["event"] == "finished"))
380
381
    def test_journal_fixture_binding_rejects_missing_or_conflicting_attempt_hashes(self):
382
        plan = self.plan()
383
        trial = plan["schedule"][0]
384
        record = {"manifest": {"config": {"sha256": "a" * 64}}}
385
        fingerprint = campaign.digest(record["manifest"])
386
        start = {"event": "started", "trial_id": trial["trial_id"], "fixture_manifest_sha256": fingerprint}
387
        finish = {"event": "finished", "trial_id": trial["trial_id"], "fixture_manifest_sha256": fingerprint}
388
        campaign.verify_journal_fixture_binding(trial["workflow"], record, plan, [start, finish])
389
        for value in (None, "b" * 64):
390
            for index in (0, 1):
391
                events = [dict(start), dict(finish)]
392
                events[index]["fixture_manifest_sha256"] = value
393
                with self.assertRaisesRegex(campaign.CampaignError, "journal fixture binding"):
394
                    campaign.verify_journal_fixture_binding(trial["workflow"], record, plan, events)
395
396
    def test_completed_infrastructure_error_not_retried_on_resume(self):
397
        self.plan()
398
        directory = self.root / "campaign"
399
        import agent_workflow
400
        import workflow_ab
401
        from oakbench.runlock import LockInfo
402
        from contextlib import contextmanager
403
        @contextmanager
404
        def locked(label):
405
            yield LockInfo(0, True)
406
        agent = agent_workflow.AgentConfig("mock", "deterministic_mock", "local", None, None, "1")
407
        subject = workflow_ab.Subject("git", "git", "Git", Path("/git"))
408
        with mock.patch.object(campaign, "preflight", return_value=({"mock": agent}, {"git": subject, "oak_installed": subject})), \
409
             mock.patch("oakbench.runlock.measurement_lock", locked), \
410
             mock.patch.object(agent_workflow, "run_one", side_effect=RuntimeError("fixture setup failed")) as run:
411
            state = campaign.run_campaign(directory, limit=1)
412
            self.assertEqual(state["finished"], 1)
413
            campaign.run_campaign(directory, resume=True, limit=1)
414
            self.assertEqual(run.call_count, 2)
415
            events = campaign.load_events(directory, campaign.load_plan(directory))
416
            self.assertEqual(len({event["trial_id"] for event in events if event["event"] == "started"}), 2)
417
            self.assertEqual([event["outcome"] for event in events if event["event"] == "finished"], ["error", "error"])
418
419
    def test_staging_move_crash_after_durable_start_is_not_pending(self):
420
        from contextlib import contextmanager
421
        from oakbench.runlock import LockInfo
422
        self.plan()
423
        directory = self.root / "campaign"
424
        @contextmanager
425
        def locked(label):
426
            yield LockInfo(0, True)
427
        original = Path.rename
428
        def crash(path, target):
429
            if target.parent.name == "trials":
430
                raise KeyboardInterrupt("simulated process crash")
431
            return original(path, target)
432
        with mock.patch.object(campaign, "preflight", return_value=({}, {})), \
433
             mock.patch("oakbench.runlock.measurement_lock", locked), \
434
             mock.patch.object(Path, "rename", crash):
435
            with self.assertRaises(KeyboardInterrupt):
436
                campaign.run_campaign(directory)
437
        plan = campaign.load_plan(directory)
438
        events = campaign.load_events(directory, plan)
439
        self.assertEqual(len(events), 1)
440
        state = campaign.campaign_status(plan, events)
441
        self.assertEqual(state["interrupted_trial_ids"], [plan["schedule"][0]["trial_id"]])
442
        self.assertNotIn(plan["schedule"][0]["trial_id"], state["pending_trial_ids"])
443
444
    def test_schema_rejected_row_remains_in_terminal_evidence(self):
445
        import agent_workflow
446
        from contextlib import contextmanager
447
        from oakbench.runlock import LockInfo
448
        plan = self.plan()
449
        directory = self.root / "campaign"
450
        @contextmanager
451
        def locked(label):
452
            yield LockInfo(0, True)
453
        subjects = {trial["subject"]: object() for trial in plan["schedule"]}
454
        with mock.patch.object(campaign, "preflight", return_value=({"mock": object()}, subjects)), \
455
             mock.patch("oakbench.runlock.measurement_lock", locked), \
456
             mock.patch.object(agent_workflow, "run_one", return_value={"invalid": "original evidence"}):
457
            campaign.run_campaign(directory, limit=1)
458
        events = campaign.load_events(directory, plan)
459
        self.assertEqual(events[1]["outcome"], "error")
460
        self.assertEqual(events[1]["rejected_row"]["invalid"], "original evidence")
461
        self.assertEqual((directory / "rows.jsonl").read_bytes(), b"")
462
463
    def test_noop_resume_repairs_missing_or_stale_rows_from_finished_journal(self):
464
        from agent_metrics_schema import example_row
465
        self.raw.update(subjects=["git"], repetitions=1)
466
        self.write_manifest()
467
        plan = self.plan()
468
        directory = self.root / "campaign"
469
        trial = plan["schedule"][0]
470
        import workflow_ab
471
        campaign.cached_fixture(directory, plan, workflow_ab.WORKFLOWS[trial["workflow"]])
472
        record = campaign.read_json(directory / "fixture-integrity" / (trial["workflow"] + ".json"))
473
        fingerprint = campaign.digest(record["manifest"])
474
        row = example_row()
475
        row.update(task_id=trial["trial_id"], bench_id=plan["campaign_id"], subject=trial["subject"],
476
                   scenario=trial["workflow"], run=trial["repetition"], outcome="pass")
477
        row["agent"].update(name=trial["agent"], model=trial["model"],
478
                             instruction_level=trial["instruction_level"], environment="minimal")
479
        campaign.append_event(directory, plan, {"event": "started", "trial_id": trial["trial_id"], "timeout_seconds": 60, "fixture_manifest_sha256": fingerprint})
480
        campaign.append_event(directory, plan, {"event": "finished", "trial_id": trial["trial_id"],
481
            "outcome": "pass", "row": row, "wall_seconds": 1, "fixture_manifest_sha256": fingerprint})
482
        journal = (directory / "events.jsonl").read_bytes()
483
        for state in ("missing", "stale"):
484
            with self.subTest(state=state):
485
                if state == "stale":
486
                    (directory / "rows.jsonl").write_bytes(b"stale output")
487
                with mock.patch.object(campaign, "preflight", return_value=({}, {})), \
488
                     mock.patch("agent_workflow.run_one") as run:
489
                    status = campaign.run_campaign(directory, resume=True)
490
                    run.assert_not_called()
491
                self.assertEqual(status["finished"], 1)
492
                self.assertEqual((directory / "rows.jsonl").read_bytes(), campaign.canonical(row) + b"\n")
493
                self.assertEqual((directory / "events.jsonl").read_bytes(), journal)
494
495
    def test_disabled_measurement_lock_starts_nothing(self):
496
        self.plan()
497
        directory = self.root / "campaign"
498
        with mock.patch.object(campaign, "preflight", return_value=({}, {})), \
499
             mock.patch.dict(os.environ, {"OAK_BENCH_LOCK": "off"}):
500
            with self.assertRaises(campaign.CampaignError):
501
                campaign.run_campaign(directory)
502
        self.assertFalse((directory / "events.jsonl").exists())
503
504
    def test_artifact_hashes_exclude_links(self):
505
        trial = self.root / "trial"
506
        (trial / "artifacts").mkdir(parents=True)
507
        (trial / "artifacts/prompt.txt").write_text("fixed prompt")
508
        (trial / "artifacts/escape").symlink_to(self.manifest_path)
509
        hashes = campaign.artifact_hashes(trial)
510
        self.assertEqual(set(hashes), {"artifacts/prompt.txt"})
511
        self.assertEqual(hashes["artifacts/prompt.txt"], campaign.file_hash(trial / "artifacts/prompt.txt"))
512
513
514
if __name__ == "__main__":
515
    unittest.main()