| 1 | """Campaign contracts and transaction boundaries; no paid agent invocations."""
|
| 2 | from __future__ import annotations
|
| 3 | import copy
|
| 4 | import json
|
| 5 | import os
|
| 6 | from pathlib import Path
|
| 7 | import sys
|
| 8 | import tempfile
|
| 9 | import unittest
|
| 10 | from unittest import mock
|
| 11 |
|
| 12 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 13 | from oakbench import agent_campaign as campaign
|
| 14 |
|
| 15 |
|
| 16 | class CampaignTest(unittest.TestCase):
|
| 17 | def setUp(self):
|
| 18 | self.tmp = tempfile.TemporaryDirectory()
|
| 19 | self.root = Path(self.tmp.name).resolve()
|
| 20 | self.manifest_path = self.root / "manifest.json"
|
| 21 | self.raw = json.loads((campaign.ROOT / "config/agent-campaign-smoke.json").read_text())
|
| 22 | self.raw.update(agents_config=str(campaign.ROOT / "config/agents.toml"),
|
| 23 | subjects_config=str(campaign.ROOT / "config/subjects.toml"))
|
| 24 | self.write_manifest()
|
| 25 |
|
| 26 | def tearDown(self):
|
| 27 | self.tmp.cleanup()
|
| 28 |
|
| 29 | def write_manifest(self):
|
| 30 | self.manifest_path.write_text(json.dumps(self.raw))
|
| 31 |
|
| 32 | def manifest(self):
|
| 33 | self.write_manifest()
|
| 34 | return campaign.load_manifest(self.manifest_path)
|
| 35 |
|
| 36 | def plan(self):
|
| 37 | with mock.patch.object(campaign, "resolve_runtime", return_value=({}, {}, {})):
|
| 38 | return campaign.create_plan(self.manifest_path, self.root / "campaign")
|
| 39 |
|
| 40 | def test_oracle_instrument_identity_is_frozen_and_drift_rejected(self):
|
| 41 | from oakbench import workflow_oracles
|
| 42 | original = {"git_head_instrument_binary": "/instrument/git", "git_head_instrument_sha256": "a" * 64,
|
| 43 | "git_head_instrument_version": "git version synthetic"}
|
| 44 | with mock.patch.object(workflow_oracles, 'resolve_git_instrument', return_value=('/instrument/git', original)), \
|
| 45 | mock.patch.dict(os.environ, {'OAK_BENCH_OAK_READER_PYTHON': ''}):
|
| 46 | pinned = campaign.oracle_instrument_provenance()
|
| 47 | changed = copy.deepcopy(pinned)
|
| 48 | changed['git']['git_head_instrument_sha256'] = 'b' * 64
|
| 49 | plan = self.plan()
|
| 50 | plan['provenance'] = {'oracle_instruments': pinned}
|
| 51 | with mock.patch.object(campaign, 'resolve_runtime', return_value=({}, {}, {'oracle_instruments': changed})):
|
| 52 | with self.assertRaisesRegex(campaign.CampaignError, 'oracle_instruments'):
|
| 53 | campaign.preflight(plan, self.root / 'campaign')
|
| 54 |
|
| 55 | def test_optional_reader_identity_records_isolated_dependency_probe(self):
|
| 56 | from types import SimpleNamespace
|
| 57 | binary = self.root / 'reader-python'
|
| 58 | binary.write_bytes(b'configured interpreter bytes')
|
| 59 | metadata = {'python': '3.test', 'dependencies': {'blake3': '1.test', 'zstandard': '2.test'}}
|
| 60 | with mock.patch.dict(os.environ, {'OAK_BENCH_OAK_READER_PYTHON': str(binary)}), \
|
| 61 | mock.patch.object(campaign.shutil, 'which', return_value=str(binary)), \
|
| 62 | mock.patch.object(campaign.subprocess, 'run', return_value=SimpleNamespace(returncode=0, stdout=json.dumps(metadata), stderr='')) as probe:
|
| 63 | from oakbench import workflow_oracles
|
| 64 | with mock.patch.object(workflow_oracles, 'resolve_git_instrument', return_value=(None, {})):
|
| 65 | identity = campaign.oracle_instrument_provenance()['oak_reader']
|
| 66 | self.assertEqual(identity['metadata'], metadata)
|
| 67 | self.assertEqual(identity['sha256'], campaign.file_hash(binary))
|
| 68 | self.assertEqual(probe.call_args.args[0][:3], [str(binary), '-I', '-c'])
|
| 69 | self.assertEqual(identity['reader_module_sha256'], campaign.file_hash(campaign.ROOT / 'scripts/oakbench/oak_sqlite_reader.py'))
|
| 70 |
|
| 71 | def test_schedule_determinism_balance_and_identity(self):
|
| 72 | first = campaign.schedule(self.manifest())
|
| 73 | self.assertEqual(first, campaign.schedule(self.manifest()))
|
| 74 | self.assertEqual(len({trial["trial_id"] for trial in first}), len(first))
|
| 75 | self.assertEqual([x["subject"] for x in first[:2]], list(reversed([x["subject"] for x in first[2:]])))
|
| 76 | self.assertEqual(first[0]["block_id"], first[1]["block_id"])
|
| 77 | self.assertNotEqual(first[0]["block_id"], first[2]["block_id"])
|
| 78 | self.raw["instruction_levels"] = ["full-docs", "zero-shot"]
|
| 79 | self.raw["max_trials"] = 8
|
| 80 | self.assertEqual(len(campaign.schedule(self.manifest())), 8)
|
| 81 |
|
| 82 | def test_invalid_limits_unknown_fields_duplicates_and_paths(self):
|
| 83 | for key, value in (("schema_version", True), ("repetitions", True), ("repetitions", 0),
|
| 84 | ("max_trials", -1), ("max_wall_seconds", float("nan")),
|
| 85 | ("trial_timeout_seconds", float("inf")), ("seed", True),
|
| 86 | ("subjects", ["git", "git"]), ("subjects", ["../escape"]),
|
| 87 | ("workflows", ["history_archaeology"]), ("typo", 1)):
|
| 88 | with self.subTest(key=key, value=value):
|
| 89 | raw = dict(self.raw)
|
| 90 | raw[key] = value
|
| 91 | self.manifest_path.write_text(json.dumps(raw))
|
| 92 | with self.assertRaises(campaign.CampaignError):
|
| 93 | campaign.load_manifest(self.manifest_path)
|
| 94 | self.manifest_path.write_text('{"schema_version":1,"schema_version":1}')
|
| 95 | with self.assertRaises(campaign.CampaignError):
|
| 96 | campaign.load_manifest(self.manifest_path)
|
| 97 |
|
| 98 | def test_huge_schedule_rejected_before_materialization(self):
|
| 99 | self.raw["repetitions"] = 10 ** 30
|
| 100 | with self.assertRaises(campaign.CampaignError):
|
| 101 | campaign.schedule(self.manifest())
|
| 102 |
|
| 103 | def test_freeze_collision_and_tamper_rejected(self):
|
| 104 | plan = self.plan()
|
| 105 | directory = self.root / "campaign"
|
| 106 | self.assertEqual(campaign.load_plan(directory), plan)
|
| 107 | with mock.patch.object(campaign, "resolve_runtime", return_value=({}, {}, {})):
|
| 108 | with self.assertRaises(campaign.CampaignError):
|
| 109 | campaign.create_plan(self.manifest_path, directory)
|
| 110 | plan["schedule"][0]["subject"] = "wrong"
|
| 111 | (directory / "plan.json").write_text(json.dumps(plan))
|
| 112 | with self.assertRaises(campaign.CampaignError):
|
| 113 | campaign.load_plan(directory)
|
| 114 |
|
| 115 | def test_interrupted_trial_is_never_pending_and_reserves_budget(self):
|
| 116 | plan = self.plan()
|
| 117 | directory = self.root / "campaign"
|
| 118 | trial = plan["schedule"][0]["trial_id"]
|
| 119 | campaign.append_event(directory, plan, {"event": "started", "trial_id": trial, "timeout_seconds": 60})
|
| 120 | status = campaign.campaign_status(plan, campaign.load_events(directory, plan))
|
| 121 | self.assertEqual(status["interrupted_trial_ids"], [trial])
|
| 122 | self.assertNotIn(trial, status["pending_trial_ids"])
|
| 123 | self.assertEqual(status["charged_episode_seconds"], 60)
|
| 124 |
|
| 125 | def test_torn_duplicate_unknown_nonfinite_and_bad_duration_journal_fail_closed(self):
|
| 126 | plan = self.plan()
|
| 127 | directory = self.root / "campaign"
|
| 128 | trial = plan["schedule"][0]["trial_id"]
|
| 129 | start = {"event": "started", "trial_id": trial, "plan_sha256": plan["plan_sha256"], "timeout_seconds": 60}
|
| 130 | variants = [b'{"event":', b'[]\n', (json.dumps(start) + '\n') .encode() * 2,
|
| 131 | (json.dumps({**start, "trial_id": "other"}) + '\n').encode(),
|
| 132 | (json.dumps({**start, "timeout_seconds": float("nan")}) + '\n').encode()]
|
| 133 | for wall in (True, -1, float("inf")):
|
| 134 | finished = {"event": "finished", "trial_id": trial, "plan_sha256": plan["plan_sha256"],
|
| 135 | "outcome": "error", "error": "test", "wall_seconds": wall}
|
| 136 | variants.append((json.dumps(start) + '\n' + json.dumps(finished) + '\n').encode())
|
| 137 | for content in variants:
|
| 138 | with self.subTest(content=content):
|
| 139 | (directory / "events.jsonl").write_bytes(content)
|
| 140 | with self.assertRaises(campaign.CampaignError):
|
| 141 | campaign.load_events(directory, plan)
|
| 142 | self.assertEqual((directory / "events.jsonl").read_bytes(), content)
|
| 143 |
|
| 144 | def test_torn_new_start_after_old_interruption_reserves_next_trial(self):
|
| 145 | plan = self.plan()
|
| 146 | directory = self.root / "campaign"
|
| 147 | first, second, third = plan["schedule"][:3]
|
| 148 | for trial in (first, second):
|
| 149 | campaign.append_event(directory, plan, {"event": "started", "trial_id": trial["trial_id"], "timeout_seconds": 60})
|
| 150 | campaign.append_event(directory, plan, {"event": "finished", "trial_id": second["trial_id"],
|
| 151 | "outcome": "error", "wall_seconds": 1})
|
| 152 | with (directory / "events.jsonl").open("ab") as stream:
|
| 153 | stream.write(b'{"event":"started"')
|
| 154 | events = campaign.load_events(directory, plan, tolerate_torn=True)
|
| 155 | self.assertEqual(events[-2]["trial_id"], third["trial_id"])
|
| 156 | self.assertTrue(events[-2]["uncertain_torn_start"])
|
| 157 | state = campaign.campaign_status(plan, events)
|
| 158 | self.assertNotIn(third["trial_id"], state["pending_trial_ids"])
|
| 159 |
|
| 160 | def test_system_tmp_alias_is_accepted_but_arbitrary_symlinks_are_not(self):
|
| 161 | self.assertEqual(campaign.safe_path(Path("/tmp") / "campaign-safe-path-check"),
|
| 162 | (Path("/tmp") / "campaign-safe-path-check").resolve())
|
| 163 |
|
| 164 | def test_terminal_row_identity_cannot_be_rebound(self):
|
| 165 | from agent_metrics_schema import example_row
|
| 166 | plan = self.plan()
|
| 167 | directory = self.root / "campaign"
|
| 168 | trial = plan["schedule"][0]
|
| 169 | row = example_row()
|
| 170 | row.update(task_id=trial["trial_id"], bench_id=plan["campaign_id"], subject=trial["subject"],
|
| 171 | scenario=trial["workflow"], run=trial["repetition"], outcome="pass")
|
| 172 | row["agent"].update(name=trial["agent"], model=trial["model"],
|
| 173 | instruction_level=trial["instruction_level"], environment="minimal")
|
| 174 | campaign.validate_trial_row(row, trial, plan)
|
| 175 | for key, value in (("subject", "wrong"), ("scenario", "wrong"), ("run", 999), ("bench_id", "other")):
|
| 176 | wrong = copy.deepcopy(row)
|
| 177 | wrong[key] = value
|
| 178 | with self.subTest(key=key), self.assertRaises(campaign.CampaignError):
|
| 179 | campaign.validate_trial_row(wrong, trial, plan)
|
| 180 | for key in ("name", "model", "instruction_level"):
|
| 181 | wrong = copy.deepcopy(row)
|
| 182 | wrong["agent"][key] = "wrong"
|
| 183 | with self.subTest(agent_key=key), self.assertRaises(campaign.CampaignError):
|
| 184 | campaign.validate_trial_row(wrong, trial, plan)
|
| 185 | campaign.append_event(directory, plan, {"event": "started", "trial_id": trial["trial_id"], "timeout_seconds": 60})
|
| 186 | row["subject"] = "wrong"
|
| 187 | campaign.append_event(directory, plan, {"event": "finished", "trial_id": trial["trial_id"],
|
| 188 | "outcome": "pass", "row": row, "wall_seconds": 1})
|
| 189 | with self.assertRaises(campaign.CampaignError):
|
| 190 | campaign.load_events(directory, plan)
|
| 191 |
|
| 192 | def test_symlink_destinations_and_lock_rejected(self):
|
| 193 | alias = self.root / "alias"
|
| 194 | alias.symlink_to(self.root, target_is_directory=True)
|
| 195 | with self.assertRaises(campaign.CampaignError):
|
| 196 | campaign.safe_path(alias / "campaign")
|
| 197 | plan = self.plan()
|
| 198 | directory = self.root / "campaign"
|
| 199 | for name in ("events.jsonl", ".run.lock", "trials"):
|
| 200 | path = directory / name
|
| 201 | path.symlink_to(self.root / "outside")
|
| 202 | with self.assertRaises(campaign.CampaignError):
|
| 203 | campaign.safe_path(path)
|
| 204 | path.unlink()
|
| 205 |
|
| 206 | def test_exclusive_runner_ownership(self):
|
| 207 | self.plan()
|
| 208 | with campaign.campaign_lock(self.root / "campaign"):
|
| 209 | with self.assertRaises(campaign.CampaignError):
|
| 210 | with campaign.campaign_lock(self.root / "campaign"):
|
| 211 | pass
|
| 212 |
|
| 213 | def test_preflight_drift_real_optin_and_attestation(self):
|
| 214 | plan = self.plan()
|
| 215 | directory = self.root / "campaign"
|
| 216 | with mock.patch.object(campaign, "resolve_runtime", return_value=({}, {}, {"changed": True})):
|
| 217 | with self.assertRaises(campaign.CampaignError):
|
| 218 | campaign.preflight(plan, directory)
|
| 219 | from agent_workflow import AgentConfig
|
| 220 | real = AgentConfig("real", "codex_exec_json", "openai", Path("/binary"), "pinned", "1")
|
| 221 | with mock.patch.object(campaign, "resolve_runtime", return_value=({"real": real}, {}, {})):
|
| 222 | with self.assertRaises(campaign.CampaignError):
|
| 223 | campaign.preflight(plan, directory)
|
| 224 | with self.assertRaises(campaign.CampaignError):
|
| 225 | campaign.preflight(plan, directory, enable_real=True)
|
| 226 | plan["manifest"]["isolation_attestation"] = {
|
| 227 | "worker_root": str(self.root), "unrestricted_cli_acknowledged": True}
|
| 228 | self.assertEqual(campaign.preflight(plan, directory, enable_real=True)[0]["real"], real)
|
| 229 |
|
| 230 | def test_lock_provenance_drift_names_the_key(self):
|
| 231 | plan = self.plan()
|
| 232 | plan["provenance"] = {"measurement_lock_path": "/old/lock"}
|
| 233 | with mock.patch.object(campaign, "resolve_runtime", return_value=({}, {}, {"measurement_lock_path": "/new/lock"})):
|
| 234 | with self.assertRaisesRegex(campaign.CampaignError, "measurement_lock_path"):
|
| 235 | campaign.preflight(plan, self.root / "campaign")
|
| 236 |
|
| 237 | def test_fixture_cache_reuses_generation_and_rejects_mutated_payload(self):
|
| 238 | import workflow_ab
|
| 239 | from dataclasses import replace
|
| 240 | counter = mock.Mock(side_effect=workflow_ab.WORKFLOWS["wide_config_refactor"].make_fixture)
|
| 241 | workflow = replace(workflow_ab.WORKFLOWS["wide_config_refactor"], make_fixture=counter)
|
| 242 | plan = self.plan()
|
| 243 | directory = self.root / "campaign"
|
| 244 | fixture = campaign.cached_fixture(directory, plan, workflow)
|
| 245 | self.assertEqual(fixture, directory / "fixtures" / workflow.name)
|
| 246 | self.assertEqual(campaign.cached_fixture(directory, plan, workflow), fixture)
|
| 247 | self.assertEqual(counter.call_count, 1)
|
| 248 | payload = next(fixture.glob("services/*/config.toml"))
|
| 249 | payload.chmod(0o600)
|
| 250 | payload.write_text("corrupt fixture")
|
| 251 | with self.assertRaisesRegex(campaign.CampaignError, "fixture.*drift"):
|
| 252 | campaign.cached_fixture(directory, plan, workflow)
|
| 253 | self.assertEqual(counter.call_count, 1)
|
| 254 |
|
| 255 | def test_cache_permissions_allow_rebuild_and_plain_rmtree(self):
|
| 256 | import workflow_ab
|
| 257 | import shutil
|
| 258 | plan = self.plan()
|
| 259 | directory = self.root / "campaign"
|
| 260 | workflow = workflow_ab.WORKFLOWS["wide_config_refactor"]
|
| 261 | fixture = campaign.cached_fixture(directory, plan, workflow)
|
| 262 | self.assertTrue(fixture.stat().st_mode & 0o200)
|
| 263 | self.assertTrue(all(path.stat().st_mode & 0o200 for path in fixture.rglob("*") if path.is_dir()))
|
| 264 | self.assertTrue(all(not path.stat().st_mode & 0o222 for path in fixture.rglob("*") if path.is_file()))
|
| 265 | marker = fixture / ".fixture-ready"
|
| 266 | marker.chmod(0o600)
|
| 267 | marker.write_text("old fixture version")
|
| 268 | rebuilt = workflow_ab.prepare_fixture(directory, workflow)
|
| 269 | self.assertEqual((rebuilt / ".fixture-ready").read_text().strip(), workflow_ab.FIXTURE_VERSION)
|
| 270 | campaign.fixture_permissions(rebuilt, writable=False)
|
| 271 | shutil.rmtree(directory)
|
| 272 | self.assertFalse(directory.exists())
|
| 273 |
|
| 274 | def test_permissions_do_not_require_chmod_nofollow_support(self):
|
| 275 | fixture = self.root / "permission-fixture"
|
| 276 | fixture.mkdir()
|
| 277 | (fixture / "file").write_text("content")
|
| 278 | chmod = os.chmod
|
| 279 | def portable(path, mode, **kwargs):
|
| 280 | if kwargs.get("follow_symlinks") is False:
|
| 281 | raise NotImplementedError("chmod nofollow unavailable on this platform")
|
| 282 | return chmod(path, mode, **kwargs)
|
| 283 | with mock.patch.object(os, "chmod", side_effect=portable):
|
| 284 | campaign.fixture_permissions(fixture, writable=False)
|
| 285 | campaign.fixture_permissions(fixture, writable=True)
|
| 286 |
|
| 287 | def test_fifo_cache_is_a_named_campaign_error(self):
|
| 288 | import workflow_ab
|
| 289 | plan = self.plan()
|
| 290 | directory = self.root / "campaign"
|
| 291 | fixture = campaign.cached_fixture(directory, plan, workflow_ab.WORKFLOWS["wide_config_refactor"])
|
| 292 | fixture.chmod(0o700)
|
| 293 | os.mkfifo(fixture / "unexpected-pipe")
|
| 294 | with self.assertRaisesRegex(campaign.CampaignError, "wide_config_refactor"):
|
| 295 | campaign.cached_fixture(directory, plan, workflow_ab.WORKFLOWS["wide_config_refactor"])
|
| 296 |
|
| 297 | def test_corrupted_cache_aborts_before_any_trial_admission(self):
|
| 298 | import workflow_ab
|
| 299 | plan = self.plan()
|
| 300 | directory = self.root / "campaign"
|
| 301 | fixture = campaign.cached_fixture(directory, plan, workflow_ab.WORKFLOWS["wide_config_refactor"])
|
| 302 | payload = next(fixture.glob("services/*/config.toml"))
|
| 303 | payload.chmod(0o600)
|
| 304 | payload.write_text("bad cached baseline")
|
| 305 | with mock.patch.object(campaign, "preflight", return_value=({}, {})), mock.patch("agent_workflow.run_one") as run:
|
| 306 | with self.assertRaisesRegex(campaign.CampaignError, "fixture.*drift"):
|
| 307 | campaign.run_campaign(directory, resume=True)
|
| 308 | run.assert_not_called()
|
| 309 | self.assertFalse((directory / "events.jsonl").exists())
|
| 310 | self.assertEqual(len(campaign.campaign_status(plan, [])['pending_trial_ids']), 4)
|
| 311 |
|
| 312 | def test_generation_crash_and_incomplete_cache_preserve_all_pending_trials(self):
|
| 313 | import workflow_ab
|
| 314 | plan = self.plan()
|
| 315 | directory = self.root / "campaign"
|
| 316 | def crash(root, workflow):
|
| 317 | path = root / "fixtures" / workflow.name
|
| 318 | path.mkdir(parents=True)
|
| 319 | (path / "partial").write_text("interrupted generation")
|
| 320 | raise KeyboardInterrupt("crash before fixture integrity record")
|
| 321 | with mock.patch.object(campaign, "preflight", return_value=({}, {})), mock.patch.object(workflow_ab, "prepare_fixture", side_effect=crash):
|
| 322 | with self.assertRaises(KeyboardInterrupt):
|
| 323 | campaign.run_campaign(directory)
|
| 324 | self.assertFalse((directory / "events.jsonl").exists())
|
| 325 | with mock.patch.object(campaign, "preflight", return_value=({}, {})):
|
| 326 | with self.assertRaisesRegex(campaign.CampaignError, "fixture cache is incomplete"):
|
| 327 | campaign.run_campaign(directory, resume=True)
|
| 328 | self.assertFalse((directory / "events.jsonl").exists())
|
| 329 | self.assertEqual(len(campaign.campaign_status(plan, [])['pending_trial_ids']), 4)
|
| 330 |
|
| 331 | def test_real_git_mock_cache_poisoning_invalidates_attempt_and_preserves_next(self):
|
| 332 | def mutate(directory):
|
| 333 | payload = next((directory / "fixtures" / "wide_config_refactor").glob("services/*/config.toml"))
|
| 334 | payload.chmod(0o600) # Same-uid processes can defeat read-only files.
|
| 335 | payload.write_text("poisoned shared baseline")
|
| 336 | self.assert_midtrial_fixture_invalidation(mutate)
|
| 337 |
|
| 338 | def test_real_git_mock_deleted_cache_record_invalidates_attempt(self):
|
| 339 | self.assert_midtrial_fixture_invalidation(lambda directory:
|
| 340 | (directory / "fixture-integrity/wide_config_refactor.json").unlink())
|
| 341 |
|
| 342 | def test_real_git_mock_corrupt_cache_record_invalidates_attempt(self):
|
| 343 | self.assert_midtrial_fixture_invalidation(lambda directory:
|
| 344 | (directory / "fixture-integrity/wide_config_refactor.json").write_text("invalid JSON"))
|
| 345 |
|
| 346 | def assert_midtrial_fixture_invalidation(self, mutate):
|
| 347 | import agent_workflow
|
| 348 | import workflow_ab
|
| 349 | import shutil
|
| 350 | if not shutil.which("git"):
|
| 351 | self.skipTest("Git required")
|
| 352 | self.raw.update(subjects=["git"], repetitions=2)
|
| 353 | self.write_manifest()
|
| 354 | plan = self.plan()
|
| 355 | directory = self.root / "campaign"
|
| 356 | agent = agent_workflow.AgentConfig("mock", "deterministic_mock", "local", None, None, "1")
|
| 357 | subject = workflow_ab.Subject("git", "git", "Git", Path(shutil.which("git")))
|
| 358 | original = agent_workflow.run_mock_agent
|
| 359 | def poison(*args, **kwargs):
|
| 360 | execution = original(*args, **kwargs)
|
| 361 | mutate(directory)
|
| 362 | return execution
|
| 363 | with mock.patch.object(campaign, "preflight", return_value=({"mock": agent}, {"git": subject})), \
|
| 364 | mock.patch.object(agent_workflow, "run_mock_agent", side_effect=poison) as execute:
|
| 365 | state = campaign.run_campaign(directory)
|
| 366 | self.assertEqual(execute.call_count, 1)
|
| 367 | self.assertEqual(len(state["pending_trial_ids"]), 1)
|
| 368 | self.assertEqual(state["stopped_reason"], "fixture_integrity_drift")
|
| 369 | events = campaign.load_events(directory, plan)
|
| 370 | self.assertEqual(len(events), 2)
|
| 371 | self.assertTrue(events[-1]["measurement_invalidated"])
|
| 372 | self.assertTrue(events[-1]["rejected_row"]["success"])
|
| 373 | self.assertNotIn("row", events[-1])
|
| 374 | self.assertEqual((directory / "rows.jsonl").read_bytes(), b"")
|
| 375 | from oakbench.campaign_report import report_campaign
|
| 376 | report = report_campaign(plan, events)
|
| 377 | self.assertEqual(report["counts"]["skipped"], 1)
|
| 378 | self.assertEqual(report["counts"]["unstarted"], 1)
|
| 379 |
|
| 380 | def test_copied_fixture_race_is_rejected_before_real_agent_launch(self):
|
| 381 | import agent_workflow
|
| 382 | import workflow_ab
|
| 383 | import shutil
|
| 384 | if not shutil.which("git"):
|
| 385 | self.skipTest("Git required")
|
| 386 | self.raw.update(subjects=["git"], repetitions=2)
|
| 387 | self.write_manifest()
|
| 388 | plan = self.plan()
|
| 389 | directory = self.root / "campaign"
|
| 390 | agent = agent_workflow.AgentConfig("mock", "deterministic_mock", "local", None, None, "1")
|
| 391 | subject = workflow_ab.Subject("git", "git", "Git", Path(shutil.which("git")))
|
| 392 | original = agent_workflow.copy_fixture
|
| 393 | def corrupt_copy(src, dest):
|
| 394 | original(src, dest)
|
| 395 | payload = next(dest.glob("services/*/config.toml"))
|
| 396 | payload.chmod(0o600)
|
| 397 | payload.write_text("changed during copy")
|
| 398 | with mock.patch.object(campaign, "preflight", return_value=({"mock": agent}, {"git": subject})), \
|
| 399 | mock.patch.object(agent_workflow, "copy_fixture", side_effect=corrupt_copy), \
|
| 400 | mock.patch.object(agent_workflow, "run_mock_agent") as execute:
|
| 401 | with self.assertRaisesRegex(campaign.CampaignError, "copied fixture identity drift"):
|
| 402 | campaign.run_campaign(directory)
|
| 403 | self.assertEqual(execute.call_count, 0)
|
| 404 | self.assertFalse((directory / "events.jsonl").exists())
|
| 405 | self.assertEqual(len(campaign.campaign_status(plan, [])["pending_trial_ids"]), 2)
|
| 406 | with mock.patch.object(campaign, "preflight", return_value=({"mock": agent}, {"git": subject})):
|
| 407 | state = campaign.run_campaign(directory, resume=True)
|
| 408 | self.assertEqual(state["finished"], 2)
|
| 409 | self.assertEqual(state["pending_trial_ids"], [])
|
| 410 | self.assertTrue(all(event["row"]["success"] for event in campaign.load_events(directory, plan) if event["event"] == "finished"))
|
| 411 |
|
| 412 | def test_journal_fixture_binding_rejects_missing_or_conflicting_attempt_hashes(self):
|
| 413 | plan = self.plan()
|
| 414 | trial = plan["schedule"][0]
|
| 415 | record = {"manifest": {"config": {"sha256": "a" * 64}}}
|
| 416 | fingerprint = campaign.digest(record["manifest"])
|
| 417 | start = {"event": "started", "trial_id": trial["trial_id"], "fixture_manifest_sha256": fingerprint}
|
| 418 | finish = {"event": "finished", "trial_id": trial["trial_id"], "fixture_manifest_sha256": fingerprint}
|
| 419 | campaign.verify_journal_fixture_binding(trial["workflow"], record, plan, [start, finish])
|
| 420 | for value in (None, "b" * 64):
|
| 421 | for index in (0, 1):
|
| 422 | events = [dict(start), dict(finish)]
|
| 423 | events[index]["fixture_manifest_sha256"] = value
|
| 424 | with self.assertRaisesRegex(campaign.CampaignError, "journal fixture binding"):
|
| 425 | campaign.verify_journal_fixture_binding(trial["workflow"], record, plan, events)
|
| 426 |
|
| 427 | def test_completed_infrastructure_error_not_retried_on_resume(self):
|
| 428 | self.plan()
|
| 429 | directory = self.root / "campaign"
|
| 430 | import agent_workflow
|
| 431 | import workflow_ab
|
| 432 | from oakbench.runlock import LockInfo
|
| 433 | from contextlib import contextmanager
|
| 434 | @contextmanager
|
| 435 | def locked(label):
|
| 436 | yield LockInfo(0, True)
|
| 437 | agent = agent_workflow.AgentConfig("mock", "deterministic_mock", "local", None, None, "1")
|
| 438 | subject = workflow_ab.Subject("git", "git", "Git", Path("/git"))
|
| 439 | with mock.patch.object(campaign, "preflight", return_value=({"mock": agent}, {"git": subject, "oak_installed": subject})), \
|
| 440 | mock.patch("oakbench.runlock.measurement_lock", locked), \
|
| 441 | mock.patch.object(agent_workflow, "run_one", side_effect=RuntimeError("fixture setup failed")) as run:
|
| 442 | state = campaign.run_campaign(directory, limit=1)
|
| 443 | self.assertEqual(state["finished"], 1)
|
| 444 | campaign.run_campaign(directory, resume=True, limit=1)
|
| 445 | self.assertEqual(run.call_count, 2)
|
| 446 | events = campaign.load_events(directory, campaign.load_plan(directory))
|
| 447 | self.assertEqual(len({event["trial_id"] for event in events if event["event"] == "started"}), 2)
|
| 448 | self.assertEqual([event["outcome"] for event in events if event["event"] == "finished"], ["error", "error"])
|
| 449 |
|
| 450 | def test_staging_move_crash_after_durable_start_is_not_pending(self):
|
| 451 | from contextlib import contextmanager
|
| 452 | from oakbench.runlock import LockInfo
|
| 453 | self.plan()
|
| 454 | directory = self.root / "campaign"
|
| 455 | @contextmanager
|
| 456 | def locked(label):
|
| 457 | yield LockInfo(0, True)
|
| 458 | original = Path.rename
|
| 459 | def crash(path, target):
|
| 460 | if target.parent.name == "trials":
|
| 461 | raise KeyboardInterrupt("simulated process crash")
|
| 462 | return original(path, target)
|
| 463 | with mock.patch.object(campaign, "preflight", return_value=({}, {})), \
|
| 464 | mock.patch("oakbench.runlock.measurement_lock", locked), \
|
| 465 | mock.patch.object(Path, "rename", crash):
|
| 466 | with self.assertRaises(KeyboardInterrupt):
|
| 467 | campaign.run_campaign(directory)
|
| 468 | plan = campaign.load_plan(directory)
|
| 469 | events = campaign.load_events(directory, plan)
|
| 470 | self.assertEqual(len(events), 1)
|
| 471 | state = campaign.campaign_status(plan, events)
|
| 472 | self.assertEqual(state["interrupted_trial_ids"], [plan["schedule"][0]["trial_id"]])
|
| 473 | self.assertNotIn(plan["schedule"][0]["trial_id"], state["pending_trial_ids"])
|
| 474 |
|
| 475 | def test_schema_rejected_row_remains_in_terminal_evidence(self):
|
| 476 | import agent_workflow
|
| 477 | from contextlib import contextmanager
|
| 478 | from oakbench.runlock import LockInfo
|
| 479 | plan = self.plan()
|
| 480 | directory = self.root / "campaign"
|
| 481 | @contextmanager
|
| 482 | def locked(label):
|
| 483 | yield LockInfo(0, True)
|
| 484 | subjects = {trial["subject"]: object() for trial in plan["schedule"]}
|
| 485 | with mock.patch.object(campaign, "preflight", return_value=({"mock": object()}, subjects)), \
|
| 486 | mock.patch("oakbench.runlock.measurement_lock", locked), \
|
| 487 | mock.patch.object(agent_workflow, "run_one", return_value={"invalid": "original evidence"}):
|
| 488 | campaign.run_campaign(directory, limit=1)
|
| 489 | events = campaign.load_events(directory, plan)
|
| 490 | self.assertEqual(events[1]["outcome"], "error")
|
| 491 | self.assertEqual(events[1]["rejected_row"]["invalid"], "original evidence")
|
| 492 | self.assertEqual((directory / "rows.jsonl").read_bytes(), b"")
|
| 493 |
|
| 494 | def test_noop_resume_repairs_missing_or_stale_rows_from_finished_journal(self):
|
| 495 | from agent_metrics_schema import example_row
|
| 496 | self.raw.update(subjects=["git"], repetitions=1)
|
| 497 | self.write_manifest()
|
| 498 | plan = self.plan()
|
| 499 | directory = self.root / "campaign"
|
| 500 | trial = plan["schedule"][0]
|
| 501 | import workflow_ab
|
| 502 | campaign.cached_fixture(directory, plan, workflow_ab.WORKFLOWS[trial["workflow"]])
|
| 503 | record = campaign.read_json(directory / "fixture-integrity" / (trial["workflow"] + ".json"))
|
| 504 | fingerprint = campaign.digest(record["manifest"])
|
| 505 | row = example_row()
|
| 506 | row.update(task_id=trial["trial_id"], bench_id=plan["campaign_id"], subject=trial["subject"],
|
| 507 | scenario=trial["workflow"], run=trial["repetition"], outcome="pass")
|
| 508 | row["agent"].update(name=trial["agent"], model=trial["model"],
|
| 509 | instruction_level=trial["instruction_level"], environment="minimal")
|
| 510 | campaign.append_event(directory, plan, {"event": "started", "trial_id": trial["trial_id"], "timeout_seconds": 60, "fixture_manifest_sha256": fingerprint})
|
| 511 | campaign.append_event(directory, plan, {"event": "finished", "trial_id": trial["trial_id"],
|
| 512 | "outcome": "pass", "row": row, "wall_seconds": 1, "fixture_manifest_sha256": fingerprint})
|
| 513 | journal = (directory / "events.jsonl").read_bytes()
|
| 514 | for state in ("missing", "stale"):
|
| 515 | with self.subTest(state=state):
|
| 516 | if state == "stale":
|
| 517 | (directory / "rows.jsonl").write_bytes(b"stale output")
|
| 518 | with mock.patch.object(campaign, "preflight", return_value=({}, {})), \
|
| 519 | mock.patch("agent_workflow.run_one") as run:
|
| 520 | status = campaign.run_campaign(directory, resume=True)
|
| 521 | run.assert_not_called()
|
| 522 | self.assertEqual(status["finished"], 1)
|
| 523 | self.assertEqual((directory / "rows.jsonl").read_bytes(), campaign.canonical(row) + b"\n")
|
| 524 | self.assertEqual((directory / "events.jsonl").read_bytes(), journal)
|
| 525 |
|
| 526 | def test_disabled_measurement_lock_starts_nothing(self):
|
| 527 | self.plan()
|
| 528 | directory = self.root / "campaign"
|
| 529 | with mock.patch.object(campaign, "preflight", return_value=({}, {})), \
|
| 530 | mock.patch.dict(os.environ, {"OAK_BENCH_LOCK": "off"}):
|
| 531 | with self.assertRaises(campaign.CampaignError):
|
| 532 | campaign.run_campaign(directory)
|
| 533 | self.assertFalse((directory / "events.jsonl").exists())
|
| 534 |
|
| 535 | def test_artifact_hashes_exclude_links(self):
|
| 536 | trial = self.root / "trial"
|
| 537 | (trial / "artifacts").mkdir(parents=True)
|
| 538 | (trial / "artifacts/prompt.txt").write_text("fixed prompt")
|
| 539 | (trial / "artifacts/escape").symlink_to(self.manifest_path)
|
| 540 | hashes = campaign.artifact_hashes(trial)
|
| 541 | self.assertEqual(set(hashes), {"artifacts/prompt.txt"})
|
| 542 | self.assertEqual(hashes["artifacts/prompt.txt"], campaign.file_hash(trial / "artifacts/prompt.txt"))
|
| 543 |
|
| 544 |
|
| 545 | if __name__ == "__main__":
|
| 546 | unittest.main()
|