| 1 | import copy
|
| 2 | from pathlib import Path
|
| 3 | import sys
|
| 4 | import unittest
|
| 5 |
|
| 6 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 7 | from oakbench.campaign_report import report_campaign, render_markdown
|
| 8 |
|
| 9 |
|
| 10 | def plan(repetitions=2):
|
| 11 | return {"schema_version": 1, "campaign_id": "example", "plan_sha256": "digest", "manifest": {},
|
| 12 | "schedule": [dict(trial_id=f"{rep}-{subject}", block_id=f"block-{rep}", ordinal=rep * 2 + index,
|
| 13 | repetition=rep, workflow="wide_config_refactor", instruction_level="zero-shot",
|
| 14 | agent="codex", model="pinned-model", subject=subject)
|
| 15 | for rep in range(repetitions) for index, subject in enumerate(("git", "oak"))]}
|
| 16 |
|
| 17 |
|
| 18 | def event(trial_id, kind, **extra):
|
| 19 | return dict(event=kind, trial_id=trial_id, plan_sha256="digest", timestamp_utc="2026-09-10T00:00:00Z", **extra)
|
| 20 |
|
| 21 |
|
| 22 | def outcome(trial_id, name="pass", tokens=10):
|
| 23 | return [event(trial_id, "started"), event(trial_id, "finished", row={
|
| 24 | "outcome": name, "success": name == "pass", "wall_clock_metrics": {"agent_active_ms": 20, "total_ms": 30},
|
| 25 | "token_metrics": {"total_tokens_reported": tokens, "cost_weighted_total": 40},
|
| 26 | "turn_metrics": {"assistant_turns_total": 2}, "tool_call_metrics": {"tool_calls_total": 3},
|
| 27 | })]
|
| 28 |
|
| 29 |
|
| 30 | class CampaignReportTest(unittest.TestCase):
|
| 31 | def report(self, configuration, events):
|
| 32 | return report_campaign(configuration, events, bootstrap_samples=100, seed=7)
|
| 33 |
|
| 34 | def test_schedule_and_started_denominators_survive_interruption(self):
|
| 35 | summary = self.report(plan(), outcome("0-git") + outcome("0-oak", "fail") + [event("1-git", "started")])
|
| 36 | self.assertEqual(summary["counts"], dict(scheduled=4, started=3, finished=2, budget_truncated=0, success=1, failure=1,
|
| 37 | timeout=0, infrastructure_failure=0, interrupted=1, skipped=0, unstarted=1))
|
| 38 | subjects = {item["subject"]: item for item in summary["groups"][0]["subjects"]}
|
| 39 | self.assertEqual(subjects["git"]["verified_success_fraction_of_started"], .5)
|
| 40 | self.assertIsNone(subjects["git"]["success_wilson_interval"])
|
| 41 | self.assertIsNone(subjects["git"]["resources"]["reported_tokens"]["complete_total"])
|
| 42 | self.assertEqual(summary["groups"][0]["paired_comparisons"][0]["incomplete_pairs"], 1)
|
| 43 | self.assertFalse(summary["claim_eligible"])
|
| 44 |
|
| 45 | def test_legacy_shortened_timeout_is_not_a_complete_pair(self):
|
| 46 | configuration = plan(1)
|
| 47 | configuration["manifest"]["trial_timeout_seconds"] = 60
|
| 48 | events = outcome("0-git") + outcome("0-oak", "timeout")
|
| 49 | events[0]["timeout_seconds"] = 60
|
| 50 | events[2]["timeout_seconds"] = .15
|
| 51 | summary = self.report(configuration, events)
|
| 52 | self.assertEqual(summary["counts"]["budget_truncated"], 1)
|
| 53 | self.assertEqual(summary["counts"]["interrupted"], 1)
|
| 54 | pair = summary["groups"][0]["paired_comparisons"][0]
|
| 55 | self.assertEqual(pair["complete_pairs"], 0)
|
| 56 | self.assertIsNone(pair["success_delta_percentage_points"])
|
| 57 |
|
| 58 | def test_missing_instrument_is_unmeasured_not_a_paired_failure(self):
|
| 59 | events = outcome("0-git", "error") + outcome("0-oak")
|
| 60 | events[1]["row"].update(returncode=77, skip_reason="required oracle unavailable")
|
| 61 | summary = self.report(plan(1), events)
|
| 62 | self.assertEqual(summary["counts"]["skipped"], 1)
|
| 63 | self.assertEqual(summary["counts"]["started"], 2)
|
| 64 | pair = summary["groups"][0]["paired_comparisons"][0]
|
| 65 | self.assertEqual(pair["complete_pairs"], 0)
|
| 66 | self.assertIsNone(pair["success_delta_percentage_points"])
|
| 67 | git = next(subject for subject in summary["groups"][0]["subjects"] if subject["subject"] == "git")
|
| 68 | self.assertIsNone(git["success_wilson_interval"])
|
| 69 |
|
| 70 | def test_fixture_invalidated_trial_is_not_a_complete_paired_failure(self):
|
| 71 | events = outcome("0-git") + [event("0-oak", "started"), event("0-oak", "finished",
|
| 72 | outcome="error", measurement_invalidated=True, invalidation_reason="fixture content drift",
|
| 73 | rejected_row={"outcome": "pass", "success": True})]
|
| 74 | summary = self.report(plan(1), events)
|
| 75 | self.assertEqual(summary["counts"]["skipped"], 1)
|
| 76 | self.assertEqual(summary["invalidated_count"], 1)
|
| 77 | self.assertEqual(summary["invalidated_trials"][0]["reason"], "fixture content drift")
|
| 78 | self.assertTrue(summary["invalidated_trials"][0]["rejected_row_retained"])
|
| 79 | self.assertIn("0-oak: fixture content drift", render_markdown(summary))
|
| 80 | self.assertEqual(summary["counts"]["infrastructure_failure"], 0)
|
| 81 | pair = summary["groups"][0]["paired_comparisons"][0]
|
| 82 | self.assertEqual(pair["complete_pairs"], 0)
|
| 83 | self.assertIsNone(pair["success_delta_percentage_points"])
|
| 84 |
|
| 85 | def test_infrastructure_attempts_stay_resources_not_success_pairs(self):
|
| 86 | events = outcome("0-git") + outcome("1-git")
|
| 87 | for trial in ("0-oak", "1-oak"):
|
| 88 | events.extend([event(trial, "started"), event(trial, "finished", outcome="error", error="harness setup failed")])
|
| 89 | summary = self.report(plan(2), events)
|
| 90 | pair = summary["groups"][0]["paired_comparisons"][0]
|
| 91 | self.assertEqual(pair["complete_pairs"], 0)
|
| 92 | self.assertIsNone(pair["success_delta_percentage_points"])
|
| 93 | self.assertIn("schedule_or_attempt_accounting_incomplete", summary["claim_ineligibility_reasons"])
|
| 94 | oak = next(item for item in summary["groups"][0]["subjects"] if item["subject"] == "oak")
|
| 95 | self.assertIsNone(oak["success_wilson_interval"])
|
| 96 | self.assertEqual(oak["resources"]["reported_tokens"]["attempted_n"], 2)
|
| 97 |
|
| 98 | def test_measured_precommit_failure_keeps_pair_but_success_proof_classes_must_match(self):
|
| 99 | events = outcome("0-git") + outcome("0-oak", "fail")
|
| 100 | events[1]["row"].update(workflow_integrity_passed=True, workflow_integrity_evidence_class="independent_committed_tree")
|
| 101 | events[3]["row"].update(workflow_integrity_passed=False, workflow_integrity_evidence_class="trusted_worktree_payload", agent_launched=True, outcome_attribution="agent")
|
| 102 | summary = self.report(plan(1), events)
|
| 103 | pair = summary["groups"][0]["paired_comparisons"][0]
|
| 104 | self.assertEqual(pair["complete_pairs"], 1)
|
| 105 | self.assertEqual(pair["success_delta_percentage_points"], -100)
|
| 106 | events[3]["row"].update(outcome="pass", success=True, workflow_integrity_passed=True)
|
| 107 | pair = self.report(plan(1), events)["groups"][0]["paired_comparisons"][0]
|
| 108 | self.assertTrue(pair["comparison_refused"])
|
| 109 | self.assertIn("workflow_integrity_evidence_class", pair["incompatible_evidence_fields"])
|
| 110 | self.assertEqual(pair["complete_pairs"], 0)
|
| 111 |
|
| 112 | def test_refused_comparison_reports_no_biased_success_delta(self):
|
| 113 | # QA P1 A2 (reviewer's stageA/camp1/c2 shape): default-config classes
|
| 114 | # differ, one git trial is a measured failure. Refusal drops the
|
| 115 | # concordant successes; the remainder would say +100 pp vs a true +33.
|
| 116 | events = []
|
| 117 | for rep in range(3):
|
| 118 | git_pass = rep != 0
|
| 119 | events += outcome(f"{rep}-git", "pass" if git_pass else "fail") + outcome(f"{rep}-oak")
|
| 120 | for entry in events:
|
| 121 | row = entry.get("row")
|
| 122 | if not row:
|
| 123 | continue
|
| 124 | is_git = entry["trial_id"].endswith("-git")
|
| 125 | row.update(workflow_integrity_evidence_class="subject_binary_committed_tree" if is_git else "subject_produced_committed_tree",
|
| 126 | workflow_integrity_passed=row["success"])
|
| 127 | if not row["success"]:
|
| 128 | row.update(agent_launched=True, outcome_attribution="agent")
|
| 129 | summary = self.report(plan(3), events)
|
| 130 | pair = summary["groups"][0]["paired_comparisons"][0]
|
| 131 | self.assertTrue(pair["comparison_refused"])
|
| 132 | self.assertIsNone(pair["success_delta_percentage_points"])
|
| 133 | self.assertIsNone(pair["confidence_interval"])
|
| 134 | self.assertIsNone(pair["success_conditioned_efficiency"])
|
| 135 | text = render_markdown(summary)
|
| 136 | self.assertIn("comparison REFUSED", text)
|
| 137 | self.assertNotIn("percentage points on", text)
|
| 138 |
|
| 139 | def test_every_outcome_remains_visible(self):
|
| 140 | configuration = plan(4)
|
| 141 | events = (outcome("0-git") + outcome("0-oak", "fail") + outcome("1-git", "timeout") +
|
| 142 | [event("1-oak", "started"), event("1-oak", "finished", outcome="error", error="crash"),
|
| 143 | event("2-git", "started"), event("2-oak", "finished", outcome="skipped")])
|
| 144 | counts = self.report(configuration, events)["counts"]
|
| 145 | for field in ("success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped"):
|
| 146 | self.assertEqual(counts[field], 1)
|
| 147 | self.assertEqual(counts["unstarted"], 2)
|
| 148 | self.assertEqual(sum(counts[name] for name in ("success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped", "unstarted")), counts["scheduled"])
|
| 149 |
|
| 150 | def test_paired_bootstrap_preserves_block_pairs_and_is_seeded(self):
|
| 151 | events = outcome("0-git") + outcome("0-oak", "fail") + outcome("1-git", "fail") + outcome("1-oak")
|
| 152 | first = self.report(plan(), events)
|
| 153 | second = self.report(plan(), events)
|
| 154 | self.assertEqual(first, second)
|
| 155 | pair = first["groups"][0]["paired_comparisons"][0]
|
| 156 | self.assertEqual(pair["success_delta_percentage_points"], 0)
|
| 157 | self.assertEqual(pair["confidence_interval"], [-100, 100])
|
| 158 | self.assertEqual(pair["complete_pairs"], 2)
|
| 159 | self.assertFalse(pair["multiplicity_adjusted"])
|
| 160 |
|
| 161 | def test_singleton_pair_does_not_get_zero_width_interval(self):
|
| 162 | pair = self.report(plan(1), outcome("0-git") + outcome("0-oak"))["groups"][0]["paired_comparisons"][0]
|
| 163 | self.assertEqual(pair["success_delta_percentage_points"], 0)
|
| 164 | self.assertIsNone(pair["confidence_interval"])
|
| 165 |
|
| 166 | def test_missing_usage_does_not_become_zero_or_complete_total(self):
|
| 167 | summary = self.report(plan(), outcome("0-git", tokens=None) + outcome("1-git", tokens=0))
|
| 168 | values = summary["groups"][0]["subjects"][0]["resources"]["reported_tokens"]
|
| 169 | self.assertEqual(values["measured_n"], 1)
|
| 170 | self.assertEqual(values["missing_n"], 1)
|
| 171 | self.assertEqual(values["observed_sum"], 0)
|
| 172 | self.assertIsNone(values["complete_total"])
|
| 173 | self.assertIsNone(values["observed_cost_per_success"])
|
| 174 | self.assertEqual(values["success_conditioned"]["median"], 0)
|
| 175 |
|
| 176 | def test_partial_capture_counters_are_not_complete_usage(self):
|
| 177 | for changes in ({"outcome": "timeout", "success": False}, {"output_truncated": True},
|
| 178 | {"source": {"agent_execution": {"capture_complete": False}}},
|
| 179 | {"source": {"agent_execution": {"usage_complete": False}}},
|
| 180 | {"source": {"agent_execution": {"counters_coverage": "observed_lower_bounds"}}},
|
| 181 | {"source": {"agent_execution": {"output_limited": True}}}):
|
| 182 | events = outcome("0-git")
|
| 183 | events[-1]["row"].update(changes)
|
| 184 | resources = self.report(plan(1), events)["groups"][0]["subjects"][0]["resources"]
|
| 185 | for metric in ("reported_tokens", "assistant_turns", "tool_calls"):
|
| 186 | self.assertIsNone(resources[metric]["complete_total"])
|
| 187 | self.assertEqual(resources[metric]["missing_n"], 1)
|
| 188 | self.assertEqual(resources["agent_active_ms"]["complete_total"], 20)
|
| 189 |
|
| 190 | def test_capture_error_row_keeps_infrastructure_denominator_and_time(self):
|
| 191 | events = outcome("0-git")
|
| 192 | events[-1]["outcome"] = "error"
|
| 193 | events[-1]["row"].update(outcome="error", success=False, output_truncated=True)
|
| 194 | summary = self.report(plan(1), events)
|
| 195 | self.assertEqual(summary["counts"]["infrastructure_failure"], 1)
|
| 196 | resources = summary["groups"][0]["subjects"][0]["resources"]
|
| 197 | self.assertEqual(resources["agent_active_ms"]["complete_total"], 20)
|
| 198 | self.assertIsNone(resources["tool_calls"]["complete_total"])
|
| 199 |
|
| 200 | def test_failures_contribute_resource_cost_but_not_success_latency(self):
|
| 201 | events = outcome("0-git", tokens=10) + outcome("1-git", "fail", tokens=50)
|
| 202 | subject = self.report(plan(), events)["groups"][0]["subjects"][0]
|
| 203 | values = subject["resources"]["reported_tokens"]
|
| 204 | self.assertEqual(values["complete_total"], 60)
|
| 205 | self.assertEqual(values["observed_cost_per_success"], 60)
|
| 206 | self.assertEqual(values["success_conditioned"]["median"], 10)
|
| 207 | self.assertAlmostEqual(subject["success_wilson_interval"][0], .094531, places=5)
|
| 208 | self.assertAlmostEqual(subject["success_wilson_interval"][1], .905469, places=5)
|
| 209 |
|
| 210 | def test_zero_success_or_no_attempts_keeps_undefined_metrics_null(self):
|
| 211 | summary = self.report(plan(1), outcome("0-git", "fail"))
|
| 212 | git, oak = summary["groups"][0]["subjects"]
|
| 213 | self.assertIsNone(git["resources"]["reported_tokens"]["observed_cost_per_success"])
|
| 214 | self.assertIsNone(oak["verified_success_fraction_of_started"])
|
| 215 | self.assertIsNone(oak["resources"]["reported_tokens"]["observed_sum"])
|
| 216 | self.assertIn("unmeasured", render_markdown(summary))
|
| 217 |
|
| 218 | def test_instruction_and_model_conditions_never_pool(self):
|
| 219 | configuration = plan()
|
| 220 | configuration["schedule"][2]["instruction_level"] = "full-docs"
|
| 221 | configuration["schedule"][3]["model"] = "other-model"
|
| 222 | summary = self.report(configuration, [])
|
| 223 | self.assertEqual(len(summary["groups"]), 3)
|
| 224 |
|
| 225 | def test_duplicate_or_foreign_journal_evidence_rejected(self):
|
| 226 | for events in ([event("0-git", "started")] * 2,
|
| 227 | outcome("0-git") + [outcome("0-git")[1]],
|
| 228 | [event("unknown", "started")],
|
| 229 | [dict(event("0-git", "started"), plan_sha256="wrong")],
|
| 230 | [outcome("0-git")[1]],
|
| 231 | [event("0-git", "other")]):
|
| 232 | with self.subTest(events=events), self.assertRaises(ValueError):
|
| 233 | self.report(plan(), events)
|
| 234 |
|
| 235 | def test_contradictory_result_does_not_count_as_success(self):
|
| 236 | events = outcome("0-git")
|
| 237 | events[1]["row"]["outcome"] = "fail"
|
| 238 | with self.assertRaises(ValueError):
|
| 239 | self.report(plan(), events)
|
| 240 | events = outcome("0-git")
|
| 241 | events[1]["row"]["agent"] = {"model": "unplanned"}
|
| 242 | with self.assertRaises(ValueError):
|
| 243 | self.report(plan(), events)
|
| 244 |
|
| 245 | def test_mock_plan_never_becomes_claim_eligible(self):
|
| 246 | configuration = plan(1)
|
| 247 | for trial in configuration["schedule"]:
|
| 248 | trial.update(agent="mock", model=None)
|
| 249 | summary = self.report(configuration, outcome("0-git") + outcome("0-oak"))
|
| 250 | self.assertFalse(summary["claim_eligible"])
|
| 251 | self.assertIn("mock_agent_measurements_are_instrument_checks", summary["claim_ineligibility_reasons"])
|
| 252 |
|
| 253 | def test_duplicate_scheduled_subject_block_is_rejected(self):
|
| 254 | configuration = plan()
|
| 255 | duplicate = copy.deepcopy(configuration["schedule"][0])
|
| 256 | duplicate["trial_id"] = "new-id-same-block-subject"
|
| 257 | configuration["schedule"].append(duplicate)
|
| 258 | with self.assertRaises(ValueError):
|
| 259 | self.report(configuration, [])
|
| 260 |
|
| 261 | def test_runtime_environment_and_version_drift_cannot_pool_or_pair(self):
|
| 262 | events = outcome("0-git") + outcome("1-git")
|
| 263 | events[-1]["row"]["agent"] = {"environment": "local-default"}
|
| 264 | with self.assertRaisesRegex(ValueError, "incompatible campaign population"):
|
| 265 | self.report(plan(), events)
|
| 266 |
|
| 267 | def test_row_cannot_be_reassigned_to_another_trial(self):
|
| 268 | for field, value in (("task_id", "other"), ("bench_id", "other"), ("run", 99)):
|
| 269 | events = outcome("0-git")
|
| 270 | events[-1]["row"][field] = value
|
| 271 | with self.subTest(field=field), self.assertRaisesRegex(ValueError, "row trial identity"):
|
| 272 | self.report(plan(), events)
|
| 273 | events = outcome("0-git")
|
| 274 | events[-1]["outcome"] = "fail"
|
| 275 | with self.assertRaisesRegex(ValueError, "event outcome contradicts"):
|
| 276 | self.report(plan(), events)
|
| 277 |
|
| 278 | def test_efficiency_is_paired_and_only_joint_success_with_same_sources(self):
|
| 279 | events = outcome("0-git", tokens=10) + outcome("0-oak", tokens=20) + outcome("1-git", tokens=100) + outcome("1-oak", tokens=110)
|
| 280 | for entry in events:
|
| 281 | if "row" in entry:
|
| 282 | entry["row"]["token_metrics"]["token_source"] = "provider_reported"
|
| 283 | entry["row"]["wall_clock_metrics"]["measurement_source"] = "outer_cli"
|
| 284 | pair = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]
|
| 285 | tokens = pair["success_conditioned_efficiency"]["reported_tokens"]
|
| 286 | self.assertEqual(tokens["candidate_minus_reference_mean_delta"], 10)
|
| 287 | self.assertEqual(tokens["confidence_interval"], [10, 10])
|
| 288 | self.assertEqual(tokens["measured_pairs"], 2)
|
| 289 | events[-1]["row"]["token_metrics"]["token_source"] = "other_convention"
|
| 290 | tokens = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]["success_conditioned_efficiency"]["reported_tokens"]
|
| 291 | self.assertEqual(tokens["measured_pairs"], 1)
|
| 292 | self.assertEqual(tokens["incompatible_or_unknown_source_pairs"], 1)
|
| 293 | self.assertIsNone(tokens["confidence_interval"])
|
| 294 | events[-1]["row"].update(success=False, outcome="fail")
|
| 295 | tokens = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]["success_conditioned_efficiency"]["reported_tokens"]
|
| 296 | self.assertEqual(tokens["jointly_successful_pairs"], 1)
|
| 297 | events = outcome("0-git") + outcome("0-oak")
|
| 298 | events[-1]["row"]["oracle_version"] = "different"
|
| 299 | summary = self.report(plan(), events)
|
| 300 | pair = summary["groups"][0]["paired_comparisons"][0]
|
| 301 | self.assertTrue(pair["comparison_refused"])
|
| 302 | self.assertEqual(pair["incompatible_evidence_fields"], ["oracle_version"])
|
| 303 | self.assertEqual(pair["complete_pairs"], 0)
|
| 304 | self.assertIsNone(pair["success_delta_percentage_points"])
|
| 305 | self.assertIn("paired_evidence_incompatible", summary["claim_ineligibility_reasons"])
|
| 306 |
|
| 307 |
|
| 308 | if __name__ == "__main__":
|
| 309 | unittest.main()
|