Log in
benchmarksPublic

Viewing files at commit 1c9b819ab86bCommit detailsView latest

Browse files
tests/test_campaign_report.py 282 lines · 16.5 KB · python
1
import copy
2
from pathlib import Path
3
import sys
4
import unittest
5
6
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
7
from oakbench.campaign_report import report_campaign, render_markdown
8
9
10
def plan(repetitions=2):
11
    return {"schema_version": 1, "campaign_id": "example", "plan_sha256": "digest", "manifest": {},
12
            "schedule": [dict(trial_id=f"{rep}-{subject}", block_id=f"block-{rep}", ordinal=rep * 2 + index,
13
                              repetition=rep, workflow="wide_config_refactor", instruction_level="zero-shot",
14
                              agent="codex", model="pinned-model", subject=subject)
15
                         for rep in range(repetitions) for index, subject in enumerate(("git", "oak"))]}
16
17
18
def event(trial_id, kind, **extra):
19
    return dict(event=kind, trial_id=trial_id, plan_sha256="digest", timestamp_utc="2026-09-10T00:00:00Z", **extra)
20
21
22
def outcome(trial_id, name="pass", tokens=10):
23
    return [event(trial_id, "started"), event(trial_id, "finished", row={
24
        "outcome": name, "success": name == "pass", "wall_clock_metrics": {"agent_active_ms": 20, "total_ms": 30},
25
        "token_metrics": {"total_tokens_reported": tokens, "cost_weighted_total": 40},
26
        "turn_metrics": {"assistant_turns_total": 2}, "tool_call_metrics": {"tool_calls_total": 3},
27
    })]
28
29
30
class CampaignReportTest(unittest.TestCase):
31
    def report(self, configuration, events):
32
        return report_campaign(configuration, events, bootstrap_samples=100, seed=7)
33
34
    def test_schedule_and_started_denominators_survive_interruption(self):
35
        summary = self.report(plan(), outcome("0-git") + outcome("0-oak", "fail") + [event("1-git", "started")])
36
        self.assertEqual(summary["counts"], dict(scheduled=4, started=3, finished=2, budget_truncated=0, success=1, failure=1,
37
                         timeout=0, infrastructure_failure=0, interrupted=1, skipped=0, unstarted=1))
38
        subjects = {item["subject"]: item for item in summary["groups"][0]["subjects"]}
39
        self.assertEqual(subjects["git"]["verified_success_fraction_of_started"], .5)
40
        self.assertIsNone(subjects["git"]["success_wilson_interval"])
41
        self.assertIsNone(subjects["git"]["resources"]["reported_tokens"]["complete_total"])
42
        self.assertEqual(summary["groups"][0]["paired_comparisons"][0]["incomplete_pairs"], 1)
43
        self.assertFalse(summary["claim_eligible"])
44
45
    def test_legacy_shortened_timeout_is_not_a_complete_pair(self):
46
        configuration = plan(1)
47
        configuration["manifest"]["trial_timeout_seconds"] = 60
48
        events = outcome("0-git") + outcome("0-oak", "timeout")
49
        events[0]["timeout_seconds"] = 60
50
        events[2]["timeout_seconds"] = .15
51
        summary = self.report(configuration, events)
52
        self.assertEqual(summary["counts"]["budget_truncated"], 1)
53
        self.assertEqual(summary["counts"]["interrupted"], 1)
54
        pair = summary["groups"][0]["paired_comparisons"][0]
55
        self.assertEqual(pair["complete_pairs"], 0)
56
        self.assertIsNone(pair["success_delta_percentage_points"])
57
58
    def test_missing_instrument_is_unmeasured_not_a_paired_failure(self):
59
        events = outcome("0-git", "error") + outcome("0-oak")
60
        events[1]["row"].update(returncode=77, skip_reason="required oracle unavailable")
61
        summary = self.report(plan(1), events)
62
        self.assertEqual(summary["counts"]["skipped"], 1)
63
        self.assertEqual(summary["counts"]["started"], 2)
64
        pair = summary["groups"][0]["paired_comparisons"][0]
65
        self.assertEqual(pair["complete_pairs"], 0)
66
        self.assertIsNone(pair["success_delta_percentage_points"])
67
        git = next(subject for subject in summary["groups"][0]["subjects"] if subject["subject"] == "git")
68
        self.assertIsNone(git["success_wilson_interval"])
69
70
    def test_fixture_invalidated_trial_is_not_a_complete_paired_failure(self):
71
        events = outcome("0-git") + [event("0-oak", "started"), event("0-oak", "finished",
72
            outcome="error", measurement_invalidated=True, invalidation_reason="fixture content drift",
73
            rejected_row={"outcome": "pass", "success": True})]
74
        summary = self.report(plan(1), events)
75
        self.assertEqual(summary["counts"]["skipped"], 1)
76
        self.assertEqual(summary["invalidated_count"], 1)
77
        self.assertEqual(summary["invalidated_trials"][0]["reason"], "fixture content drift")
78
        self.assertTrue(summary["invalidated_trials"][0]["rejected_row_retained"])
79
        self.assertIn("0-oak: fixture content drift", render_markdown(summary))
80
        self.assertEqual(summary["counts"]["infrastructure_failure"], 0)
81
        pair = summary["groups"][0]["paired_comparisons"][0]
82
        self.assertEqual(pair["complete_pairs"], 0)
83
        self.assertIsNone(pair["success_delta_percentage_points"])
84
85
    def test_infrastructure_attempts_stay_resources_not_success_pairs(self):
86
        events = outcome("0-git") + outcome("1-git")
87
        for trial in ("0-oak", "1-oak"):
88
            events.extend([event(trial, "started"), event(trial, "finished", outcome="error", error="harness setup failed")])
89
        summary = self.report(plan(2), events)
90
        pair = summary["groups"][0]["paired_comparisons"][0]
91
        self.assertEqual(pair["complete_pairs"], 0)
92
        self.assertIsNone(pair["success_delta_percentage_points"])
93
        self.assertIn("schedule_or_attempt_accounting_incomplete", summary["claim_ineligibility_reasons"])
94
        oak = next(item for item in summary["groups"][0]["subjects"] if item["subject"] == "oak")
95
        self.assertIsNone(oak["success_wilson_interval"])
96
        self.assertEqual(oak["resources"]["reported_tokens"]["attempted_n"], 2)
97
98
    def test_measured_precommit_failure_keeps_pair_but_success_proof_classes_must_match(self):
99
        events = outcome("0-git") + outcome("0-oak", "fail")
100
        events[1]["row"].update(workflow_integrity_passed=True, workflow_integrity_evidence_class="independent_committed_tree")
101
        events[3]["row"].update(workflow_integrity_passed=False, workflow_integrity_evidence_class="trusted_worktree_payload", agent_launched=True, outcome_attribution="agent")
102
        summary = self.report(plan(1), events)
103
        pair = summary["groups"][0]["paired_comparisons"][0]
104
        self.assertEqual(pair["complete_pairs"], 1)
105
        self.assertEqual(pair["success_delta_percentage_points"], -100)
106
        events[3]["row"].update(outcome="pass", success=True, workflow_integrity_passed=True)
107
        pair = self.report(plan(1), events)["groups"][0]["paired_comparisons"][0]
108
        self.assertTrue(pair["comparison_refused"])
109
        self.assertIn("workflow_integrity_evidence_class", pair["incompatible_evidence_fields"])
110
        self.assertEqual(pair["complete_pairs"], 0)
111
112
    def test_every_outcome_remains_visible(self):
113
        configuration = plan(4)
114
        events = (outcome("0-git") + outcome("0-oak", "fail") + outcome("1-git", "timeout") +
115
                  [event("1-oak", "started"), event("1-oak", "finished", outcome="error", error="crash"),
116
                   event("2-git", "started"), event("2-oak", "finished", outcome="skipped")])
117
        counts = self.report(configuration, events)["counts"]
118
        for field in ("success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped"):
119
            self.assertEqual(counts[field], 1)
120
        self.assertEqual(counts["unstarted"], 2)
121
        self.assertEqual(sum(counts[name] for name in ("success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped", "unstarted")), counts["scheduled"])
122
123
    def test_paired_bootstrap_preserves_block_pairs_and_is_seeded(self):
124
        events = outcome("0-git") + outcome("0-oak", "fail") + outcome("1-git", "fail") + outcome("1-oak")
125
        first = self.report(plan(), events)
126
        second = self.report(plan(), events)
127
        self.assertEqual(first, second)
128
        pair = first["groups"][0]["paired_comparisons"][0]
129
        self.assertEqual(pair["success_delta_percentage_points"], 0)
130
        self.assertEqual(pair["confidence_interval"], [-100, 100])
131
        self.assertEqual(pair["complete_pairs"], 2)
132
        self.assertFalse(pair["multiplicity_adjusted"])
133
134
    def test_singleton_pair_does_not_get_zero_width_interval(self):
135
        pair = self.report(plan(1), outcome("0-git") + outcome("0-oak"))["groups"][0]["paired_comparisons"][0]
136
        self.assertEqual(pair["success_delta_percentage_points"], 0)
137
        self.assertIsNone(pair["confidence_interval"])
138
139
    def test_missing_usage_does_not_become_zero_or_complete_total(self):
140
        summary = self.report(plan(), outcome("0-git", tokens=None) + outcome("1-git", tokens=0))
141
        values = summary["groups"][0]["subjects"][0]["resources"]["reported_tokens"]
142
        self.assertEqual(values["measured_n"], 1)
143
        self.assertEqual(values["missing_n"], 1)
144
        self.assertEqual(values["observed_sum"], 0)
145
        self.assertIsNone(values["complete_total"])
146
        self.assertIsNone(values["observed_cost_per_success"])
147
        self.assertEqual(values["success_conditioned"]["median"], 0)
148
149
    def test_partial_capture_counters_are_not_complete_usage(self):
150
        for changes in ({"outcome": "timeout", "success": False}, {"output_truncated": True},
151
                        {"source": {"agent_execution": {"capture_complete": False}}},
152
                        {"source": {"agent_execution": {"usage_complete": False}}},
153
                        {"source": {"agent_execution": {"counters_coverage": "observed_lower_bounds"}}},
154
                        {"source": {"agent_execution": {"output_limited": True}}}):
155
            events = outcome("0-git")
156
            events[-1]["row"].update(changes)
157
            resources = self.report(plan(1), events)["groups"][0]["subjects"][0]["resources"]
158
            for metric in ("reported_tokens", "assistant_turns", "tool_calls"):
159
                self.assertIsNone(resources[metric]["complete_total"])
160
                self.assertEqual(resources[metric]["missing_n"], 1)
161
            self.assertEqual(resources["agent_active_ms"]["complete_total"], 20)
162
163
    def test_capture_error_row_keeps_infrastructure_denominator_and_time(self):
164
        events = outcome("0-git")
165
        events[-1]["outcome"] = "error"
166
        events[-1]["row"].update(outcome="error", success=False, output_truncated=True)
167
        summary = self.report(plan(1), events)
168
        self.assertEqual(summary["counts"]["infrastructure_failure"], 1)
169
        resources = summary["groups"][0]["subjects"][0]["resources"]
170
        self.assertEqual(resources["agent_active_ms"]["complete_total"], 20)
171
        self.assertIsNone(resources["tool_calls"]["complete_total"])
172
173
    def test_failures_contribute_resource_cost_but_not_success_latency(self):
174
        events = outcome("0-git", tokens=10) + outcome("1-git", "fail", tokens=50)
175
        subject = self.report(plan(), events)["groups"][0]["subjects"][0]
176
        values = subject["resources"]["reported_tokens"]
177
        self.assertEqual(values["complete_total"], 60)
178
        self.assertEqual(values["observed_cost_per_success"], 60)
179
        self.assertEqual(values["success_conditioned"]["median"], 10)
180
        self.assertAlmostEqual(subject["success_wilson_interval"][0], .094531, places=5)
181
        self.assertAlmostEqual(subject["success_wilson_interval"][1], .905469, places=5)
182
183
    def test_zero_success_or_no_attempts_keeps_undefined_metrics_null(self):
184
        summary = self.report(plan(1), outcome("0-git", "fail"))
185
        git, oak = summary["groups"][0]["subjects"]
186
        self.assertIsNone(git["resources"]["reported_tokens"]["observed_cost_per_success"])
187
        self.assertIsNone(oak["verified_success_fraction_of_started"])
188
        self.assertIsNone(oak["resources"]["reported_tokens"]["observed_sum"])
189
        self.assertIn("unmeasured", render_markdown(summary))
190
191
    def test_instruction_and_model_conditions_never_pool(self):
192
        configuration = plan()
193
        configuration["schedule"][2]["instruction_level"] = "full-docs"
194
        configuration["schedule"][3]["model"] = "other-model"
195
        summary = self.report(configuration, [])
196
        self.assertEqual(len(summary["groups"]), 3)
197
198
    def test_duplicate_or_foreign_journal_evidence_rejected(self):
199
        for events in ([event("0-git", "started")] * 2,
200
                       outcome("0-git") + [outcome("0-git")[1]],
201
                       [event("unknown", "started")],
202
                       [dict(event("0-git", "started"), plan_sha256="wrong")],
203
                       [outcome("0-git")[1]],
204
                       [event("0-git", "other")]):
205
            with self.subTest(events=events), self.assertRaises(ValueError):
206
                self.report(plan(), events)
207
208
    def test_contradictory_result_does_not_count_as_success(self):
209
        events = outcome("0-git")
210
        events[1]["row"]["outcome"] = "fail"
211
        with self.assertRaises(ValueError):
212
            self.report(plan(), events)
213
        events = outcome("0-git")
214
        events[1]["row"]["agent"] = {"model": "unplanned"}
215
        with self.assertRaises(ValueError):
216
            self.report(plan(), events)
217
218
    def test_mock_plan_never_becomes_claim_eligible(self):
219
        configuration = plan(1)
220
        for trial in configuration["schedule"]:
221
            trial.update(agent="mock", model=None)
222
        summary = self.report(configuration, outcome("0-git") + outcome("0-oak"))
223
        self.assertFalse(summary["claim_eligible"])
224
        self.assertIn("mock_agent_measurements_are_instrument_checks", summary["claim_ineligibility_reasons"])
225
226
    def test_duplicate_scheduled_subject_block_is_rejected(self):
227
        configuration = plan()
228
        duplicate = copy.deepcopy(configuration["schedule"][0])
229
        duplicate["trial_id"] = "new-id-same-block-subject"
230
        configuration["schedule"].append(duplicate)
231
        with self.assertRaises(ValueError):
232
            self.report(configuration, [])
233
234
    def test_runtime_environment_and_version_drift_cannot_pool_or_pair(self):
235
        events = outcome("0-git") + outcome("1-git")
236
        events[-1]["row"]["agent"] = {"environment": "local-default"}
237
        with self.assertRaisesRegex(ValueError, "incompatible campaign population"):
238
            self.report(plan(), events)
239
240
    def test_row_cannot_be_reassigned_to_another_trial(self):
241
        for field, value in (("task_id", "other"), ("bench_id", "other"), ("run", 99)):
242
            events = outcome("0-git")
243
            events[-1]["row"][field] = value
244
            with self.subTest(field=field), self.assertRaisesRegex(ValueError, "row trial identity"):
245
                self.report(plan(), events)
246
        events = outcome("0-git")
247
        events[-1]["outcome"] = "fail"
248
        with self.assertRaisesRegex(ValueError, "event outcome contradicts"):
249
            self.report(plan(), events)
250
251
    def test_efficiency_is_paired_and_only_joint_success_with_same_sources(self):
252
        events = outcome("0-git", tokens=10) + outcome("0-oak", tokens=20) + outcome("1-git", tokens=100) + outcome("1-oak", tokens=110)
253
        for entry in events:
254
            if "row" in entry:
255
                entry["row"]["token_metrics"]["token_source"] = "provider_reported"
256
                entry["row"]["wall_clock_metrics"]["measurement_source"] = "outer_cli"
257
        pair = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]
258
        tokens = pair["success_conditioned_efficiency"]["reported_tokens"]
259
        self.assertEqual(tokens["candidate_minus_reference_mean_delta"], 10)
260
        self.assertEqual(tokens["confidence_interval"], [10, 10])
261
        self.assertEqual(tokens["measured_pairs"], 2)
262
        events[-1]["row"]["token_metrics"]["token_source"] = "other_convention"
263
        tokens = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]["success_conditioned_efficiency"]["reported_tokens"]
264
        self.assertEqual(tokens["measured_pairs"], 1)
265
        self.assertEqual(tokens["incompatible_or_unknown_source_pairs"], 1)
266
        self.assertIsNone(tokens["confidence_interval"])
267
        events[-1]["row"].update(success=False, outcome="fail")
268
        tokens = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]["success_conditioned_efficiency"]["reported_tokens"]
269
        self.assertEqual(tokens["jointly_successful_pairs"], 1)
270
        events = outcome("0-git") + outcome("0-oak")
271
        events[-1]["row"]["oracle_version"] = "different"
272
        summary = self.report(plan(), events)
273
        pair = summary["groups"][0]["paired_comparisons"][0]
274
        self.assertTrue(pair["comparison_refused"])
275
        self.assertEqual(pair["incompatible_evidence_fields"], ["oracle_version"])
276
        self.assertEqual(pair["complete_pairs"], 0)
277
        self.assertIsNone(pair["success_delta_percentage_points"])
278
        self.assertIn("paired_evidence_incompatible", summary["claim_ineligibility_reasons"])
279
280
281
if __name__ == "__main__":
282
    unittest.main()