Log in
benchmarksPublic

Viewing files at commit 3d84c25e26dbCommit detailsView latest

Browse files
tests/test_campaign_report.py 250 lines · 14.3 KB · python
1
import copy
2
from pathlib import Path
3
import sys
4
import unittest
5
6
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
7
from oakbench.campaign_report import report_campaign, render_markdown
8
9
10
def plan(repetitions=2):
11
    return {"schema_version": 1, "campaign_id": "example", "plan_sha256": "digest", "manifest": {},
12
            "schedule": [dict(trial_id=f"{rep}-{subject}", block_id=f"block-{rep}", ordinal=rep * 2 + index,
13
                              repetition=rep, workflow="wide_config_refactor", instruction_level="zero-shot",
14
                              agent="codex", model="pinned-model", subject=subject)
15
                         for rep in range(repetitions) for index, subject in enumerate(("git", "oak"))]}
16
17
18
def event(trial_id, kind, **extra):
19
    return dict(event=kind, trial_id=trial_id, plan_sha256="digest", timestamp_utc="2026-09-10T00:00:00Z", **extra)
20
21
22
def outcome(trial_id, name="pass", tokens=10):
23
    return [event(trial_id, "started"), event(trial_id, "finished", row={
24
        "outcome": name, "success": name == "pass", "wall_clock_metrics": {"agent_active_ms": 20, "total_ms": 30},
25
        "token_metrics": {"total_tokens_reported": tokens, "cost_weighted_total": 40},
26
        "turn_metrics": {"assistant_turns_total": 2}, "tool_call_metrics": {"tool_calls_total": 3},
27
    })]
28
29
30
class CampaignReportTest(unittest.TestCase):
31
    def report(self, configuration, events):
32
        return report_campaign(configuration, events, bootstrap_samples=100, seed=7)
33
34
    def test_schedule_and_started_denominators_survive_interruption(self):
35
        summary = self.report(plan(), outcome("0-git") + outcome("0-oak", "fail") + [event("1-git", "started")])
36
        self.assertEqual(summary["counts"], dict(scheduled=4, started=3, finished=2, budget_truncated=0, success=1, failure=1,
37
                         timeout=0, infrastructure_failure=0, interrupted=1, skipped=0, unstarted=1))
38
        subjects = {item["subject"]: item for item in summary["groups"][0]["subjects"]}
39
        self.assertEqual(subjects["git"]["verified_success_fraction_of_started"], .5)
40
        self.assertIsNone(subjects["git"]["success_wilson_interval"])
41
        self.assertIsNone(subjects["git"]["resources"]["reported_tokens"]["complete_total"])
42
        self.assertEqual(summary["groups"][0]["paired_comparisons"][0]["incomplete_pairs"], 1)
43
        self.assertFalse(summary["claim_eligible"])
44
45
    def test_legacy_shortened_timeout_is_not_a_complete_pair(self):
46
        configuration = plan(1)
47
        configuration["manifest"]["trial_timeout_seconds"] = 60
48
        events = outcome("0-git") + outcome("0-oak", "timeout")
49
        events[0]["timeout_seconds"] = 60
50
        events[2]["timeout_seconds"] = .15
51
        summary = self.report(configuration, events)
52
        self.assertEqual(summary["counts"]["budget_truncated"], 1)
53
        self.assertEqual(summary["counts"]["interrupted"], 1)
54
        pair = summary["groups"][0]["paired_comparisons"][0]
55
        self.assertEqual(pair["complete_pairs"], 0)
56
        self.assertIsNone(pair["success_delta_percentage_points"])
57
58
    def test_missing_instrument_is_unmeasured_not_a_paired_failure(self):
59
        events = outcome("0-git", "error") + outcome("0-oak")
60
        events[1]["row"].update(returncode=77, skip_reason="required oracle unavailable")
61
        summary = self.report(plan(1), events)
62
        self.assertEqual(summary["counts"]["skipped"], 1)
63
        self.assertEqual(summary["counts"]["started"], 2)
64
        pair = summary["groups"][0]["paired_comparisons"][0]
65
        self.assertEqual(pair["complete_pairs"], 0)
66
        self.assertIsNone(pair["success_delta_percentage_points"])
67
        git = next(subject for subject in summary["groups"][0]["subjects"] if subject["subject"] == "git")
68
        self.assertIsNone(git["success_wilson_interval"])
69
70
    def test_fixture_invalidated_trial_is_not_a_complete_paired_failure(self):
71
        events = outcome("0-git") + [event("0-oak", "started"), event("0-oak", "finished",
72
            outcome="error", measurement_invalidated=True, invalidation_reason="fixture content drift",
73
            rejected_row={"outcome": "pass", "success": True})]
74
        summary = self.report(plan(1), events)
75
        self.assertEqual(summary["counts"]["skipped"], 1)
76
        self.assertEqual(summary["invalidated_count"], 1)
77
        self.assertEqual(summary["invalidated_trials"][0]["reason"], "fixture content drift")
78
        self.assertTrue(summary["invalidated_trials"][0]["rejected_row_retained"])
79
        self.assertIn("0-oak: fixture content drift", render_markdown(summary))
80
        self.assertEqual(summary["counts"]["infrastructure_failure"], 0)
81
        pair = summary["groups"][0]["paired_comparisons"][0]
82
        self.assertEqual(pair["complete_pairs"], 0)
83
        self.assertIsNone(pair["success_delta_percentage_points"])
84
85
    def test_every_outcome_remains_visible(self):
86
        configuration = plan(4)
87
        events = (outcome("0-git") + outcome("0-oak", "fail") + outcome("1-git", "timeout") +
88
                  [event("1-oak", "started"), event("1-oak", "finished", outcome="error", error="crash"),
89
                   event("2-git", "started"), event("2-oak", "finished", outcome="skipped")])
90
        counts = self.report(configuration, events)["counts"]
91
        for field in ("success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped"):
92
            self.assertEqual(counts[field], 1)
93
        self.assertEqual(counts["unstarted"], 2)
94
        self.assertEqual(sum(counts[name] for name in ("success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped", "unstarted")), counts["scheduled"])
95
96
    def test_paired_bootstrap_preserves_block_pairs_and_is_seeded(self):
97
        events = outcome("0-git") + outcome("0-oak", "fail") + outcome("1-git", "fail") + outcome("1-oak")
98
        first = self.report(plan(), events)
99
        second = self.report(plan(), events)
100
        self.assertEqual(first, second)
101
        pair = first["groups"][0]["paired_comparisons"][0]
102
        self.assertEqual(pair["success_delta_percentage_points"], 0)
103
        self.assertEqual(pair["confidence_interval"], [-100, 100])
104
        self.assertEqual(pair["complete_pairs"], 2)
105
        self.assertFalse(pair["multiplicity_adjusted"])
106
107
    def test_singleton_pair_does_not_get_zero_width_interval(self):
108
        pair = self.report(plan(1), outcome("0-git") + outcome("0-oak"))["groups"][0]["paired_comparisons"][0]
109
        self.assertEqual(pair["success_delta_percentage_points"], 0)
110
        self.assertIsNone(pair["confidence_interval"])
111
112
    def test_missing_usage_does_not_become_zero_or_complete_total(self):
113
        summary = self.report(plan(), outcome("0-git", tokens=None) + outcome("1-git", tokens=0))
114
        values = summary["groups"][0]["subjects"][0]["resources"]["reported_tokens"]
115
        self.assertEqual(values["measured_n"], 1)
116
        self.assertEqual(values["missing_n"], 1)
117
        self.assertEqual(values["observed_sum"], 0)
118
        self.assertIsNone(values["complete_total"])
119
        self.assertIsNone(values["observed_cost_per_success"])
120
        self.assertEqual(values["success_conditioned"]["median"], 0)
121
122
    def test_partial_capture_counters_are_not_complete_usage(self):
123
        for changes in ({"outcome": "timeout", "success": False}, {"output_truncated": True},
124
                        {"source": {"agent_execution": {"capture_complete": False}}},
125
                        {"source": {"agent_execution": {"usage_complete": False}}},
126
                        {"source": {"agent_execution": {"counters_coverage": "observed_lower_bounds"}}},
127
                        {"source": {"agent_execution": {"output_limited": True}}}):
128
            events = outcome("0-git")
129
            events[-1]["row"].update(changes)
130
            resources = self.report(plan(1), events)["groups"][0]["subjects"][0]["resources"]
131
            for metric in ("reported_tokens", "assistant_turns", "tool_calls"):
132
                self.assertIsNone(resources[metric]["complete_total"])
133
                self.assertEqual(resources[metric]["missing_n"], 1)
134
            self.assertEqual(resources["agent_active_ms"]["complete_total"], 20)
135
136
    def test_capture_error_row_keeps_infrastructure_denominator_and_time(self):
137
        events = outcome("0-git")
138
        events[-1]["outcome"] = "error"
139
        events[-1]["row"].update(outcome="error", success=False, output_truncated=True)
140
        summary = self.report(plan(1), events)
141
        self.assertEqual(summary["counts"]["infrastructure_failure"], 1)
142
        resources = summary["groups"][0]["subjects"][0]["resources"]
143
        self.assertEqual(resources["agent_active_ms"]["complete_total"], 20)
144
        self.assertIsNone(resources["tool_calls"]["complete_total"])
145
146
    def test_failures_contribute_resource_cost_but_not_success_latency(self):
147
        events = outcome("0-git", tokens=10) + outcome("1-git", "fail", tokens=50)
148
        subject = self.report(plan(), events)["groups"][0]["subjects"][0]
149
        values = subject["resources"]["reported_tokens"]
150
        self.assertEqual(values["complete_total"], 60)
151
        self.assertEqual(values["observed_cost_per_success"], 60)
152
        self.assertEqual(values["success_conditioned"]["median"], 10)
153
        self.assertAlmostEqual(subject["success_wilson_interval"][0], .094531, places=5)
154
        self.assertAlmostEqual(subject["success_wilson_interval"][1], .905469, places=5)
155
156
    def test_zero_success_or_no_attempts_keeps_undefined_metrics_null(self):
157
        summary = self.report(plan(1), outcome("0-git", "fail"))
158
        git, oak = summary["groups"][0]["subjects"]
159
        self.assertIsNone(git["resources"]["reported_tokens"]["observed_cost_per_success"])
160
        self.assertIsNone(oak["verified_success_fraction_of_started"])
161
        self.assertIsNone(oak["resources"]["reported_tokens"]["observed_sum"])
162
        self.assertIn("unmeasured", render_markdown(summary))
163
164
    def test_instruction_and_model_conditions_never_pool(self):
165
        configuration = plan()
166
        configuration["schedule"][2]["instruction_level"] = "full-docs"
167
        configuration["schedule"][3]["model"] = "other-model"
168
        summary = self.report(configuration, [])
169
        self.assertEqual(len(summary["groups"]), 3)
170
171
    def test_duplicate_or_foreign_journal_evidence_rejected(self):
172
        for events in ([event("0-git", "started")] * 2,
173
                       outcome("0-git") + [outcome("0-git")[1]],
174
                       [event("unknown", "started")],
175
                       [dict(event("0-git", "started"), plan_sha256="wrong")],
176
                       [outcome("0-git")[1]],
177
                       [event("0-git", "other")]):
178
            with self.subTest(events=events), self.assertRaises(ValueError):
179
                self.report(plan(), events)
180
181
    def test_contradictory_result_does_not_count_as_success(self):
182
        events = outcome("0-git")
183
        events[1]["row"]["outcome"] = "fail"
184
        with self.assertRaises(ValueError):
185
            self.report(plan(), events)
186
        events = outcome("0-git")
187
        events[1]["row"]["agent"] = {"model": "unplanned"}
188
        with self.assertRaises(ValueError):
189
            self.report(plan(), events)
190
191
    def test_mock_plan_never_becomes_claim_eligible(self):
192
        configuration = plan(1)
193
        for trial in configuration["schedule"]:
194
            trial.update(agent="mock", model=None)
195
        summary = self.report(configuration, outcome("0-git") + outcome("0-oak"))
196
        self.assertFalse(summary["claim_eligible"])
197
        self.assertIn("mock_agent_measurements_are_instrument_checks", summary["claim_ineligibility_reasons"])
198
199
    def test_duplicate_scheduled_subject_block_is_rejected(self):
200
        configuration = plan()
201
        duplicate = copy.deepcopy(configuration["schedule"][0])
202
        duplicate["trial_id"] = "new-id-same-block-subject"
203
        configuration["schedule"].append(duplicate)
204
        with self.assertRaises(ValueError):
205
            self.report(configuration, [])
206
207
    def test_runtime_environment_and_version_drift_cannot_pool_or_pair(self):
208
        events = outcome("0-git") + outcome("1-git")
209
        events[-1]["row"]["agent"] = {"environment": "local-default"}
210
        with self.assertRaisesRegex(ValueError, "incompatible campaign population"):
211
            self.report(plan(), events)
212
213
    def test_row_cannot_be_reassigned_to_another_trial(self):
214
        for field, value in (("task_id", "other"), ("bench_id", "other"), ("run", 99)):
215
            events = outcome("0-git")
216
            events[-1]["row"][field] = value
217
            with self.subTest(field=field), self.assertRaisesRegex(ValueError, "row trial identity"):
218
                self.report(plan(), events)
219
        events = outcome("0-git")
220
        events[-1]["outcome"] = "fail"
221
        with self.assertRaisesRegex(ValueError, "event outcome contradicts"):
222
            self.report(plan(), events)
223
224
    def test_efficiency_is_paired_and_only_joint_success_with_same_sources(self):
225
        events = outcome("0-git", tokens=10) + outcome("0-oak", tokens=20) + outcome("1-git", tokens=100) + outcome("1-oak", tokens=110)
226
        for entry in events:
227
            if "row" in entry:
228
                entry["row"]["token_metrics"]["token_source"] = "provider_reported"
229
                entry["row"]["wall_clock_metrics"]["measurement_source"] = "outer_cli"
230
        pair = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]
231
        tokens = pair["success_conditioned_efficiency"]["reported_tokens"]
232
        self.assertEqual(tokens["candidate_minus_reference_mean_delta"], 10)
233
        self.assertEqual(tokens["confidence_interval"], [10, 10])
234
        self.assertEqual(tokens["measured_pairs"], 2)
235
        events[-1]["row"]["token_metrics"]["token_source"] = "other_convention"
236
        tokens = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]["success_conditioned_efficiency"]["reported_tokens"]
237
        self.assertEqual(tokens["measured_pairs"], 1)
238
        self.assertEqual(tokens["incompatible_or_unknown_source_pairs"], 1)
239
        self.assertIsNone(tokens["confidence_interval"])
240
        events[-1]["row"].update(success=False, outcome="fail")
241
        tokens = self.report(plan(), events)["groups"][0]["paired_comparisons"][0]["success_conditioned_efficiency"]["reported_tokens"]
242
        self.assertEqual(tokens["jointly_successful_pairs"], 1)
243
        events = outcome("0-git") + outcome("0-oak")
244
        events[-1]["row"]["oracle_version"] = "different"
245
        with self.assertRaisesRegex(ValueError, "incompatible paired evidence"):
246
            self.report(plan(), events)
247
248
249
if __name__ == "__main__":
250
    unittest.main()