Log in
benchmarksPublic

Viewing files at commit 2caecb1e6549Commit detailsView latest

Browse files
tests/test_review_fixes.py 287 lines · 11.8 KB · python
1
"""Regression tests for the 2026-06-10 external review fixes.
2
3
Each test pins a bug the review reproduced: fixture identity poisoning,
4
fabricated provider-token zeros, tool-call double counting, path-qualified
5
VCS classification, and skip-row contract coherence.
6
"""
7
8
from __future__ import annotations
9
10
import importlib.util
11
import json
12
import shutil
13
import sys
14
import tempfile
15
import unittest
16
from pathlib import Path
17
18
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
19
20
from oakbench.classify import classify_command, vcs_info_family  # noqa: E402
21
from oakbench.rows import validate_rows  # noqa: E402
22
23
24
def load_script(name: str):
25
    path = Path(__file__).resolve().parents[1] / "scripts" / f"{name}.py"
26
    spec = importlib.util.spec_from_file_location(name, path)
27
    module = importlib.util.module_from_spec(spec)
28
    sys.modules[name] = module
29
    spec.loader.exec_module(module)
30
    return module
31
32
33
class FixtureIdentityTest(unittest.TestCase):
34
    """A smoke run must never poison a later run with a different shape."""
35
36
    def setUp(self) -> None:
37
        self.bench = load_script("bench")
38
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-fixture-test-"))
39
40
    def tearDown(self) -> None:
41
        shutil.rmtree(self.tmp, ignore_errors=True)
42
43
    def test_different_shapes_get_different_fixture_dirs(self) -> None:
44
        small = self.bench.Scenario("same_name", 3, 64, 1, False, 1)
45
        large = self.bench.Scenario("same_name", 7, 64, 1, False, 1)
46
        dir_small = self.bench.ensure_fixture(self.tmp, small)
47
        dir_large = self.bench.ensure_fixture(self.tmp, large)
48
        self.assertNotEqual(dir_small, dir_large)
49
        self.assertEqual(sum(1 for _ in dir_small.rglob("*.txt")), 3)
50
        self.assertEqual(sum(1 for _ in dir_large.rglob("*.txt")), 7)
51
52
    def test_stale_marker_with_wrong_shape_rebuilds(self) -> None:
53
        scenario = self.bench.Scenario("stale", 4, 64, 1, False, 1)
54
        dest = self.bench.fixture_path(self.tmp, scenario)
55
        dest.mkdir(parents=True)
56
        (dest / ".fixture-ready").write_text("poisoned\n")
57
        rebuilt = self.bench.ensure_fixture(self.tmp, scenario)
58
        self.assertEqual(sum(1 for _ in rebuilt.rglob("*.txt")), 4)
59
60
    def test_no_profile_reuses_a_scenario_name_with_a_different_shape(self) -> None:
61
        """ADR-0005: same scenario name must mean the same byte shape everywhere."""
62
        shapes: dict[str, str] = {}
63
        for profile_scenarios in self.bench.PROFILES.values():
64
            for scenario in profile_scenarios:
65
                shape = self.bench.scenario_shape_id(scenario)
66
                if scenario.name in shapes:
67
                    self.assertEqual(
68
                        shapes[scenario.name],
69
                        shape,
70
                        f"scenario {scenario.name!r} has conflicting shapes across profiles",
71
                    )
72
                shapes[scenario.name] = shape
73
74
75
class CollectUsageHonestyTest(unittest.TestCase):
76
    def setUp(self) -> None:
77
        self.aw = load_script("agent_workflow")
78
79
    def test_total_only_stream_leaves_components_null(self) -> None:
80
        usage = self.aw.collect_usage([{"usage": {"total_tokens": 500}}])
81
        self.assertEqual(usage["total_tokens_reported"], 500)
82
        self.assertIsNone(usage["input_tokens_reported"])
83
        self.assertIsNone(usage["output_tokens_reported"])
84
        self.assertIsNone(usage["reasoning_tokens_reported"])
85
        self.assertIsNone(usage["cache_read_tokens_reported"])
86
87
    def test_no_usage_is_all_null(self) -> None:
88
        usage = self.aw.collect_usage([{"type": "text", "text": "hello"}])
89
        self.assertTrue(all(value is None for value in usage.values()))
90
91
    def test_components_sum_into_missing_total(self) -> None:
92
        usage = self.aw.collect_usage([{"usage": {"input_tokens": 100, "output_tokens": 30}}])
93
        self.assertEqual(usage["total_tokens_reported"], 130)
94
        self.assertEqual(usage["input_tokens_reported"], 100)
95
        self.assertIsNone(usage["reasoning_tokens_reported"])
96
97
    def test_codex_cached_input_not_weighted_twice(self) -> None:
98
        usage = {
99
            "input_tokens_reported": 111415,
100
            "output_tokens_reported": 1934,
101
            "reasoning_tokens_reported": 110,
102
            "cache_read_tokens_reported": 81920,
103
            "cache_write_tokens_reported": None,
104
        }
105
        self.assertEqual(
106
            self.aw.cost_weighted_total_from_usage(usage, cache_read_included_in_input=True),
107
            47907.0,
108
        )
109
110
    def test_claude_cache_write_is_weighted_as_ingested(self) -> None:
111
        usage = {
112
            "input_tokens_reported": 225,
113
            "output_tokens_reported": 1223,
114
            "reasoning_tokens_reported": None,
115
            "cache_read_tokens_reported": 95468,
116
            "cache_write_tokens_reported": 20059,
117
        }
118
        self.assertEqual(
119
            self.aw.cost_weighted_total_from_usage(usage, cache_read_included_in_input=False),
120
            35945.8,
121
        )
122
123
124
class ToolCallCountTest(unittest.TestCase):
125
    def test_claude_tool_use_counts_once(self) -> None:
126
        aw = load_script("agent_workflow")
127
        with (Path(__file__).parent / "fixtures" / "claude_stream.jsonl").open() as fh:
128
            events = [json.loads(line) for line in fh]
129
        tool_use_blocks = sum(
130
            1
131
            for event in events
132
            for item in aw.walk_json(event)
133
            if isinstance(item, dict) and item.get("type") == "tool_use"
134
        )
135
        counts, _ = aw.collect_tool_metrics("", "", events)
136
        self.assertEqual(counts["tool_calls_total"], tool_use_blocks)
137
138
    def test_vcs_attribution_survives_dedupe(self) -> None:
139
        aw = load_script("agent_workflow")
140
        events = [
141
            {
142
                "type": "assistant",
143
                "message": {
144
                    "content": [
145
                        {"type": "tool_use", "name": "Bash", "id": "t1", "input": {"command": "git status"}}
146
                    ]
147
                },
148
            }
149
        ]
150
        counts, _ = aw.collect_tool_metrics("", "", events)
151
        self.assertEqual(counts["tool_calls_total"], 1)
152
        self.assertEqual(counts["vcs_commands_total"], 1)
153
154
155
class PathQualifiedClassificationTest(unittest.TestCase):
156
    def test_absolute_path_binaries_classify_as_vcs(self) -> None:
157
        self.assertIn("vcs", classify_command("/usr/bin/git status"))
158
        self.assertIn("vcs", classify_command("../oak/target/release/oak commit --no-verify"))
159
160
    def test_no_substring_false_positives(self) -> None:
161
        self.assertNotIn("vcs", classify_command("digit 5"))
162
        self.assertNotIn("vcs", classify_command("mygit status"))
163
164
    def test_info_family_handles_paths(self) -> None:
165
        self.assertEqual(vcs_info_family("/usr/bin/git status --short"), "status")
166
        self.assertEqual(vcs_info_family("/opt/oak diff"), "diff")
167
168
169
class VcsMetricsConsistencyTest(unittest.TestCase):
170
    """Sub-counts and totals must come from one classification authority."""
171
172
    def test_subcounts_never_exceed_total(self) -> None:
173
        aw = load_script("agent_workflow")
174
        transcript = (
175
            "$ /usr/bin/git status\n"
176
            "$ git diff --stat\n"
177
            "$ git add .\n"
178
            "$ git commit -m fix\n"
179
            "$ ls -la\n"
180
        )
181
        metrics = aw.vcs_metrics_from_tools(None, transcript, "", 0, 1)
182
        self.assertEqual(metrics["commands_total"], 4)  # add counts in total only
183
        self.assertEqual(metrics["status_commands"], 1)  # path-qualified git seen
184
        self.assertEqual(metrics["diff_commands"], 1)
185
        self.assertEqual(metrics["snapshot_commands"], 1)
186
        family_sum = (
187
            metrics["status_commands"]
188
            + metrics["diff_commands"]
189
            + metrics["snapshot_commands"]
190
            + metrics["branch_commands"]
191
            + metrics["merge_commands"]
192
        )
193
        self.assertLessEqual(family_sum, metrics["commands_total"])
194
195
    def test_bytes_read_by_agent_is_null_not_transcript_proxy(self) -> None:
196
        from oakbench.classify import vcs_subcommand
197
198
        self.assertEqual(vcs_subcommand("/usr/bin/git add ."), "add")
199
        self.assertIsNone(vcs_subcommand("ls -la"))
200
201
    def test_unavailable_command_stream_is_null_not_zero(self) -> None:
202
        aw = load_script("agent_workflow")
203
        metrics = aw.vcs_metrics_from_tools(None, "agent prose only", "", 3, 0)
204
        self.assertIsNone(metrics["commands_total"])
205
        self.assertIsNone(metrics["status_commands"])
206
        self.assertIsNone(metrics["snapshot_commands"])
207
        self.assertIsNone(metrics["branch_commands"])
208
        self.assertEqual(metrics["branches_created"], 0)
209
        self.assertEqual(metrics["dirty_files_at_end"], 3)
210
211
    def test_git_add_and_commit_chain_counts_snapshot(self) -> None:
212
        aw = load_script("agent_workflow")
213
        metrics = aw.vcs_metrics_from_tools(None, '$ git add . && git commit -m "agent task"\n', "", 0, 1)
214
        self.assertEqual(metrics["commands_total"], 2)
215
        self.assertEqual(metrics["snapshot_commands"], 1)
216
217
    def test_commit_message_mentioning_vcs_command_does_not_create_phantom_invocation(self) -> None:
218
        from oakbench.classify import vcs_invocations, vcs_subcommands
219
220
        command = 'git commit -m "run git status afterwards"'
221
222
        self.assertEqual(vcs_invocations(command), [(command, "commit")])
223
        self.assertEqual(vcs_subcommands(command), ["commit"])
224
225
    def test_commit_message_vcs_text_does_not_inflate_agent_metrics(self) -> None:
226
        aw = load_script("agent_workflow")
227
        metrics = aw.vcs_metrics_from_tools(
228
            None,
229
            '$ git commit -m "run git status afterwards"\n',
230
            "",
231
            0,
232
            1,
233
        )
234
        self.assertEqual(metrics["commands_total"], 1)
235
        self.assertEqual(metrics["snapshot_commands"], 1)
236
        self.assertEqual(metrics["status_commands"], 0)
237
238
    def test_git_add_and_clean_chain_counts_one_cleanup(self) -> None:
239
        aw = load_script("agent_workflow")
240
        metrics = aw.vcs_metrics_from_tools(None, "$ git add . && git clean -fd\n", "", 0, 0)
241
        self.assertEqual(metrics["commands_total"], 2)
242
        self.assertEqual(metrics["cleanup_commands"], 1)
243
244
    def test_branch_create_chain_counts_only_creator(self) -> None:
245
        aw = load_script("agent_workflow")
246
        metrics = aw.vcs_metrics_from_tools(None, "$ git checkout -b work && git switch main\n", "", 0, 0)
247
        self.assertEqual(metrics["commands_total"], 2)
248
        self.assertEqual(metrics["branch_commands"], 1)
249
        self.assertEqual(metrics["branches_created"], 1)
250
251
252
class SkipRowCoherenceTest(unittest.TestCase):
253
    BASE = {
254
        "bench_id": "t",
255
        "profile": "micro",
256
        "scenario": "s",
257
        "operation": "op",
258
        "run": 0,
259
        "subject": "git",
260
        "subject_kind": "git",
261
        "elapsed_ms": 0.0,
262
        "command": [],
263
    }
264
265
    def test_well_formed_skip_row_passes(self) -> None:
266
        row = {**self.BASE, "returncode": 77, "skipped": True, "skip_reason": "remote not configured"}
267
        self.assertEqual(validate_rows([row], "core"), [])
268
269
    def test_returncode_77_without_skipped_fails(self) -> None:
270
        row = {**self.BASE, "returncode": 77}
271
        self.assertTrue(any("skipped" in error for error in validate_rows([row], "core")))
272
273
    def test_skip_without_reason_fails(self) -> None:
274
        row = {**self.BASE, "returncode": 77, "skipped": True}
275
        self.assertTrue(any("skip_reason" in error for error in validate_rows([row], "core")))
276
277
    def test_skipped_flag_with_zero_returncode_fails(self) -> None:
278
        row = {**self.BASE, "returncode": 0, "skipped": True, "skip_reason": "x"}
279
        self.assertTrue(any("returncode 77" in error for error in validate_rows([row], "core")))
280
281
    def test_information_recall_range_enforced(self) -> None:
282
        row = {**self.BASE, "returncode": 0, "information_recall": 1.5}
283
        self.assertTrue(any("information_recall" in error for error in validate_rows([row], "core")))
284
285
286
if __name__ == "__main__":
287
    unittest.main()