Log in
benchmarksPublic

Viewing files at commit 63ed254f5df1Commit detailsView latest

Browse files
tests/test_new_instruments.py 332 lines · 14.4 KB · python
1
"""Tests for the reviewer-gap instruments.
2
3
Covers: tool-call envelope token accounting, tail-latency sample honesty,
4
output information recall and piped-diff compatibility oracles, peak-RSS
5
capture, and follow-up info-command detection. A benchmark that measures with
6
untested instruments cannot make accuracy claims.
7
"""
8
9
from __future__ import annotations
10
11
import os
12
import sys
13
import unittest
14
from pathlib import Path
15
16
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
17
18
from oakbench import tokens  # noqa: E402
19
from oakbench.classify import vcs_info_family  # noqa: E402
20
from oakbench.execution import run_timed  # noqa: E402
21
from oakbench.output_semantics import (  # noqa: E402
22
    information_recall_fields,
23
    reported_paths,
24
    unified_diff_compat_fields,
25
)
26
from oakbench.reporting import tail_latency_summary  # noqa: E402
27
from oakbench.stream_adapters import analyze_stream  # noqa: E402
28
29
30
class EnvelopeTokenTest(unittest.TestCase):
31
    def test_one_tool_call_adds_envelope_fields(self) -> None:
32
        fields = tokens.interaction_token_fields("git status", "ok", "", 2, 0, False, False)
33
        self.assertEqual(
34
            fields["estimated_tokens_envelope_emitted"], tokens.TOOL_CALL_ENVELOPE_EMITTED_TOKENS
35
        )
36
        self.assertEqual(
37
            fields["estimated_tokens_envelope_ingested"], tokens.TOOL_RESULT_ENVELOPE_INGESTED_TOKENS
38
        )
39
        self.assertEqual(
40
            fields["estimated_tokens_total_with_envelope"],
41
            fields["estimated_tokens_total"]
42
            + tokens.TOOL_CALL_ENVELOPE_EMITTED_TOKENS
43
            + tokens.TOOL_RESULT_ENVELOPE_INGESTED_TOKENS,
44
        )
45
46
    def test_zero_tool_calls_means_zero_envelope(self) -> None:
47
        fields = tokens.interaction_token_fields("probe", "", "", 0, 0, False, False, tool_call_count=0)
48
        self.assertEqual(fields["estimated_tokens_envelope_emitted"], 0)
49
        self.assertEqual(fields["estimated_tokens_total_with_envelope"], fields["estimated_tokens_total"])
50
51
    def test_historical_fields_unchanged_by_envelope_addition(self) -> None:
52
        """Row parity (ADR-0005): envelope accounting is additive only."""
53
        fields = tokens.interaction_token_fields("git status", "out", "err", 3, 3, False, False)
54
        command_tokens = tokens.estimate_tokens("git status")
55
        output_tokens = tokens.estimate_tokens("out") + tokens.estimate_tokens("err")
56
        self.assertEqual(fields["estimated_tokens_total"], command_tokens + output_tokens)
57
        self.assertEqual(
58
            fields["estimated_cost_weighted_tokens"],
59
            tokens.cost_weighted_total(command_tokens, output_tokens),
60
        )
61
62
    def test_summed_fields_aggregate_envelopes(self) -> None:
63
        sub_rows = [
64
            tokens.interaction_token_fields("git add .", "", "", 0, 0, False, False),
65
            tokens.interaction_token_fields("git commit -m x", "done", "", 4, 0, False, False),
66
        ]
67
        summed = tokens.summed_token_fields(sub_rows, "test_sum")
68
        self.assertEqual(
69
            summed["estimated_tokens_envelope_emitted"], 2 * tokens.TOOL_CALL_ENVELOPE_EMITTED_TOKENS
70
        )
71
        self.assertEqual(
72
            summed["estimated_tokens_total_with_envelope"],
73
            summed["estimated_tokens_total"]
74
            + 2 * tokens.TOOL_CALL_ENVELOPE_EMITTED_TOKENS
75
            + 2 * tokens.TOOL_RESULT_ENVELOPE_INGESTED_TOKENS,
76
        )
77
78
    def test_two_call_snapshot_costs_more_envelope_than_one(self) -> None:
79
        """The reason envelope modeling matters: git's stage+commit pays two
80
        envelopes where oak's single commit pays one."""
81
        git_like = tokens.summed_token_fields(
82
            [
83
                tokens.interaction_token_fields("git add .", "", "", 0, 0, False, False),
84
                tokens.interaction_token_fields("git commit -m m", "", "", 0, 0, False, False),
85
            ],
86
            "s",
87
        )
88
        oak_like = tokens.summed_token_fields(
89
            [tokens.interaction_token_fields("oak commit --no-verify", "", "", 0, 0, False, False)], "s"
90
        )
91
        self.assertGreater(
92
            git_like["estimated_tokens_envelope_emitted"], oak_like["estimated_tokens_envelope_emitted"]
93
        )
94
95
96
class TailLatencyHonestyTest(unittest.TestCase):
97
    def test_small_sample_nulls_tail_percentiles(self) -> None:
98
        summary = tail_latency_summary([10.0, 20.0, 30.0])
99
        self.assertEqual(summary["n"], 3)
100
        self.assertIsNone(summary["p95_ms"])
101
        self.assertIsNone(summary["p99_ms"])
102
        self.assertEqual(summary["tail_supported"], [])
103
        self.assertEqual(summary["max_ms"], 30.0)
104
105
    def test_p95_supported_at_twenty_samples(self) -> None:
106
        summary = tail_latency_summary([float(i) for i in range(1, 21)])
107
        self.assertEqual(summary["p95_ms"], 19.0)
108
        self.assertIsNone(summary["p99_ms"])
109
        self.assertEqual(summary["tail_supported"], ["p95"])
110
111
    def test_p99_supported_at_hundred_samples(self) -> None:
112
        summary = tail_latency_summary([float(i) for i in range(1, 101)])
113
        self.assertEqual(summary["p99_ms"], 99.0)
114
        self.assertEqual(summary["tail_supported"], ["p95", "p99"])
115
116
    def test_empty_sample_is_all_null(self) -> None:
117
        summary = tail_latency_summary([])
118
        self.assertEqual(summary["n"], 0)
119
        self.assertIsNone(summary["p50_ms"])
120
        self.assertIsNone(summary["max_ms"])
121
122
123
class InformationRecallTest(unittest.TestCase):
124
    def test_full_recall_when_all_paths_named(self) -> None:
125
        output = " M src/a.txt\n M src/b.txt\n"
126
        fields = information_recall_fields(output, ["src/a.txt", "src/b.txt"], len(output), False)
127
        self.assertEqual(fields["information_recall"], 1.0)
128
        self.assertEqual(fields["information_precision"], 1.0)
129
        self.assertEqual(fields["changed_files_recovered"], 2)
130
        self.assertEqual(fields["changed_files_false_positive"], 0)
131
        self.assertIsNotNone(fields["bytes_per_changed_file_named"])
132
133
    def test_root_level_file_paths_are_reported_for_precision(self) -> None:
134
        output = " M README.md\n?? notes.txt\n"
135
        fields = information_recall_fields(output, ["README.md"], len(output), False)
136
        self.assertEqual(fields["information_recall"], 1.0)
137
        self.assertEqual(fields["changed_files_reported"], 2)
138
        self.assertEqual(fields["changed_files_false_positive"], 1)
139
        self.assertEqual(fields["information_precision"], 0.5)
140
141
    def test_lossy_compact_output_scores_below_one(self) -> None:
142
        output = "2 files changed\n"  # bytes saved, information lost
143
        fields = information_recall_fields(output, ["src/a.txt", "src/b.txt"], len(output), False)
144
        self.assertEqual(fields["information_recall"], 0.0)
145
        self.assertIsNone(fields["information_precision"])
146
        self.assertIsNone(fields["bytes_per_changed_file_named"])
147
148
    def test_no_ground_truth_is_unmeasured_not_perfect(self) -> None:
149
        fields = information_recall_fields("anything", [], 8, False)
150
        self.assertIsNone(fields["information_recall"])
151
        self.assertIsNone(fields["information_precision"])
152
153
    def test_git_prefixed_paths_still_recovered(self) -> None:
154
        output = "diff --git a/src/a.txt b/src/a.txt\n"
155
        fields = information_recall_fields(output, ["src/a.txt"], len(output), False)
156
        self.assertEqual(fields["information_recall"], 1.0)
157
        self.assertEqual(fields["information_precision"], 1.0)
158
159
    def test_noisy_output_gets_low_precision_despite_full_recall(self) -> None:
160
        output = " M src/a.txt\n M src/unrelated.txt\n?? docs/notes.md\n"
161
        fields = information_recall_fields(output, ["src/a.txt"], len(output), False)
162
        self.assertEqual(fields["information_recall"], 1.0)
163
        self.assertAlmostEqual(fields["information_precision"], 1 / 3, places=4)
164
        self.assertEqual(fields["changed_files_reported"], 3)
165
        self.assertEqual(fields["changed_files_false_positive"], 2)
166
167
    def test_diff_body_dotted_identifiers_do_not_pollute_precision(self) -> None:
168
        output = (
169
            "diff --git a/README.md b/README.md\n"
170
            "--- a/README.md\n"
171
            "+++ b/README.md\n"
172
            "@@ -1 +1 @@\n"
173
            "+self.foo = config.value + v0.97.0 + 1.5\n"
174
        )
175
        fields = information_recall_fields(output, ["README.md"], len(output), False)
176
        self.assertEqual(fields["information_recall"], 1.0)
177
        self.assertEqual(fields["information_precision"], 1.0)
178
        self.assertEqual(fields["changed_files_reported"], 1)
179
180
    def test_indented_diff_context_does_not_parse_as_status_rows(self) -> None:
181
        output = (
182
            "diff --git a/src/main.rs b/src/main.rs\n"
183
            "--- a/src/main.rs\n"
184
            "+++ b/src/main.rs\n"
185
            "@@ -1,4 +1,4 @@\n"
186
            "  foo.bar.baz\n"
187
            "  let cfg = config.value\n"
188
            "+ let changed = true\n"
189
        )
190
        self.assertEqual(reported_paths(output), ["src/main.rs"])
191
192
193
class PipeCompatTest(unittest.TestCase):
194
    UNIFIED = (
195
        "diff --git a/f.txt b/f.txt\n"
196
        "--- a/f.txt\n"
197
        "+++ b/f.txt\n"
198
        "@@ -1,2 +1,2 @@\n"
199
        "-old\n"
200
        "+new\n"
201
    )
202
203
    def test_unified_diff_is_pipe_compatible(self) -> None:
204
        fields = unified_diff_compat_fields(self.UNIFIED)
205
        self.assertTrue(fields["pipe_compatible_unified"])
206
        self.assertEqual(fields["unified_hunk_header_count"], 1)
207
        self.assertEqual(fields["unified_file_header_count"], 1)
208
209
    def test_stat_output_is_not_pipe_compatible(self) -> None:
210
        fields = unified_diff_compat_fields(" f.txt | 2 +-\n 1 file changed\n")
211
        self.assertFalse(fields["pipe_compatible_unified"])
212
213
    def test_binary_notice_counts_as_compatible(self) -> None:
214
        fields = unified_diff_compat_fields("Binary files a/x.bin and b/x.bin differ\n")
215
        self.assertTrue(fields["pipe_compatible_unified"])
216
        self.assertEqual(fields["binary_notice_count"], 1)
217
218
219
class PeakRssTest(unittest.TestCase):
220
    def test_full_output_capture_does_not_change_admitted_window(self) -> None:
221
        capture = run_timed(
222
            [sys.executable, "-c", "import sys; sys.stdout.write('abcdef')"],
223
            Path.cwd(),
224
            dict(os.environ),
225
            3,
226
            full_output_bytes=10,
227
        )
228
229
        self.assertEqual(capture.returncode, 0)
230
        self.assertEqual(capture.stdout_text, "abc")
231
        self.assertTrue(capture.stdout_truncated)
232
        self.assertEqual(capture.stdout_full_text, "abcdef")
233
        self.assertFalse(capture.stdout_full_text_truncated)
234
235
    def test_full_output_capture_is_honestly_unmeasured_over_byte_limit(self) -> None:
236
        capture = run_timed(
237
            [sys.executable, "-c", "import sys; sys.stdout.write('abcdef')"],
238
            Path.cwd(),
239
            dict(os.environ),
240
            3,
241
            full_output_bytes=5,
242
        )
243
244
        self.assertEqual(capture.stdout_text, "abc")
245
        self.assertTrue(capture.stdout_truncated)
246
        self.assertIsNone(capture.stdout_full_text)
247
        self.assertTrue(capture.stdout_full_text_truncated)
248
249
    def test_peak_rss_captured_on_posix(self) -> None:
250
        if not hasattr(os, "wait4"):
251
            self.skipTest("os.wait4 unavailable on this platform")
252
        capture = run_timed(
253
            [sys.executable, "-c", "print('hi')"], Path.cwd(), dict(os.environ), 100
254
        )
255
        self.assertEqual(capture.returncode, 0)
256
        self.assertIsNotNone(capture.peak_rss_bytes)
257
        self.assertGreater(capture.peak_rss_bytes, 1024 * 1024)  # > 1 MiB for a python process
258
259
260
def claude_tool_event(tool_id: str, command: str, usage_input: int = 100) -> dict:
261
    return {
262
        "type": "assistant",
263
        "message": {
264
            "role": "assistant",
265
            "usage": {"input_tokens": usage_input, "output_tokens": 10, "cache_read_input_tokens": 0},
266
            "content": [{"type": "tool_use", "name": "Bash", "id": tool_id, "input": {"command": command}}],
267
        },
268
    }
269
270
271
def claude_result_event(tool_id: str, content: str, is_error: bool = False) -> dict:
272
    return {
273
        "type": "user",
274
        "message": {
275
            "role": "user",
276
            "content": [{"type": "tool_result", "tool_use_id": tool_id, "is_error": is_error, "content": content}],
277
        },
278
    }
279
280
281
class FollowupInfoCommandTest(unittest.TestCase):
282
    def test_vcs_info_family_classification(self) -> None:
283
        self.assertEqual(vcs_info_family("git status --short"), "status")
284
        self.assertEqual(vcs_info_family("oak diff --stat"), "diff")
285
        self.assertIsNone(vcs_info_family("git commit -m x"))
286
        self.assertIsNone(vcs_info_family("ls -la"))
287
288
    def test_followup_counted_when_same_family_reasked(self) -> None:
289
        events = [
290
            claude_tool_event("t1", "git status --short"),
291
            claude_result_event("t1", " M a.txt"),
292
            claude_tool_event("t2", "git status"),
293
            claude_result_event("t2", "modified: a.txt"),
294
        ]
295
        analysis = analyze_stream(events, "claude_print_stream_json")
296
        self.assertEqual(analysis.vcs_info_followup_calls_total, 1)
297
298
    def test_no_followup_when_action_intervenes(self) -> None:
299
        events = [
300
            claude_tool_event("t1", "git status --short"),
301
            claude_result_event("t1", " M a.txt"),
302
            claude_tool_event("t2", "git commit -m fix"),
303
            claude_result_event("t2", "committed"),
304
            claude_tool_event("t3", "git status"),
305
            claude_result_event("t3", "clean"),
306
        ]
307
        analysis = analyze_stream(events, "claude_print_stream_json")
308
        self.assertEqual(analysis.vcs_info_followup_calls_total, 0)
309
310
    def test_unresolvable_commands_leave_metric_unmeasured(self) -> None:
311
        # Results reference ids with no tool_use input.command anywhere:
312
        # the metric must be None (unmeasured), not a fabricated zero.
313
        events = [
314
            {"type": "assistant", "message": {"role": "assistant", "usage": {"input_tokens": 1, "output_tokens": 1}, "content": [{"type": "text", "text": "hi"}]}},
315
            claude_result_event("ghost", "output"),
316
        ]
317
        analysis = analyze_stream(events, "claude_print_stream_json")
318
        self.assertIsNone(analysis.vcs_info_followup_calls_total)
319
320
    def test_failed_call_breaks_followup_chain(self) -> None:
321
        events = [
322
            claude_tool_event("t1", "git status --short"),
323
            claude_result_event("t1", "error", is_error=True),
324
            claude_tool_event("t2", "git status"),
325
            claude_result_event("t2", "ok"),
326
        ]
327
        analysis = analyze_stream(events, "claude_print_stream_json")
328
        self.assertEqual(analysis.vcs_info_followup_calls_total, 0)
329
330
331
if __name__ == "__main__":
332
    unittest.main()