| 1 | """Tests for the reviewer-gap instruments.
|
| 2 |
|
| 3 | Covers: tool-call envelope token accounting, tail-latency sample honesty,
|
| 4 | output information recall and piped-diff compatibility oracles, peak-RSS
|
| 5 | capture, and follow-up info-command detection. A benchmark that measures with
|
| 6 | untested instruments cannot make accuracy claims.
|
| 7 | """
|
| 8 |
|
| 9 | from __future__ import annotations
|
| 10 |
|
| 11 | import os
|
| 12 | import sys
|
| 13 | import tempfile
|
| 14 | import time
|
| 15 | import unittest
|
| 16 | from pathlib import Path
|
| 17 |
|
| 18 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 19 |
|
| 20 | from oakbench import tokens # noqa: E402
|
| 21 | from oakbench.classify import vcs_info_family # noqa: E402
|
| 22 | from oakbench.execution import run_timed # noqa: E402
|
| 23 | from oakbench.output_semantics import ( # noqa: E402
|
| 24 | information_recall_fields,
|
| 25 | reported_paths,
|
| 26 | unified_diff_compat_fields,
|
| 27 | )
|
| 28 | from oakbench.reporting import tail_latency_summary # noqa: E402
|
| 29 | from oakbench.stream_adapters import analyze_stream # noqa: E402
|
| 30 |
|
| 31 |
|
| 32 | class EnvelopeTokenTest(unittest.TestCase):
|
| 33 | def test_one_tool_call_adds_envelope_fields(self) -> None:
|
| 34 | fields = tokens.interaction_token_fields("git status", "ok", "", 2, 0, False, False)
|
| 35 | self.assertEqual(
|
| 36 | fields["estimated_tokens_envelope_emitted"], tokens.TOOL_CALL_ENVELOPE_EMITTED_TOKENS
|
| 37 | )
|
| 38 | self.assertEqual(
|
| 39 | fields["estimated_tokens_envelope_ingested"], tokens.TOOL_RESULT_ENVELOPE_INGESTED_TOKENS
|
| 40 | )
|
| 41 | self.assertEqual(
|
| 42 | fields["estimated_tokens_total_with_envelope"],
|
| 43 | fields["estimated_tokens_total"]
|
| 44 | + tokens.TOOL_CALL_ENVELOPE_EMITTED_TOKENS
|
| 45 | + tokens.TOOL_RESULT_ENVELOPE_INGESTED_TOKENS,
|
| 46 | )
|
| 47 |
|
| 48 | def test_zero_tool_calls_means_zero_envelope(self) -> None:
|
| 49 | fields = tokens.interaction_token_fields("probe", "", "", 0, 0, False, False, tool_call_count=0)
|
| 50 | self.assertEqual(fields["estimated_tokens_envelope_emitted"], 0)
|
| 51 | self.assertEqual(fields["estimated_tokens_total_with_envelope"], fields["estimated_tokens_total"])
|
| 52 |
|
| 53 | def test_historical_fields_unchanged_by_envelope_addition(self) -> None:
|
| 54 | """Row parity (ADR-0005): envelope accounting is additive only."""
|
| 55 | fields = tokens.interaction_token_fields("git status", "out", "err", 3, 3, False, False)
|
| 56 | command_tokens = tokens.estimate_tokens("git status")
|
| 57 | output_tokens = tokens.estimate_tokens("out") + tokens.estimate_tokens("err")
|
| 58 | self.assertEqual(fields["estimated_tokens_total"], command_tokens + output_tokens)
|
| 59 | self.assertEqual(
|
| 60 | fields["estimated_cost_weighted_tokens"],
|
| 61 | tokens.cost_weighted_total(command_tokens, output_tokens),
|
| 62 | )
|
| 63 |
|
| 64 | def test_summed_fields_aggregate_envelopes(self) -> None:
|
| 65 | sub_rows = [
|
| 66 | tokens.interaction_token_fields("git add .", "", "", 0, 0, False, False),
|
| 67 | tokens.interaction_token_fields("git commit -m x", "done", "", 4, 0, False, False),
|
| 68 | ]
|
| 69 | summed = tokens.summed_token_fields(sub_rows, "test_sum")
|
| 70 | self.assertEqual(
|
| 71 | summed["estimated_tokens_envelope_emitted"], 2 * tokens.TOOL_CALL_ENVELOPE_EMITTED_TOKENS
|
| 72 | )
|
| 73 | self.assertEqual(
|
| 74 | summed["estimated_tokens_total_with_envelope"],
|
| 75 | summed["estimated_tokens_total"]
|
| 76 | + 2 * tokens.TOOL_CALL_ENVELOPE_EMITTED_TOKENS
|
| 77 | + 2 * tokens.TOOL_RESULT_ENVELOPE_INGESTED_TOKENS,
|
| 78 | )
|
| 79 |
|
| 80 | def test_two_call_snapshot_costs_more_envelope_than_one(self) -> None:
|
| 81 | """The reason envelope modeling matters: git's stage+commit pays two
|
| 82 | envelopes where oak's single commit pays one."""
|
| 83 | git_like = tokens.summed_token_fields(
|
| 84 | [
|
| 85 | tokens.interaction_token_fields("git add .", "", "", 0, 0, False, False),
|
| 86 | tokens.interaction_token_fields("git commit -m m", "", "", 0, 0, False, False),
|
| 87 | ],
|
| 88 | "s",
|
| 89 | )
|
| 90 | oak_like = tokens.summed_token_fields(
|
| 91 | [tokens.interaction_token_fields("oak commit --no-verify", "", "", 0, 0, False, False)], "s"
|
| 92 | )
|
| 93 | self.assertGreater(
|
| 94 | git_like["estimated_tokens_envelope_emitted"], oak_like["estimated_tokens_envelope_emitted"]
|
| 95 | )
|
| 96 |
|
| 97 |
|
| 98 | class TailLatencyHonestyTest(unittest.TestCase):
|
| 99 | def test_small_sample_nulls_tail_percentiles(self) -> None:
|
| 100 | summary = tail_latency_summary([10.0, 20.0, 30.0])
|
| 101 | self.assertEqual(summary["n"], 3)
|
| 102 | self.assertIsNone(summary["p95_ms"])
|
| 103 | self.assertIsNone(summary["p99_ms"])
|
| 104 | self.assertEqual(summary["tail_supported"], [])
|
| 105 | self.assertEqual(summary["max_ms"], 30.0)
|
| 106 |
|
| 107 | def test_p95_supported_at_twenty_samples(self) -> None:
|
| 108 | summary = tail_latency_summary([float(i) for i in range(1, 21)])
|
| 109 | self.assertEqual(summary["p95_ms"], 19.0)
|
| 110 | self.assertIsNone(summary["p99_ms"])
|
| 111 | self.assertEqual(summary["tail_supported"], ["p95"])
|
| 112 |
|
| 113 | def test_p99_supported_at_hundred_samples(self) -> None:
|
| 114 | summary = tail_latency_summary([float(i) for i in range(1, 101)])
|
| 115 | self.assertEqual(summary["p99_ms"], 99.0)
|
| 116 | self.assertEqual(summary["tail_supported"], ["p95", "p99"])
|
| 117 |
|
| 118 | def test_empty_sample_is_all_null(self) -> None:
|
| 119 | summary = tail_latency_summary([])
|
| 120 | self.assertEqual(summary["n"], 0)
|
| 121 | self.assertIsNone(summary["p50_ms"])
|
| 122 | self.assertIsNone(summary["max_ms"])
|
| 123 |
|
| 124 |
|
| 125 | class InformationRecallTest(unittest.TestCase):
|
| 126 | def test_full_recall_when_all_paths_named(self) -> None:
|
| 127 | output = " M src/a.txt\n M src/b.txt\n"
|
| 128 | fields = information_recall_fields(output, ["src/a.txt", "src/b.txt"], len(output), False)
|
| 129 | self.assertEqual(fields["information_recall"], 1.0)
|
| 130 | self.assertEqual(fields["information_precision"], 1.0)
|
| 131 | self.assertEqual(fields["changed_files_recovered"], 2)
|
| 132 | self.assertEqual(fields["changed_files_false_positive"], 0)
|
| 133 | self.assertIsNotNone(fields["bytes_per_changed_file_named"])
|
| 134 |
|
| 135 | def test_root_level_file_paths_are_reported_for_precision(self) -> None:
|
| 136 | output = " M README.md\n?? notes.txt\n"
|
| 137 | fields = information_recall_fields(output, ["README.md"], len(output), False)
|
| 138 | self.assertEqual(fields["information_recall"], 1.0)
|
| 139 | self.assertEqual(fields["changed_files_reported"], 2)
|
| 140 | self.assertEqual(fields["changed_files_false_positive"], 1)
|
| 141 | self.assertEqual(fields["information_precision"], 0.5)
|
| 142 |
|
| 143 | def test_lossy_compact_output_scores_below_one(self) -> None:
|
| 144 | output = "2 files changed\n" # bytes saved, information lost
|
| 145 | fields = information_recall_fields(output, ["src/a.txt", "src/b.txt"], len(output), False)
|
| 146 | self.assertEqual(fields["information_recall"], 0.0)
|
| 147 | self.assertIsNone(fields["information_precision"])
|
| 148 | self.assertIsNone(fields["bytes_per_changed_file_named"])
|
| 149 |
|
| 150 | def test_no_ground_truth_is_unmeasured_not_perfect(self) -> None:
|
| 151 | fields = information_recall_fields("anything", [], 8, False)
|
| 152 | self.assertIsNone(fields["information_recall"])
|
| 153 | self.assertIsNone(fields["information_precision"])
|
| 154 |
|
| 155 | def test_git_prefixed_paths_still_recovered(self) -> None:
|
| 156 | output = "diff --git a/src/a.txt b/src/a.txt\n"
|
| 157 | fields = information_recall_fields(output, ["src/a.txt"], len(output), False)
|
| 158 | self.assertEqual(fields["information_recall"], 1.0)
|
| 159 | self.assertEqual(fields["information_precision"], 1.0)
|
| 160 |
|
| 161 | def test_noisy_output_gets_low_precision_despite_full_recall(self) -> None:
|
| 162 | output = " M src/a.txt\n M src/unrelated.txt\n?? docs/notes.md\n"
|
| 163 | fields = information_recall_fields(output, ["src/a.txt"], len(output), False)
|
| 164 | self.assertEqual(fields["information_recall"], 1.0)
|
| 165 | self.assertAlmostEqual(fields["information_precision"], 1 / 3, places=4)
|
| 166 | self.assertEqual(fields["changed_files_reported"], 3)
|
| 167 | self.assertEqual(fields["changed_files_false_positive"], 2)
|
| 168 |
|
| 169 | def test_diff_body_dotted_identifiers_do_not_pollute_precision(self) -> None:
|
| 170 | output = (
|
| 171 | "diff --git a/README.md b/README.md\n"
|
| 172 | "--- a/README.md\n"
|
| 173 | "+++ b/README.md\n"
|
| 174 | "@@ -1 +1 @@\n"
|
| 175 | "+self.foo = config.value + v0.97.0 + 1.5\n"
|
| 176 | )
|
| 177 | fields = information_recall_fields(output, ["README.md"], len(output), False)
|
| 178 | self.assertEqual(fields["information_recall"], 1.0)
|
| 179 | self.assertEqual(fields["information_precision"], 1.0)
|
| 180 | self.assertEqual(fields["changed_files_reported"], 1)
|
| 181 |
|
| 182 | def test_indented_diff_context_does_not_parse_as_status_rows(self) -> None:
|
| 183 | output = (
|
| 184 | "diff --git a/src/main.rs b/src/main.rs\n"
|
| 185 | "--- a/src/main.rs\n"
|
| 186 | "+++ b/src/main.rs\n"
|
| 187 | "@@ -1,4 +1,4 @@\n"
|
| 188 | " foo.bar.baz\n"
|
| 189 | " let cfg = config.value\n"
|
| 190 | "+ let changed = true\n"
|
| 191 | )
|
| 192 | self.assertEqual(reported_paths(output), ["src/main.rs"])
|
| 193 |
|
| 194 |
|
| 195 | class PipeCompatTest(unittest.TestCase):
|
| 196 | UNIFIED = (
|
| 197 | "diff --git a/f.txt b/f.txt\n"
|
| 198 | "--- a/f.txt\n"
|
| 199 | "+++ b/f.txt\n"
|
| 200 | "@@ -1,2 +1,2 @@\n"
|
| 201 | "-old\n"
|
| 202 | "+new\n"
|
| 203 | )
|
| 204 |
|
| 205 | def test_unified_diff_is_pipe_compatible(self) -> None:
|
| 206 | fields = unified_diff_compat_fields(self.UNIFIED)
|
| 207 | self.assertTrue(fields["pipe_compatible_unified"])
|
| 208 | self.assertEqual(fields["unified_hunk_header_count"], 1)
|
| 209 | self.assertEqual(fields["unified_file_header_count"], 1)
|
| 210 |
|
| 211 | def test_stat_output_is_not_pipe_compatible(self) -> None:
|
| 212 | fields = unified_diff_compat_fields(" f.txt | 2 +-\n 1 file changed\n")
|
| 213 | self.assertFalse(fields["pipe_compatible_unified"])
|
| 214 |
|
| 215 | def test_binary_notice_counts_as_compatible(self) -> None:
|
| 216 | fields = unified_diff_compat_fields("Binary files a/x.bin and b/x.bin differ\n")
|
| 217 | self.assertTrue(fields["pipe_compatible_unified"])
|
| 218 | self.assertEqual(fields["binary_notice_count"], 1)
|
| 219 |
|
| 220 |
|
| 221 | class PeakRssTest(unittest.TestCase):
|
| 222 | def test_timed_execution_bounds_and_marks_an_owned_process(self) -> None:
|
| 223 | started = time.monotonic()
|
| 224 | capture = run_timed(
|
| 225 | [sys.executable, "-c", "import time; time.sleep(60)"],
|
| 226 | Path.cwd(),
|
| 227 | dict(os.environ),
|
| 228 | 100,
|
| 229 | timeout_seconds=0.1,
|
| 230 | )
|
| 231 |
|
| 232 | self.assertLess(time.monotonic() - started, 3)
|
| 233 | self.assertTrue(capture.timed_out)
|
| 234 | self.assertEqual(capture.returncode, 124)
|
| 235 |
|
| 236 | @unittest.skipUnless(os.name == "posix", "owned process-group assertion is POSIX-specific")
|
| 237 | def test_timed_execution_stops_owned_descendants(self) -> None:
|
| 238 | with tempfile.TemporaryDirectory() as tmp:
|
| 239 | marker = Path(tmp) / "descendant-survived"
|
| 240 | script = (
|
| 241 | "import subprocess,sys,time; "
|
| 242 | "subprocess.Popen([sys.executable,'-c',"
|
| 243 | f"\"import time,pathlib; time.sleep(0.5); pathlib.Path({str(marker)!r}).write_text('bad')\"]); "
|
| 244 | "time.sleep(60)"
|
| 245 | )
|
| 246 | capture = run_timed(
|
| 247 | [sys.executable, "-c", script],
|
| 248 | Path.cwd(),
|
| 249 | dict(os.environ),
|
| 250 | 100,
|
| 251 | timeout_seconds=0.1,
|
| 252 | )
|
| 253 | time.sleep(0.7)
|
| 254 |
|
| 255 | self.assertTrue(capture.timed_out)
|
| 256 | self.assertFalse(marker.exists())
|
| 257 |
|
| 258 | @unittest.skipUnless(os.name == "posix", "owned process-group assertion is POSIX-specific")
|
| 259 | def test_timed_execution_kills_term_ignoring_descendant_after_leader_exits(self) -> None:
|
| 260 | with tempfile.TemporaryDirectory() as tmp:
|
| 261 | marker = Path(tmp) / "term-ignoring-descendant-survived"
|
| 262 | descendant = (
|
| 263 | "import pathlib,signal,sys,time; "
|
| 264 | "signal.signal(signal.SIGTERM, signal.SIG_IGN); "
|
| 265 | "time.sleep(0.6); pathlib.Path(sys.argv[1]).write_text('bad')"
|
| 266 | )
|
| 267 | parent = (
|
| 268 | "import subprocess,sys,time; "
|
| 269 | f"subprocess.Popen([sys.executable,'-c',{descendant!r},sys.argv[1]]); "
|
| 270 | "time.sleep(60)"
|
| 271 | )
|
| 272 | capture = run_timed(
|
| 273 | [sys.executable, "-c", parent, str(marker)],
|
| 274 | Path.cwd(),
|
| 275 | dict(os.environ),
|
| 276 | 100,
|
| 277 | timeout_seconds=0.1,
|
| 278 | )
|
| 279 | time.sleep(0.8)
|
| 280 |
|
| 281 | self.assertTrue(capture.timed_out)
|
| 282 | self.assertFalse(marker.exists())
|
| 283 |
|
| 284 | def test_full_output_capture_does_not_change_admitted_window(self) -> None:
|
| 285 | capture = run_timed(
|
| 286 | [sys.executable, "-c", "import sys; sys.stdout.write('abcdef')"],
|
| 287 | Path.cwd(),
|
| 288 | dict(os.environ),
|
| 289 | 3,
|
| 290 | full_output_bytes=10,
|
| 291 | )
|
| 292 |
|
| 293 | self.assertEqual(capture.returncode, 0)
|
| 294 | self.assertEqual(capture.stdout_text, "abc")
|
| 295 | self.assertTrue(capture.stdout_truncated)
|
| 296 | self.assertEqual(capture.stdout_full_text, "abcdef")
|
| 297 | self.assertFalse(capture.stdout_full_text_truncated)
|
| 298 |
|
| 299 | def test_full_output_capture_is_honestly_unmeasured_over_byte_limit(self) -> None:
|
| 300 | capture = run_timed(
|
| 301 | [sys.executable, "-c", "import sys; sys.stdout.write('abcdef')"],
|
| 302 | Path.cwd(),
|
| 303 | dict(os.environ),
|
| 304 | 3,
|
| 305 | full_output_bytes=5,
|
| 306 | )
|
| 307 |
|
| 308 | self.assertEqual(capture.stdout_text, "abc")
|
| 309 | self.assertTrue(capture.stdout_truncated)
|
| 310 | self.assertIsNone(capture.stdout_full_text)
|
| 311 | self.assertTrue(capture.stdout_full_text_truncated)
|
| 312 |
|
| 313 | def test_peak_rss_captured_on_posix(self) -> None:
|
| 314 | if not hasattr(os, "wait4"):
|
| 315 | self.skipTest("os.wait4 unavailable on this platform")
|
| 316 | capture = run_timed(
|
| 317 | [sys.executable, "-c", "print('hi')"], Path.cwd(), dict(os.environ), 100
|
| 318 | )
|
| 319 | self.assertEqual(capture.returncode, 0)
|
| 320 | self.assertIsNotNone(capture.peak_rss_bytes)
|
| 321 | self.assertGreater(capture.peak_rss_bytes, 1024 * 1024) # > 1 MiB for a python process
|
| 322 |
|
| 323 |
|
| 324 | def claude_tool_event(tool_id: str, command: str, usage_input: int = 100) -> dict:
|
| 325 | return {
|
| 326 | "type": "assistant",
|
| 327 | "message": {
|
| 328 | "role": "assistant",
|
| 329 | "usage": {"input_tokens": usage_input, "output_tokens": 10, "cache_read_input_tokens": 0},
|
| 330 | "content": [{"type": "tool_use", "name": "Bash", "id": tool_id, "input": {"command": command}}],
|
| 331 | },
|
| 332 | }
|
| 333 |
|
| 334 |
|
| 335 | def claude_result_event(tool_id: str, content: str, is_error: bool = False) -> dict:
|
| 336 | return {
|
| 337 | "type": "user",
|
| 338 | "message": {
|
| 339 | "role": "user",
|
| 340 | "content": [{"type": "tool_result", "tool_use_id": tool_id, "is_error": is_error, "content": content}],
|
| 341 | },
|
| 342 | }
|
| 343 |
|
| 344 |
|
| 345 | class FollowupInfoCommandTest(unittest.TestCase):
|
| 346 | def test_vcs_info_family_classification(self) -> None:
|
| 347 | self.assertEqual(vcs_info_family("git status --short"), "status")
|
| 348 | self.assertEqual(vcs_info_family("oak diff --stat"), "diff")
|
| 349 | self.assertIsNone(vcs_info_family("git commit -m x"))
|
| 350 | self.assertIsNone(vcs_info_family("ls -la"))
|
| 351 |
|
| 352 | def test_followup_counted_when_same_family_reasked(self) -> None:
|
| 353 | events = [
|
| 354 | claude_tool_event("t1", "git status --short"),
|
| 355 | claude_result_event("t1", " M a.txt"),
|
| 356 | claude_tool_event("t2", "git status"),
|
| 357 | claude_result_event("t2", "modified: a.txt"),
|
| 358 | ]
|
| 359 | analysis = analyze_stream(events, "claude_print_stream_json")
|
| 360 | self.assertEqual(analysis.vcs_info_followup_calls_total, 1)
|
| 361 |
|
| 362 | def test_no_followup_when_action_intervenes(self) -> None:
|
| 363 | events = [
|
| 364 | claude_tool_event("t1", "git status --short"),
|
| 365 | claude_result_event("t1", " M a.txt"),
|
| 366 | claude_tool_event("t2", "git commit -m fix"),
|
| 367 | claude_result_event("t2", "committed"),
|
| 368 | claude_tool_event("t3", "git status"),
|
| 369 | claude_result_event("t3", "clean"),
|
| 370 | ]
|
| 371 | analysis = analyze_stream(events, "claude_print_stream_json")
|
| 372 | self.assertEqual(analysis.vcs_info_followup_calls_total, 0)
|
| 373 |
|
| 374 | def test_unresolvable_commands_leave_metric_unmeasured(self) -> None:
|
| 375 | # Results reference ids with no tool_use input.command anywhere:
|
| 376 | # the metric must be None (unmeasured), not a fabricated zero.
|
| 377 | events = [
|
| 378 | {"type": "assistant", "message": {"role": "assistant", "usage": {"input_tokens": 1, "output_tokens": 1}, "content": [{"type": "text", "text": "hi"}]}},
|
| 379 | claude_result_event("ghost", "output"),
|
| 380 | ]
|
| 381 | analysis = analyze_stream(events, "claude_print_stream_json")
|
| 382 | self.assertIsNone(analysis.vcs_info_followup_calls_total)
|
| 383 |
|
| 384 | def test_failed_call_breaks_followup_chain(self) -> None:
|
| 385 | events = [
|
| 386 | claude_tool_event("t1", "git status --short"),
|
| 387 | claude_result_event("t1", "error", is_error=True),
|
| 388 | claude_tool_event("t2", "git status"),
|
| 389 | claude_result_event("t2", "ok"),
|
| 390 | ]
|
| 391 | analysis = analyze_stream(events, "claude_print_stream_json")
|
| 392 | self.assertEqual(analysis.vcs_info_followup_calls_total, 0)
|
| 393 |
|
| 394 |
|
| 395 | if __name__ == "__main__":
|
| 396 | unittest.main()
|