Log in
benchmarksPublic

Viewing files at commit 25881039a2bbCommit detailsView latest

Browse files
tests/test_thrash.py 136 lines · 6.1 KB · python
1
from __future__ import annotations
2
3
import sys
4
import unittest
5
from pathlib import Path
6
7
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
8
9
from oakbench.thrash import agent_blocked_on_vcs_ms, thrash_events, vcs_share_of_task_wall
10
11
12
class VcsBlockedTimeTest(unittest.TestCase):
13
    def test_sums_measured_vcs_command_durations(self) -> None:
14
        calls = [
15
            {"command": "git status --short", "elapsed_ms": 12.25},
16
            {"command": "python3 -m unittest", "elapsed_ms": 100.0},
17
            {"command": "oak commit", "start_ms": 200.0, "end_ms": 250.5},
18
        ]
19
        self.assertEqual(agent_blocked_on_vcs_ms(calls), 62.75)
20
21
    def test_visible_non_vcs_calls_are_measured_zero(self) -> None:
22
        self.assertEqual(agent_blocked_on_vcs_ms([{"command": "python3 -m unittest", "elapsed_ms": 20}]), 0.0)
23
24
    def test_missing_vcs_timing_is_unmeasured(self) -> None:
25
        self.assertIsNone(agent_blocked_on_vcs_ms([{"command": "git status"}]))
26
27
    def test_commandless_tool_calls_are_filtered_before_measuring(self) -> None:
28
        self.assertIsNone(agent_blocked_on_vcs_ms([{"elapsed_ms": 12.0}]))
29
        self.assertEqual(
30
            agent_blocked_on_vcs_ms([{"command": "git status", "elapsed_ms": 1.0}, {"elapsed_ms": 2.0}]),
31
            1.0,
32
        )
33
        self.assertIsNone(agent_blocked_on_vcs_ms(None))
34
35
36
class VcsShareTest(unittest.TestCase):
37
    def test_share_uses_measured_values(self) -> None:
38
        self.assertEqual(vcs_share_of_task_wall(250.0, 1000.0), 0.25)
39
40
    def test_unmeasured_or_zero_wall_is_null(self) -> None:
41
        self.assertIsNone(vcs_share_of_task_wall(None, 1000.0))
42
        self.assertIsNone(vcs_share_of_task_wall(10.0, None))
43
        self.assertIsNone(vcs_share_of_task_wall(10.0, 0.0))
44
45
46
class ThrashEventsTest(unittest.TestCase):
47
    def test_detects_three_same_family_read_only_calls_in_thirty_seconds(self) -> None:
48
        calls = [
49
            {"command": "git status --short", "start_ms": 1_000.0, "returncode": 0},
50
            {"command": "git status", "start_ms": 10_000.0, "returncode": 0},
51
            {"command": "git status --porcelain", "start_ms": 30_000.0, "returncode": 0},
52
        ]
53
        events = thrash_events(calls)
54
        self.assertEqual(len(events), 1)
55
        self.assertEqual(events[0]["type"], "vcs_polling_loop")
56
        self.assertEqual(events[0]["family"], "status")
57
        self.assertEqual(events[0]["count"], 3)
58
        self.assertEqual(events[0]["window_ms"], 29_000.0)
59
60
    def test_growing_loop_emits_one_event_updated_in_place(self) -> None:
61
        calls = [
62
            {"command": "git status --short", "start_ms": 1_000.0, "returncode": 0},
63
            {"command": "git status", "start_ms": 2_000.0, "returncode": 0},
64
            {"command": "git status --porcelain", "start_ms": 3_000.0, "returncode": 0},
65
            {"command": "git status --short", "start_ms": 4_000.0, "returncode": 0},
66
        ]
67
        events = thrash_events(calls)
68
        self.assertEqual(len(events), 1)
69
        self.assertEqual(events[0]["family"], "status")
70
        self.assertEqual(events[0]["count"], 4)
71
        self.assertEqual(events[0]["start_ms"], 1_000.0)
72
        self.assertEqual(events[0]["end_ms"], 4_000.0)
73
        self.assertEqual(events[0]["window_ms"], 3_000.0)
74
        self.assertEqual(len(events[0]["commands"]), 4)
75
76
    def test_two_separated_bursts_emit_two_events(self) -> None:
77
        calls = [
78
            {"command": "git status", "start_ms": 0.0, "returncode": 0},
79
            {"command": "git status --short", "start_ms": 1_000.0, "returncode": 0},
80
            {"command": "git status --porcelain", "start_ms": 2_000.0, "returncode": 0},
81
            # Second burst entirely outside the first burst's 30s window.
82
            {"command": "git status", "start_ms": 60_000.0, "returncode": 0},
83
            {"command": "git status --short", "start_ms": 61_000.0, "returncode": 0},
84
            {"command": "git status --porcelain", "start_ms": 62_000.0, "returncode": 0},
85
        ]
86
        events = thrash_events(calls)
87
        self.assertEqual(len(events), 2)
88
        self.assertEqual(events[0]["count"], 3)
89
        self.assertEqual(events[0]["start_ms"], 0.0)
90
        self.assertEqual(events[0]["end_ms"], 2_000.0)
91
        self.assertEqual(events[1]["count"], 3)
92
        self.assertEqual(events[1]["start_ms"], 60_000.0)
93
        self.assertEqual(events[1]["end_ms"], 62_000.0)
94
95
    def test_mutation_breaks_polling_loop(self) -> None:
96
        calls = [
97
            {"command": "oak status --json", "start_ms": 1_000.0},
98
            {"command": "oak status", "start_ms": 2_000.0},
99
            {"command": "oak commit", "start_ms": 3_000.0},
100
            {"command": "oak status --json", "start_ms": 4_000.0},
101
        ]
102
        self.assertEqual(thrash_events(calls), [])
103
104
    def test_window_must_be_within_thirty_seconds(self) -> None:
105
        calls = [
106
            {"command": "git diff --stat", "start_ms": 0.0},
107
            {"command": "git diff", "start_ms": 20_000.0},
108
            {"command": "git diff --name-only", "start_ms": 31_000.0},
109
        ]
110
        self.assertEqual(thrash_events(calls), [])
111
112
    def test_different_families_do_not_combine(self) -> None:
113
        calls = [
114
            {"command": "git status", "start_ms": 0.0},
115
            {"command": "git diff", "start_ms": 1_000.0},
116
            {"command": "git log -1", "start_ms": 2_000.0},
117
        ]
118
        self.assertEqual(thrash_events(calls), [])
119
120
    def test_failed_read_only_call_does_not_count(self) -> None:
121
        calls = [
122
            {"command": "git status", "start_ms": 0.0, "returncode": 0},
123
            {"command": "git status --bad", "start_ms": 1_000.0, "returncode": 129},
124
            {"command": "git status --short", "start_ms": 2_000.0, "returncode": 0},
125
        ]
126
        self.assertEqual(thrash_events(calls), [])
127
128
    def test_missing_vcs_timestamps_are_unmeasured_but_commandless_calls_are_filtered(self) -> None:
129
        self.assertIsNone(thrash_events([{"command": "git status"}]))
130
        self.assertIsNone(thrash_events([{"start_ms": 0.0}]))
131
        self.assertEqual(thrash_events([{"command": "git status", "start_ms": 0.0}, {"start_ms": 1.0}]), [])
132
        self.assertIsNone(thrash_events(None))
133
134
135
if __name__ == "__main__":
136
    unittest.main()