Viewing files at commit 1dc7410826b4Commit detailsView latest

scripts/oakbench/thrash.py 242 lines · 7.9 KB · python
1
"""Agent VCS-thrash metrics.
2
3
Helpers operate on adapter-normalized tool-call dictionaries. Missing command
4
text or timing stays unmeasured (None), per ADR-0002; zero means observed zero.
5
"""
6
7
from __future__ import annotations
8
9
from collections.abc import Iterable, Mapping
10
from typing import Any
11
12
from oakbench.classify import vcs_info_family, vcs_subcommands
13
14
POLLING_LOOP_MIN_CALLS = 3
15
POLLING_LOOP_WINDOW_MS = 30_000.0
16
17
_START_KEYS = ("start_ms", "started_ms", "start_time_ms", "timestamp_ms", "time_ms")
18
_END_KEYS = ("end_ms", "ended_ms", "end_time_ms")
19
_ELAPSED_KEYS = ("elapsed_ms", "duration_ms", "wall_ms")
20
_COMMAND_KEYS = ("command", "cmd", "argv", "display")
21
22
23
def agent_blocked_on_vcs_ms(tool_calls: Iterable[Mapping[str, Any]] | None) -> float | None:
24
    """Total measured time the agent spent waiting on VCS tool calls."""
25
    calls = _materialize(tool_calls)
26
    if calls is None:
27
        return None
28
29
    saw_resolvable_command = False
30
    saw_vcs = False
31
    total = 0.0
32
    for call in calls:
33
        command = _command_text(call)
34
        if command is None:
35
            continue
36
        saw_resolvable_command = True
37
        if not vcs_subcommands(command):
38
            continue
39
        saw_vcs = True
40
        duration = _duration_ms(call)
41
        if duration is None:
42
            return None
43
        total += duration
44
45
    if not saw_resolvable_command:
46
        return None
47
    if not saw_vcs:
48
        return 0.0
49
    return round(total, 3)
50
51
52
def vcs_share_of_task_wall(vcs_ms: float | int | None, task_wall_ms: float | int | None) -> float | None:
53
    vcs_value = _non_negative_float(vcs_ms)
54
    wall_value = _non_negative_float(task_wall_ms)
55
    if vcs_value is None or wall_value is None or wall_value == 0:
56
        return None
57
    return round(vcs_value / wall_value, 6)
58
59
60
def thrash_events(tool_calls: Iterable[Mapping[str, Any]] | None) -> list[dict[str, Any]] | None:
61
    """Detect VCS polling loops in adapter-visible tool calls."""
62
    calls = _materialize(tool_calls)
63
    if calls is None:
64
        return None
65
66
    observations: list[dict[str, Any]] = []
67
    saw_resolvable_command = False
68
    for index, call in enumerate(calls):
69
        command = _command_text(call)
70
        if command is None:
71
            continue
72
        saw_resolvable_command = True
73
        if not vcs_subcommands(command):
74
            continue
75
        timestamp = _timestamp_ms(call)
76
        if timestamp is None:
77
            return None
78
        observations.append(
79
            {
80
                "index": index,
81
                "command": command,
82
                "family": vcs_info_family(command),
83
                "mutates": _vcs_mutates(command),
84
                "ok": _call_succeeded(call),
85
                "timestamp_ms": timestamp,
86
            }
87
        )
88
89
    if not saw_resolvable_command:
90
        return None
91
    if not observations:
92
        return []
93
94
    events: list[dict[str, Any]] = []
95
    open_by_family: dict[str, list[dict[str, Any]]] = {}
96
    # One event per polling loop, keyed by (family, start index). When the
97
    # same loop grows by another same-family call, the already-emitted event
98
    # is updated in place rather than appending a duplicate. The window list
99
    # is pruned by POLLING_LOOP_WINDOW_MS, so once old calls age out the
100
    # window's start index changes — which naturally begins a new event for
101
    # the next, separate burst.
102
    event_by_loop: dict[tuple[str, int], dict[str, Any]] = {}
103
104
    for obs in observations:
105
        if obs["mutates"]:
106
            open_by_family.clear()
107
            continue
108
        family = obs["family"]
109
        if family is None or not obs["ok"]:
110
            continue
111
112
        window = [
113
            item
114
            for item in open_by_family.get(family, [])
115
            if obs["timestamp_ms"] - item["timestamp_ms"] <= POLLING_LOOP_WINDOW_MS
116
        ]
117
        window.append(obs)
118
        open_by_family[family] = window
119
120
        if len(window) >= POLLING_LOOP_MIN_CALLS:
121
            key = (family, int(window[0]["index"]))
122
            event = event_by_loop.get(key)
123
            if event is None:
124
                event = {
125
                    "type": "vcs_polling_loop",
126
                    "family": family,
127
                    "count": len(window),
128
                    "window_ms": round(window[-1]["timestamp_ms"] - window[0]["timestamp_ms"], 3),
129
                    "start_ms": round(window[0]["timestamp_ms"], 3),
130
                    "end_ms": round(window[-1]["timestamp_ms"], 3),
131
                    "commands": [str(item["command"]) for item in window],
132
                    "threshold": {
133
                        "min_calls": POLLING_LOOP_MIN_CALLS,
134
                        "window_ms": POLLING_LOOP_WINDOW_MS,
135
                    },
136
                }
137
                event_by_loop[key] = event
138
                events.append(event)
139
            else:
140
                # Same loop grew: refresh the emitted event in place.
141
                event["count"] = len(window)
142
                event["window_ms"] = round(window[-1]["timestamp_ms"] - window[0]["timestamp_ms"], 3)
143
                event["end_ms"] = round(window[-1]["timestamp_ms"], 3)
144
                event["commands"] = [str(item["command"]) for item in window]
145
146
    return events
147
148
149
def _materialize(tool_calls: Iterable[Mapping[str, Any]] | None) -> list[Mapping[str, Any]] | None:
150
    if tool_calls is None:
151
        return None
152
    try:
153
        calls = list(tool_calls)
154
    except TypeError:
155
        return None
156
    if any(not isinstance(call, Mapping) for call in calls):
157
        return None
158
    return calls
159
160
161
def _command_text(call: Mapping[str, Any]) -> str | None:
162
    for key in _COMMAND_KEYS:
163
        value = call.get(key)
164
        if isinstance(value, str) and value.strip():
165
            return value.strip()
166
        if isinstance(value, (list, tuple)) and value:
167
            parts = [str(part) for part in value]
168
            if parts:
169
                return " ".join(parts)
170
    return None
171
172
173
def _duration_ms(call: Mapping[str, Any]) -> float | None:
174
    for key in _ELAPSED_KEYS:
175
        value = _non_negative_float(call.get(key))
176
        if value is not None:
177
            return value
178
    start = _first_number(call, _START_KEYS)
179
    end = _first_number(call, _END_KEYS)
180
    if start is None or end is None or end < start:
181
        return None
182
    return end - start
183
184
185
def _timestamp_ms(call: Mapping[str, Any]) -> float | None:
186
    start = _first_number(call, _START_KEYS)
187
    if start is not None:
188
        return start
189
    end = _first_number(call, _END_KEYS)
190
    duration = _duration_ms(call)
191
    if end is not None and duration is not None:
192
        return end - duration
193
    return None
194
195
196
def _first_number(call: Mapping[str, Any], keys: tuple[str, ...]) -> float | None:
197
    for key in keys:
198
        value = _non_negative_float(call.get(key))
199
        if value is not None:
200
            return value
201
    return None
202
203
204
def _non_negative_float(value: Any) -> float | None:
205
    if value is None or isinstance(value, bool):
206
        return None
207
    try:
208
        parsed = float(value)
209
    except (TypeError, ValueError):
210
        return None
211
    if parsed < 0:
212
        return None
213
    return parsed
214
215
216
def _call_succeeded(call: Mapping[str, Any]) -> bool:
217
    if call.get("failed") is True or call.get("is_error") is True:
218
        return False
219
    for key in ("returncode", "exit_code", "exitCode"):
220
        value = call.get(key)
221
        if isinstance(value, int) and not isinstance(value, bool):
222
            return value == 0
223
    status = call.get("status")
224
    if isinstance(status, str) and status.lower() in {"failed", "error"}:
225
        return False
226
    return True
227
228
229
def _vcs_mutates(command: str) -> bool:
230
    subcommands = vcs_subcommands(command)
231
    if not subcommands:
232
        return False
233
    return any(vcs_info_family(segment) is None for segment, _ in _vcs_segments(command, subcommands))
234
235
236
def _vcs_segments(command: str, subcommands: list[str]) -> list[tuple[str, str]]:
237
    from oakbench.classify import vcs_invocations
238
239
    invocations = vcs_invocations(command)
240
    if invocations:
241
        return invocations
242
    return [(command, subcommand) for subcommand in subcommands]