Viewing files at commit 2caecb1e6549Commit detailsView latest

scripts/oakbench/rows.py 369 lines · 12.8 KB · python
1
"""The row contract: canonical accessors and per-lane validation.
2
3
A Row is one JSONL object for one measured operation — the durable source of
4
truth (CONTEXT.md). This module is the single definition of how consumers read
5
the best-available token/tool/output values from any lane's rows, and of what
6
every lane's emitter must include. ResultsStore validates at write time, so a
7
contract violation fails the producing run instead of corrupting a dashboard.
8
9
Null means unmeasured (ADR-0002): validation checks presence and types, and
10
treats null as a legitimate value wherever a signal is probe-dependent.
11
"""
12
13
from __future__ import annotations
14
15
import math
16
from typing import Any, Optional
17
18
LANES = ("core", "workflow", "agent", "contention", "mount", "task-loop", "platform")
19
20
# Required top-level keys per lane. Deliberately the intersection that every
21
# row of the lane carries — including diagnostic rows (determinism probes,
22
# skip rows), which is why token fields are not required at this layer.
23
LANE_REQUIRED_KEYS: dict[str, tuple[str, ...]] = {
24
    "core": (
25
        "bench_id",
26
        "profile",
27
        "scenario",
28
        "operation",
29
        "run",
30
        "subject",
31
        "subject_kind",
32
        "elapsed_ms",
33
        "returncode",
34
        "command",
35
    ),
36
    "workflow": (
37
        "bench_id",
38
        "profile",
39
        "scenario",
40
        "workflow",
41
        "phase",
42
        "operation",
43
        "run",
44
        "subject",
45
        "subject_kind",
46
        "elapsed_ms",
47
        "returncode",
48
        "process_returncode",
49
        "command",
50
    ),
51
    "agent": (
52
        "schema_version",
53
        "bench_id",
54
        "scenario",
55
        "operation",
56
        "run",
57
        "subject",
58
        "subject_kind",
59
        "elapsed_ms",
60
        "returncode",
61
        "token_metrics",
62
        "turn_metrics",
63
        "tool_call_metrics",
64
    ),
65
    "contention": (
66
        "bench_id",
67
        "profile",
68
        "scenario",
69
        "contention_mode",
70
        "workers",
71
        "operation",
72
        "run",
73
        "subject",
74
        "subject_kind",
75
        "elapsed_ms",
76
        "returncode",
77
    ),
78
    "mount": (
79
        "bench_id",
80
        "profile",
81
        "scenario",
82
        "operation",
83
        "run",
84
        "subject",
85
        "elapsed_ms",
86
        "returncode",
87
    ),
88
    "task-loop": (
89
        "bench_id",
90
        "profile",
91
        "scenario",
92
        "operation",
93
        "run",
94
        "subject",
95
        "subject_kind",
96
        "elapsed_ms",
97
        "returncode",
98
        "command",
99
    ),
100
    # Platform lane: hosted-workflow rows are only comparable when both sides
101
    # ran the same semantics class under the same protection variant, so that
102
    # fairness context is part of every row — including skip rows. Runner
103
    # identity (ADR-0007) is required because this lane measures network
104
    # services from a specific vantage point.
105
    "platform": (
106
        "bench_id",
107
        "profile",
108
        "scenario",
109
        "operation",
110
        "run",
111
        "platform",
112
        "driver",
113
        "platform_semantics_version",
114
        "platform_semantics_class",
115
        "protection_variant",
116
        "platform_comparison_key",
117
        "semantic_contract_match",
118
        "elapsed_ms",
119
        "returncode",
120
        "command",
121
        "runner_id",
122
        "runner_class",
123
    ),
124
}
125
126
NUMERIC_KEYS = ("elapsed_ms",)
127
INTEGER_KEYS = ("returncode", "run")
128
SKIP_RETURNCODE = 77
129
130
# Optional metrics with constrained types/ranges when present. None is always
131
# legal (ADR-0002); a present value must be well-formed so consumers never
132
# need defensive parsing of the durable format.
133
OPTIONAL_NON_NEGATIVE_INT_KEYS = (
134
    "peak_rss_bytes",
135
    "tool_call_count",
136
    "probe_output_bytes",
137
    "changed_files_expected",
138
    "changed_files_recovered",
139
    "changed_files_reported",
140
    "changed_files_false_positive",
141
    "first_payload_bytes_server_to_client",
142
    "poll_iterations",
143
    "false_close_count",
144
    "close_eligible_n",
145
    "false_vcs_merge_safe_count",
146
    "vcs_merge_safe_n",
147
    "false_merge_allowed_count",
148
    "merge_allowed_n",
149
    "branches_scored",
150
    "branches_expected",
151
)
152
OPTIONAL_NON_NEGATIVE_NUMBER_KEYS = (
153
    "ttfd_ms",
154
    "observed_wait_ms",
155
    "settle_elapsed_ms",
156
    "poll_interval_s",
157
    "poll_quantization_ms",
158
    "status_post_elapsed_ms",
159
    "pacing_injected_ms",
160
)
161
OPTIONAL_UNIT_INTERVAL_KEYS = ("information_recall", "information_precision", "triage_recall", "triage_precision")
162
OPTIONAL_STRING_KEYS = (
163
    "runner_id",
164
    "runner_class",
165
    "fixture_id",
166
    "fixture_generator",
167
    "fixture_version",
168
    "fixture_seed",
169
    "fixture_manifest_sha256",
170
    "binary_sha256",
171
    "env_isolation_version",
172
    "platform_semantics_version",
173
    "platform_semantics_class",
174
    "protection_variant",
175
    "platform_comparison_key",
176
)
177
OPTIONAL_OBJECT_KEYS = ("runner_profile",)
178
179
180
class RowContractViolation(SystemExit):
181
    """Raised after rows are persisted; fails the producing run loudly."""
182
183
184
def validate_rows(rows: list[dict[str, Any]], lane: str) -> list[str]:
185
    if lane not in LANE_REQUIRED_KEYS:
186
        return [f"unknown lane {lane!r}; known: {', '.join(LANES)}"]
187
    required = LANE_REQUIRED_KEYS[lane]
188
    errors: list[str] = []
189
    for index, row in enumerate(rows, 1):
190
        if not isinstance(row, dict):
191
            errors.append(f"row {index}: expected object, got {type(row).__name__}")
192
            continue
193
        for key in required:
194
            if key not in row:
195
                errors.append(f"row {index} ({row.get('operation', '?')}): missing required key {key!r}")
196
        for key in NUMERIC_KEYS:
197
            value = row.get(key)
198
            if key in row and (
199
                isinstance(value, bool)
200
                or not isinstance(value, (int, float))
201
                or not math.isfinite(float(value))
202
                or float(value) < 0
203
            ):
204
                errors.append(f"row {index}: {key} must be numeric, got {type(value).__name__}")
205
        for key in INTEGER_KEYS:
206
            value = row.get(key)
207
            if key in row and (isinstance(value, bool) or not isinstance(value, int)):
208
                errors.append(f"row {index}: {key} must be an integer, got {type(value).__name__}")
209
        for key in OPTIONAL_NON_NEGATIVE_INT_KEYS:
210
            value = row.get(key)
211
            if key in row and value is not None:
212
                if isinstance(value, bool) or not isinstance(value, int) or value < 0:
213
                    errors.append(f"row {index}: {key} must be a non-negative integer or null, got {value!r}")
214
        for key in OPTIONAL_NON_NEGATIVE_NUMBER_KEYS:
215
            value = row.get(key)
216
            if key in row and value is not None:
217
                if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
218
                    errors.append(f"row {index}: {key} must be a non-negative finite number or null, got {value!r}")
219
        for key in OPTIONAL_UNIT_INTERVAL_KEYS:
220
            value = row.get(key)
221
            if key in row and value is not None:
222
                if not isinstance(value, (int, float)) or isinstance(value, bool) or not 0.0 <= float(value) <= 1.0:
223
                    errors.append(f"row {index}: {key} must be in [0, 1] or null, got {value!r}")
224
        for key in OPTIONAL_STRING_KEYS:
225
            value = row.get(key)
226
            if key in row and value is not None and not isinstance(value, str):
227
                errors.append(f"row {index}: {key} must be a string or null, got {type(value).__name__}")
228
        for key in OPTIONAL_OBJECT_KEYS:
229
            value = row.get(key)
230
            if key in row and value is not None and not isinstance(value, dict):
231
                errors.append(f"row {index}: {key} must be an object or null, got {type(value).__name__}")
232
        # Skip-row coherence: a skip is a recorded coverage gap, and consumers
233
        # key off both signals. Half-marked skips silently become failures
234
        # (or worse, successes) downstream.
235
        returncode = row.get("returncode")
236
        if returncode == SKIP_RETURNCODE:
237
            if not row.get("skipped"):
238
                errors.append(f"row {index} ({row.get('operation', '?')}): returncode 77 requires skipped: true")
239
            if not str(row.get("skip_reason") or "").strip():
240
                errors.append(f"row {index} ({row.get('operation', '?')}): skip rows require a non-empty skip_reason")
241
        elif row.get("skipped"):
242
            errors.append(
243
                f"row {index} ({row.get('operation', '?')}): skipped: true requires returncode {SKIP_RETURNCODE}"
244
            )
245
    return errors
246
247
248
def number_or_none(value: Any) -> Optional[float]:
249
    if value is None or isinstance(value, bool):
250
        return None
251
    try:
252
        parsed = float(value)
253
    except (TypeError, ValueError):
254
        return None
255
    if not math.isfinite(parsed):
256
        return None
257
    return parsed
258
259
260
def row_returncode(row: dict[str, Any]) -> int:
261
    """Row returncode with missing/null treated as failure, never a crash."""
262
    value = row.get("returncode")
263
    if value is None:
264
        return 1
265
    try:
266
        return int(value)
267
    except (TypeError, ValueError):
268
        return 1
269
270
271
def token_total(row: dict[str, Any]) -> Optional[float]:
272
    """Best-available token total for a row, preferring provider-reported usage."""
273
    token_metrics = row.get("token_metrics")
274
    if isinstance(token_metrics, dict):
275
        value = number_or_none(token_metrics.get("total_tokens_reported"))
276
        if value is not None:
277
            return value
278
        reported_parts = [
279
            number_or_none(token_metrics.get("input_tokens_reported")),
280
            number_or_none(token_metrics.get("output_tokens_reported")),
281
            number_or_none(token_metrics.get("reasoning_tokens_reported")),
282
        ]
283
        if any(part is not None for part in reported_parts):
284
            return sum(part or 0.0 for part in reported_parts)
285
286
    for key in ("estimated_tokens_total", "token_count", "total_tokens"):
287
        value = number_or_none(row.get(key))
288
        if value is not None:
289
            return value
290
291
    token_estimate = row.get("token_estimate")
292
    if isinstance(token_estimate, dict):
293
        value = number_or_none(token_estimate.get("total"))
294
        if value is not None:
295
            return value
296
297
    tokens = row.get("tokens")
298
    if isinstance(tokens, dict):
299
        for key in ("total", "total_tokens", "provider_total", "estimated_total"):
300
            value = number_or_none(tokens.get(key))
301
            if value is not None:
302
                return value
303
        provider = tokens.get("provider_reported")
304
        if isinstance(provider, dict):
305
            value = number_or_none(provider.get("total"))
306
            if value is not None:
307
                return value
308
309
    if isinstance(token_metrics, dict):
310
        chars = number_or_none(token_metrics.get("transcript_chars"))
311
        if chars is None:
312
            input_chars = number_or_none(token_metrics.get("normalized_input_chars"))
313
            output_chars = number_or_none(token_metrics.get("normalized_output_chars"))
314
            if input_chars is not None or output_chars is not None:
315
                chars = (input_chars or 0.0) + (output_chars or 0.0)
316
        if chars is not None:
317
            return max(1.0, math.ceil(chars / 4.0))
318
    return None
319
320
321
def tool_calls(row: dict[str, Any]) -> Optional[float]:
322
    for key in ("tool_call_count", "total_tool_calls"):
323
        value = number_or_none(row.get(key))
324
        if value is not None:
325
            return value
326
327
    nested = row.get("tool_calls")
328
    if isinstance(nested, dict):
329
        for key in ("total", "all"):
330
            value = number_or_none(nested.get(key))
331
            if value is not None:
332
                return value
333
334
    tool_call_metrics = row.get("tool_call_metrics")
335
    if isinstance(tool_call_metrics, dict):
336
        value = number_or_none(tool_call_metrics.get("tool_calls_total"))
337
        if value is not None:
338
            return value
339
    return None
340
341
342
def output_bytes(row: dict[str, Any]) -> Optional[float]:
343
    value = number_or_none(row.get("raw_output_bytes"))
344
    if value is not None:
345
        return value
346
347
    stdout = number_or_none(row.get("stdout_bytes"))
348
    stderr = number_or_none(row.get("stderr_bytes"))
349
    if stdout is not None or stderr is not None:
350
        return (stdout or 0.0) + (stderr or 0.0)
351
352
    token_estimate = row.get("token_estimate")
353
    if isinstance(token_estimate, dict):
354
        stdout = number_or_none(token_estimate.get("stdout_bytes"))
355
        stderr = number_or_none(token_estimate.get("stderr_bytes"))
356
        if stdout is not None or stderr is not None:
357
            return (stdout or 0.0) + (stderr or 0.0)
358
    return None
359
360
361
def output_truncated(row: dict[str, Any]) -> bool:
362
    if bool(row.get("output_truncated")):
363
        return True
364
    if bool(row.get("stdout_truncated")) or bool(row.get("stderr_truncated")):
365
        return True
366
    token_estimate = row.get("token_estimate")
367
    if isinstance(token_estimate, dict):
368
        return bool(token_estimate.get("stdout_truncated")) or bool(token_estimate.get("stderr_truncated"))
369
    return False