Viewing files at commit f08b4a249972Commit detailsView latest

scripts/oakbench/rows.py 458 lines · 16.5 KB · python
1
"""The row contract: canonical accessors and per-lane validation.
2
3
A Row is one JSONL object for one measured operation — the durable source of
4
truth (CONTEXT.md). This module is the single definition of how consumers read
5
the best-available token/tool/output values from any lane's rows, and of what
6
every lane's emitter must include. ResultsStore validates at write time, so a
7
contract violation fails the producing run instead of corrupting a dashboard.
8
9
Null means unmeasured (ADR-0002): validation checks presence and types, and
10
treats null as a legitimate value wherever a signal is probe-dependent.
11
"""
12
13
from __future__ import annotations
14
15
import math
16
from typing import Any, Optional
17
18
LANES = ("core", "workflow", "agent", "contention", "mount", "task-loop", "platform", "robustness", "scale")
19
20
# Required top-level keys per lane. Deliberately the intersection that every
21
# row of the lane carries — including diagnostic rows (determinism probes,
22
# skip rows), which is why token fields are not required at this layer.
23
LANE_REQUIRED_KEYS: dict[str, tuple[str, ...]] = {
24
    "core": (
25
        "bench_id",
26
        "profile",
27
        "scenario",
28
        "operation",
29
        "run",
30
        "subject",
31
        "subject_kind",
32
        "elapsed_ms",
33
        "returncode",
34
        "command",
35
    ),
36
    "workflow": (
37
        "bench_id",
38
        "profile",
39
        "scenario",
40
        "workflow",
41
        "phase",
42
        "operation",
43
        "run",
44
        "subject",
45
        "subject_kind",
46
        "elapsed_ms",
47
        "returncode",
48
        "process_returncode",
49
        "command",
50
    ),
51
    "agent": (
52
        "schema_version",
53
        "bench_id",
54
        "scenario",
55
        "operation",
56
        "run",
57
        "subject",
58
        "subject_kind",
59
        "elapsed_ms",
60
        "returncode",
61
        "token_metrics",
62
        "turn_metrics",
63
        "tool_call_metrics",
64
    ),
65
    "contention": (
66
        "bench_id",
67
        "profile",
68
        "scenario",
69
        "contention_mode",
70
        "workers",
71
        "operation",
72
        "run",
73
        "subject",
74
        "subject_kind",
75
        "elapsed_ms",
76
        "returncode",
77
    ),
78
    "mount": (
79
        "bench_id",
80
        "profile",
81
        "scenario",
82
        "operation",
83
        "run",
84
        "subject",
85
        "elapsed_ms",
86
        "returncode",
87
    ),
88
    "task-loop": (
89
        "bench_id",
90
        "profile",
91
        "scenario",
92
        "operation",
93
        "run",
94
        "subject",
95
        "subject_kind",
96
        "elapsed_ms",
97
        "returncode",
98
        "command",
99
    ),
100
    # Scale lane (suites/scale): every row names its tier and fixture identity
101
    # and states that it is development evidence until the calibrated campaign.
102
    "scale": (
103
        "bench_id",
104
        "profile",
105
        "scenario",
106
        "operation",
107
        "run",
108
        "subject",
109
        "subject_kind",
110
        "elapsed_ms",
111
        "returncode",
112
        "command",
113
        "tier",
114
        "fixture_id",
115
        "fixture_manifest_sha256",
116
        "evidence_class",
117
        "claim_eligible",
118
    ),
119
    # Platform lane: hosted-workflow rows are only comparable when both sides
120
    # ran the same semantics class under the same protection variant, so that
121
    # fairness context is part of every row — including skip rows. Runner
122
    # identity (ADR-0007) is required because this lane measures network
123
    # services from a specific vantage point.
124
    "platform": (
125
        "bench_id",
126
        "profile",
127
        "scenario",
128
        "operation",
129
        "run",
130
        "platform",
131
        "driver",
132
        "platform_semantics_version",
133
        "platform_semantics_class",
134
        "protection_variant",
135
        "platform_comparison_key",
136
        "semantic_contract_match",
137
        "elapsed_ms",
138
        "returncode",
139
        "command",
140
        "runner_id",
141
        "runner_class",
142
    ),
143
    # Robustness lane (suites/robustness): one row per fault case, judged by
144
    # independent readers. classification is held | violated | unsupported |
145
    # harness_error | fault_not_fired; unsupported and fault_not_fired cases are
146
    # skip rows (returncode 77). See ROBUSTNESS_CLASS_RETURNCODE.
147
    "robustness": (
148
        "bench_id",
149
        "scenario",
150
        "operation",
151
        "run",
152
        "subject",
153
        "subject_kind",
154
        "elapsed_ms",
155
        "returncode",
156
        "classification",
157
        "fault_family",
158
        "fault_point",
159
    ),
160
}
161
162
NUMERIC_KEYS = ("elapsed_ms",)
163
INTEGER_KEYS = ("returncode", "run")
164
SKIP_RETURNCODE = 77
165
166
# Optional metrics with constrained types/ranges when present. None is always
167
# legal (ADR-0002); a present value must be well-formed so consumers never
168
# need defensive parsing of the durable format.
169
OPTIONAL_NON_NEGATIVE_INT_KEYS = (
170
    "peak_rss_bytes",
171
    "tool_call_count",
172
    "probe_output_bytes",
173
    "changed_files_expected",
174
    "changed_files_recovered",
175
    "changed_files_reported",
176
    "changed_files_false_positive",
177
    "first_payload_bytes_server_to_client",
178
    "poll_iterations",
179
    "false_close_count",
180
    "close_eligible_n",
181
    "false_vcs_merge_safe_count",
182
    "vcs_merge_safe_n",
183
    "false_merge_allowed_count",
184
    "merge_allowed_n",
185
    "branches_scored",
186
    "branches_expected",
187
)
188
OPTIONAL_NON_NEGATIVE_NUMBER_KEYS = (
189
    "ttfd_ms",
190
    "observed_wait_ms",
191
    "settle_elapsed_ms",
192
    "poll_interval_s",
193
    "poll_quantization_ms",
194
    "status_post_elapsed_ms",
195
    "pacing_injected_ms",
196
)
197
OPTIONAL_UNIT_INTERVAL_KEYS = ("information_recall", "information_precision", "triage_recall", "triage_precision")
198
OPTIONAL_STRING_KEYS = (
199
    "runner_id",
200
    "runner_class",
201
    "fixture_id",
202
    "fixture_generator",
203
    "fixture_version",
204
    "fixture_seed",
205
    "fixture_manifest_sha256",
206
    "binary_sha256",
207
    "env_isolation_version",
208
    "platform_semantics_version",
209
    "platform_semantics_class",
210
    "protection_variant",
211
    "platform_comparison_key",
212
)
213
OPTIONAL_OBJECT_KEYS = ("runner_profile",)
214
215
216
class RowContractViolation(SystemExit):
217
    """Raised after rows are persisted; fails the producing run loudly."""
218
219
220
# Robustness lane: every classification and the returncode it must carry.
221
ROBUSTNESS_CLASS_RETURNCODE = {
222
    "held": 0,
223
    "violated": 1,
224
    "harness_error": 2,
225
    "unsupported": SKIP_RETURNCODE,
226
    "fault_not_fired": SKIP_RETURNCODE,
227
}
228
229
230
def _robustness_row_errors(index: int, row: dict[str, Any]) -> list[str]:
231
    op = row.get("operation", "?")
232
    classification = row.get("classification")
233
    if classification not in ROBUSTNESS_CLASS_RETURNCODE:
234
        return [f"row {index} ({op}): unknown robustness classification {classification!r}; "
235
                f"known: {', '.join(ROBUSTNESS_CLASS_RETURNCODE)}"]
236
    errors: list[str] = []
237
    expected = ROBUSTNESS_CLASS_RETURNCODE[classification]
238
    if row.get("returncode") != expected:
239
        errors.append(f"row {index} ({op}): classification {classification!r} requires returncode {expected}, "
240
                      f"got {row.get('returncode')!r}")
241
    if classification in ("held", "violated") and row.get("fault_fired") is False:
242
        errors.append(f"row {index} ({op}): a case whose fault did not fire cannot be {classification!r} "
243
                      "(use fault_not_fired)")
244
    violations = row.get("violations")
245
    if classification == "held" and violations:
246
        errors.append(f"row {index} ({op}): held row lists violations {violations!r}")
247
    if classification == "violated" and "violations" in row and not violations:
248
        errors.append(f"row {index} ({op}): violated row lists no violations")
249
    return errors
250
251
252
def validate_rows(rows: list[dict[str, Any]], lane: str) -> list[str]:
253
    if lane not in LANE_REQUIRED_KEYS:
254
        return [f"unknown lane {lane!r}; known: {', '.join(LANES)}"]
255
    required = LANE_REQUIRED_KEYS[lane]
256
    errors: list[str] = []
257
    for index, row in enumerate(rows, 1):
258
        if not isinstance(row, dict):
259
            errors.append(f"row {index}: expected object, got {type(row).__name__}")
260
            continue
261
        for key in required:
262
            if key not in row:
263
                errors.append(f"row {index} ({row.get('operation', '?')}): missing required key {key!r}")
264
        for key in NUMERIC_KEYS:
265
            value = row.get(key)
266
            diagnostic_timing = (lane == "core" and key == "elapsed_ms" and value is None
267
                and row.get("measurement_phase") == "output_diagnostic"
268
                and row.get("operation") in {
269
                    f"{operation}.{probe}" for operation in ("status.clean", "status.dirty", "diff.dirty", "diff.full")
270
                    for probe in ("inforecall", "determinism")})
271
            if diagnostic_timing:
272
                continue
273
            if key in row and (
274
                isinstance(value, bool)
275
                or not isinstance(value, (int, float))
276
                or not math.isfinite(float(value))
277
                or float(value) < 0
278
            ):
279
                errors.append(f"row {index}: {key} must be numeric, got {type(value).__name__}")
280
        for key in INTEGER_KEYS:
281
            value = row.get(key)
282
            if key in row and (isinstance(value, bool) or not isinstance(value, int)):
283
                errors.append(f"row {index}: {key} must be an integer, got {type(value).__name__}")
284
        for key in OPTIONAL_NON_NEGATIVE_INT_KEYS:
285
            value = row.get(key)
286
            if key in row and value is not None:
287
                if isinstance(value, bool) or not isinstance(value, int) or value < 0:
288
                    errors.append(f"row {index}: {key} must be a non-negative integer or null, got {value!r}")
289
        for key in OPTIONAL_NON_NEGATIVE_NUMBER_KEYS:
290
            value = row.get(key)
291
            if key in row and value is not None:
292
                if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
293
                    errors.append(f"row {index}: {key} must be a non-negative finite number or null, got {value!r}")
294
        for key in OPTIONAL_UNIT_INTERVAL_KEYS:
295
            value = row.get(key)
296
            if key in row and value is not None:
297
                if not isinstance(value, (int, float)) or isinstance(value, bool) or not 0.0 <= float(value) <= 1.0:
298
                    errors.append(f"row {index}: {key} must be in [0, 1] or null, got {value!r}")
299
        for key in OPTIONAL_STRING_KEYS:
300
            value = row.get(key)
301
            if key in row and value is not None and not isinstance(value, str):
302
                errors.append(f"row {index}: {key} must be a string or null, got {type(value).__name__}")
303
        for key in OPTIONAL_OBJECT_KEYS:
304
            value = row.get(key)
305
            if key in row and value is not None and not isinstance(value, dict):
306
                errors.append(f"row {index}: {key} must be an object or null, got {type(value).__name__}")
307
        # Skip-row coherence: a skip is a recorded coverage gap, and consumers
308
        # key off both signals. Half-marked skips silently become failures
309
        # (or worse, successes) downstream.
310
        returncode = row.get("returncode")
311
        if returncode == SKIP_RETURNCODE:
312
            if not row.get("skipped"):
313
                errors.append(f"row {index} ({row.get('operation', '?')}): returncode 77 requires skipped: true")
314
            if not str(row.get("skip_reason") or "").strip():
315
                errors.append(f"row {index} ({row.get('operation', '?')}): skip rows require a non-empty skip_reason")
316
        elif row.get("skipped"):
317
            errors.append(
318
                f"row {index} ({row.get('operation', '?')}): skipped: true requires returncode {SKIP_RETURNCODE}"
319
            )
320
        if lane == "robustness":
321
            errors.extend(_robustness_row_errors(index, row))
322
    return errors
323
324
325
def number_or_none(value: Any) -> Optional[float]:
326
    if value is None or isinstance(value, bool):
327
        return None
328
    try:
329
        parsed = float(value)
330
    except (TypeError, ValueError):
331
        return None
332
    if not math.isfinite(parsed):
333
        return None
334
    return parsed
335
336
337
def row_returncode(row: dict[str, Any]) -> int:
338
    """Row returncode with missing/null treated as failure, never a crash."""
339
    value = row.get("returncode")
340
    if value is None:
341
        return 1
342
    try:
343
        return int(value)
344
    except (TypeError, ValueError):
345
        return 1
346
347
348
def transcript_metrics_complete(row: dict[str, Any]) -> bool:
349
    source = row.get("source")
350
    execution = source.get("agent_execution") if isinstance(source, dict) else None
351
    is_agent = isinstance(row.get("agent"), dict) or str(row.get("operation", "")).startswith("agent.") or isinstance(execution, dict)
352
    return not ((is_agent and row.get("output_truncated") is True) or (isinstance(execution, dict) and
353
        (execution.get("usage_complete") is False or execution.get("transcript_metrics_complete") is False)))
354
355
356
def token_total(row: dict[str, Any]) -> Optional[float]:
357
    """Best-available token total for a row, preferring provider-reported usage."""
358
    if not transcript_metrics_complete(row):
359
        return None
360
    token_metrics = row.get("token_metrics")
361
    if isinstance(token_metrics, dict):
362
        value = number_or_none(token_metrics.get("total_tokens_reported"))
363
        if value is not None:
364
            return value
365
        reported_parts = [
366
            number_or_none(token_metrics.get("input_tokens_reported")),
367
            number_or_none(token_metrics.get("output_tokens_reported")),
368
            number_or_none(token_metrics.get("reasoning_tokens_reported")),
369
        ]
370
        if any(part is not None for part in reported_parts):
371
            return sum(part or 0.0 for part in reported_parts)
372
373
    for key in ("estimated_tokens_total", "token_count", "total_tokens"):
374
        value = number_or_none(row.get(key))
375
        if value is not None:
376
            return value
377
378
    token_estimate = row.get("token_estimate")
379
    if isinstance(token_estimate, dict):
380
        value = number_or_none(token_estimate.get("total"))
381
        if value is not None:
382
            return value
383
384
    tokens = row.get("tokens")
385
    if isinstance(tokens, dict):
386
        for key in ("total", "total_tokens", "provider_total", "estimated_total"):
387
            value = number_or_none(tokens.get(key))
388
            if value is not None:
389
                return value
390
        provider = tokens.get("provider_reported")
391
        if isinstance(provider, dict):
392
            value = number_or_none(provider.get("total"))
393
            if value is not None:
394
                return value
395
396
    if isinstance(token_metrics, dict):
397
        chars = number_or_none(token_metrics.get("transcript_chars"))
398
        if chars is None:
399
            input_chars = number_or_none(token_metrics.get("normalized_input_chars"))
400
            output_chars = number_or_none(token_metrics.get("normalized_output_chars"))
401
            if input_chars is not None or output_chars is not None:
402
                chars = (input_chars or 0.0) + (output_chars or 0.0)
403
        if chars is not None:
404
            return max(1.0, math.ceil(chars / 4.0))
405
    return None
406
407
408
def tool_calls(row: dict[str, Any]) -> Optional[float]:
409
    if not transcript_metrics_complete(row):
410
        return None
411
    for key in ("tool_call_count", "total_tool_calls"):
412
        value = number_or_none(row.get(key))
413
        if value is not None:
414
            return value
415
416
    nested = row.get("tool_calls")
417
    if isinstance(nested, dict):
418
        for key in ("total", "all"):
419
            value = number_or_none(nested.get(key))
420
            if value is not None:
421
                return value
422
423
    tool_call_metrics = row.get("tool_call_metrics")
424
    if isinstance(tool_call_metrics, dict):
425
        value = number_or_none(tool_call_metrics.get("tool_calls_total"))
426
        if value is not None:
427
            return value
428
    return None
429
430
431
def output_bytes(row: dict[str, Any]) -> Optional[float]:
432
    value = number_or_none(row.get("raw_output_bytes"))
433
    if value is not None:
434
        return value
435
436
    stdout = number_or_none(row.get("stdout_bytes"))
437
    stderr = number_or_none(row.get("stderr_bytes"))
438
    if stdout is not None or stderr is not None:
439
        return (stdout or 0.0) + (stderr or 0.0)
440
441
    token_estimate = row.get("token_estimate")
442
    if isinstance(token_estimate, dict):
443
        stdout = number_or_none(token_estimate.get("stdout_bytes"))
444
        stderr = number_or_none(token_estimate.get("stderr_bytes"))
445
        if stdout is not None or stderr is not None:
446
            return (stdout or 0.0) + (stderr or 0.0)
447
    return None
448
449
450
def output_truncated(row: dict[str, Any]) -> bool:
451
    if bool(row.get("output_truncated")):
452
        return True
453
    if bool(row.get("stdout_truncated")) or bool(row.get("stderr_truncated")):
454
        return True
455
    token_estimate = row.get("token_estimate")
456
    if isinstance(token_estimate, dict):
457
        return bool(token_estimate.get("stdout_truncated")) or bool(token_estimate.get("stderr_truncated"))
458
    return False