Log in
benchmarksPublic

Viewing files at commit 7630895676caCommit detailsView latest

Browse files
scripts/agent_metrics_schema.py 1017 lines · 42.1 KB · python
1
#!/usr/bin/env python3
2
"""Emit and validate Oak agent workflow benchmark result row schema.
3
4
The schema is JSON-schema-like but intentionally dependency-free. Running this
5
script with no arguments prints the schema as JSON. Use --validate-jsonl to check
6
JSON or JSONL result rows against the subset of schema rules implemented here.
7
"""
8
9
from __future__ import annotations
10
11
import argparse
12
import json
13
from pathlib import Path
14
from typing import Any, Iterable, List
15
16
SCHEMA_VERSION = 2
17
18
VCS_MODES = [
19
    "git_single_checkout",
20
    "git_worktree_per_task",
21
    "oak_single_mount",
22
    "oak_mount_per_task",
23
    "oak_space_per_task",
24
]
25
26
SUBJECT_KINDS = ["git", "oak", "other"]
27
OUTCOMES = ["pass", "fail", "timeout", "error", "abandoned"]
28
TOKEN_SOURCES = ["provider_reported", "adapter_estimated", "char_only", "mixed", "unknown"]
29
TURN_SOURCES = ["adapter_stream", "estimated", "unmeasured"]
30
INSTRUCTION_LEVELS = ["zero-shot", "cheat-sheet", "full-docs", "adapter-default"]
31
32
# Schema v2 measurement contract: a field that a runner cannot actually measure
33
# MUST be null, never a fabricated zero. Dashboards must render null as
34
# "unmeasured", not as a good result. Each metric group carries an optional
35
# measurement_source describing how its values were obtained.
36
37
38
def string(description: str = "") -> dict[str, Any]:
39
    schema: dict[str, Any] = {"type": "string"}
40
    if description:
41
        schema["description"] = description
42
    return schema
43
44
45
def optional_string(description: str = "") -> dict[str, Any]:
46
    schema: dict[str, Any] = {"type": ["string", "null"]}
47
    if description:
48
        schema["description"] = description
49
    return schema
50
51
52
def non_negative_integer(description: str = "") -> dict[str, Any]:
53
    schema: dict[str, Any] = {"type": "integer", "minimum": 0}
54
    if description:
55
        schema["description"] = description
56
    return schema
57
58
59
def optional_non_negative_integer(description: str = "") -> dict[str, Any]:
60
    schema: dict[str, Any] = {"type": ["integer", "null"], "minimum": 0}
61
    if description:
62
        schema["description"] = description
63
    return schema
64
65
66
def non_negative_number(description: str = "") -> dict[str, Any]:
67
    schema: dict[str, Any] = {"type": "number", "minimum": 0}
68
    if description:
69
        schema["description"] = description
70
    return schema
71
72
73
def optional_non_negative_number(description: str = "") -> dict[str, Any]:
74
    schema: dict[str, Any] = {"type": ["number", "null"], "minimum": 0}
75
    if description:
76
        schema["description"] = description
77
    return schema
78
79
80
def boolean(description: str = "") -> dict[str, Any]:
81
    schema: dict[str, Any] = {"type": "boolean"}
82
    if description:
83
        schema["description"] = description
84
    return schema
85
86
87
def optional_boolean(description: str = "") -> dict[str, Any]:
88
    schema: dict[str, Any] = {"type": ["boolean", "null"]}
89
    if description:
90
        schema["description"] = description
91
    return schema
92
93
94
def string_array(description: str = "") -> dict[str, Any]:
95
    schema: dict[str, Any] = {"type": "array", "items": {"type": "string"}}
96
    if description:
97
        schema["description"] = description
98
    return schema
99
100
101
TOKEN_METRICS_SCHEMA: dict[str, Any] = {
102
    "type": "object",
103
    "additionalProperties": False,
104
    "required": [
105
        "token_source",
106
        "input_tokens_reported",
107
        "output_tokens_reported",
108
        "reasoning_tokens_reported",
109
        "cache_read_tokens_reported",
110
        "cache_write_tokens_reported",
111
        "total_tokens_reported",
112
        "normalized_input_chars",
113
        "normalized_output_chars",
114
        "transcript_chars",
115
        "context_truncation_events",
116
        "token_budget",
117
        "cost_weighted_total",
118
    ],
119
    "properties": {
120
        "token_source": {"type": "string", "enum": TOKEN_SOURCES},
121
        "tokenizer": optional_string("Tokenizer or accounting method, when known."),
122
        "input_tokens_reported": optional_non_negative_integer("Provider-reported input tokens."),
123
        "output_tokens_reported": optional_non_negative_integer("Provider-reported output tokens."),
124
        "reasoning_tokens_reported": optional_non_negative_integer("Provider-reported reasoning tokens."),
125
        "cache_read_tokens_reported": optional_non_negative_integer("Provider-reported cache-read tokens."),
126
        "cache_write_tokens_reported": optional_non_negative_integer("Provider-reported cache-write tokens."),
127
        "total_tokens_reported": optional_non_negative_integer("Provider-reported total tokens."),
128
        "normalized_input_chars": non_negative_integer("Prompt and admitted tool-result characters."),
129
        "normalized_output_chars": optional_non_negative_integer("Assistant output characters."),
130
        "transcript_chars": optional_non_negative_integer("Full normalized transcript characters."),
131
        "context_truncation_events": optional_non_negative_integer(
132
            "Times context was compacted or truncated. Null when the adapter does not expose compaction events."
133
        ),
134
        "token_budget": optional_non_negative_integer("Configured token budget, if any."),
135
        "cost_weighted_total": optional_non_negative_number(
136
            "Billing-direction weighted token cost. Agent-emitted text (commands, assistant output) is "
137
            "model OUTPUT and weighted higher than agent-ingested tool results (model INPUT)."
138
        ),
139
        "cost_weights": optional_string("Weights used for cost_weighted_total, e.g. 'emitted=5,ingested=1'."),
140
    },
141
}
142
143
TOOL_CALL_METRICS_SCHEMA: dict[str, Any] = {
144
    "type": "object",
145
    "additionalProperties": False,
146
    "required": [
147
        "tool_calls_total",
148
        "tool_calls_by_kind",
149
        "terminal_commands_total",
150
        "file_read_calls",
151
        "file_write_calls",
152
        "search_calls",
153
        "edit_calls",
154
        "vcs_commands_total",
155
        "test_commands_total",
156
        "network_calls_total",
157
        "privileged_calls_total",
158
        "help_calls_total",
159
        "unknown_command_failures_total",
160
        "parallel_batches_total",
161
        "max_concurrency",
162
    ],
163
    "properties": {
164
        "tool_calls_total": optional_non_negative_integer(),
165
        "tool_calls_by_kind": {
166
            "type": ["object", "null"],
167
            "additionalProperties": optional_non_negative_integer("Count by adapter-defined tool kind."),
168
            "properties": {},
169
        },
170
        "terminal_commands_total": optional_non_negative_integer(),
171
        "file_read_calls": optional_non_negative_integer(),
172
        "file_write_calls": optional_non_negative_integer(),
173
        "search_calls": optional_non_negative_integer(),
174
        "edit_calls": optional_non_negative_integer(),
175
        "vcs_commands_total": optional_non_negative_integer(),
176
        "test_commands_total": optional_non_negative_integer(),
177
        "network_calls_total": optional_non_negative_integer(),
178
        "privileged_calls_total": optional_non_negative_integer(),
179
        "help_calls_total": optional_non_negative_integer(
180
            "Discovery calls: --help/-h flags, help subcommands, and man pages. "
181
            "High counts signal the agent does not know the tool zero-shot."
182
        ),
183
        "unknown_command_failures_total": optional_non_negative_integer(
184
            "Commands rejected as unknown subcommand/flag (hallucinated usage). Null when exit "
185
            "status or error text is not adapter-visible."
186
        ),
187
        "vcs_info_followup_calls_total": optional_non_negative_integer(
188
            "Successful read-only VCS info commands immediately re-asked in the same family "
189
            "(e.g. status, then status with more flags). Each one means the first output did "
190
            "not carry enough information to act on — the hidden cost of compact output. "
191
            "Null when per-call outcomes are not adapter-visible."
192
        ),
193
        "parallel_batches_total": optional_non_negative_integer(
194
            "Tool batches issued in parallel. Null when the adapter does not expose batching."
195
        ),
196
        "max_concurrency": optional_non_negative_integer(
197
            "Peak concurrent tool calls. Null when the adapter does not expose concurrency."
198
        ),
199
        "measurement_source": optional_string("How tool calls were observed, e.g. adapter_stream_events."),
200
    },
201
}
202
203
FAILURE_RETRY_METRICS_SCHEMA: dict[str, Any] = {
204
    "type": "object",
205
    "additionalProperties": False,
206
    "required": [
207
        "failed_tool_calls_total",
208
        "failed_commands_total",
209
        "failed_vcs_commands_total",
210
        "failed_test_commands_total",
211
        "retry_attempts_total",
212
        "retry_chains_total",
213
        "same_command_retries_total",
214
        "retry_after_failure_ms",
215
        "recoverable_failures_total",
216
        "unrecoverable_failures_total",
217
        "turns_to_recovery",
218
        "tokens_to_recovery",
219
    ],
220
    "properties": {
221
        "failed_tool_calls_total": optional_non_negative_integer(
222
            "Null when per-call success is not adapter-visible."
223
        ),
224
        "failed_commands_total": optional_non_negative_integer(),
225
        "failed_vcs_commands_total": optional_non_negative_integer(),
226
        "failed_test_commands_total": optional_non_negative_integer(),
227
        "retry_attempts_total": optional_non_negative_integer(),
228
        "retry_chains_total": optional_non_negative_integer(),
229
        "same_command_retries_total": optional_non_negative_integer(),
230
        "retry_after_failure_ms": optional_non_negative_number(),
231
        "recoverable_failures_total": optional_non_negative_integer(),
232
        "unrecoverable_failures_total": optional_non_negative_integer(),
233
        "turns_to_recovery": optional_non_negative_integer(
234
            "Assistant turns between the first failed tool call and the next successful tool call. "
235
            "Measures error-message ergonomics: a good error resolves in one turn."
236
        ),
237
        "tokens_to_recovery": optional_non_negative_integer(
238
            "Model output tokens spent in the recovery window, when per-turn usage is available."
239
        ),
240
        "failed_command_examples": string_array("Small redacted examples for debugging."),
241
        "measurement_source": optional_string("How failures/retries were observed."),
242
    },
243
}
244
245
WALL_CLOCK_METRICS_SCHEMA: dict[str, Any] = {
246
    "type": "object",
247
    "additionalProperties": False,
248
    "required": [
249
        "total_ms",
250
        "setup_ms",
251
        "agent_active_ms",
252
        "tool_wait_ms",
253
        "vcs_ms",
254
        "test_ms",
255
        "merge_ms",
256
        "cleanup_ms",
257
        "idle_ms",
258
        "timeout_ms",
259
    ],
260
    "properties": {
261
        "total_ms": non_negative_number(),
262
        "setup_ms": non_negative_number(),
263
        "agent_active_ms": optional_non_negative_number(),
264
        "tool_wait_ms": optional_non_negative_number(
265
            "Null when the adapter does not separate tool wait from model time."
266
        ),
267
        "vcs_ms": non_negative_number(),
268
        "test_ms": non_negative_number(),
269
        "merge_ms": optional_non_negative_number("Null when the scenario has no merge phase."),
270
        "cleanup_ms": optional_non_negative_number("Null when cleanup is not separately timed."),
271
        "idle_ms": optional_non_negative_number("Null when harness queueing is not separately timed."),
272
        "timeout_ms": optional_non_negative_number(),
273
        "measurement_source": optional_string("How phases were attributed."),
274
    },
275
}
276
277
BYTES_METRICS_SCHEMA: dict[str, Any] = {
278
    "type": "object",
279
    "additionalProperties": False,
280
    "required": [
281
        "workspace_logical_bytes",
282
        "workspace_materialized_bytes",
283
        "bytes_read_by_agent",
284
        "bytes_written_by_agent",
285
        "bytes_hydrated",
286
        "files_hydrated",
287
        "hydration_events",
288
        "large_file_bytes_scanned",
289
        "binary_bytes_touched",
290
        "command_output_bytes",
291
    ],
292
    "properties": {
293
        "workspace_logical_bytes": non_negative_integer("Fixture size if fully materialized."),
294
        "workspace_materialized_bytes": non_negative_integer("Bytes materialized before timed work."),
295
        "bytes_read_by_agent": optional_non_negative_integer(
296
            "File bytes read through agent-visible read tools. Null when read-tool payloads are "
297
            "not adapter-visible — never transcript output bytes, which live in command_output_bytes. "
298
            "Hydration/read-efficiency claims must not cite a proxy here."
299
        ),
300
        "bytes_written_by_agent": non_negative_integer("File bytes written through agent-visible tools."),
301
        "bytes_hydrated": optional_non_negative_integer(
302
            "Bytes loaded lazily by VCS or filesystem. Null when no hydration probe ran."
303
        ),
304
        "files_hydrated": optional_non_negative_integer(
305
            "Files loaded lazily by VCS or filesystem. Null when no hydration probe ran."
306
        ),
307
        "hydration_events": optional_non_negative_integer(
308
            "Hydration events observed by the harness. Null when no hydration probe ran."
309
        ),
310
        "large_file_bytes_scanned": optional_non_negative_integer(
311
            "Bytes scanned in known large-file fixtures. Null when not instrumented."
312
        ),
313
        "binary_bytes_touched": optional_non_negative_integer(
314
            "Binary bytes read or written. Null when not instrumented."
315
        ),
316
        "command_output_bytes": non_negative_integer("Terminal output bytes admitted to transcript."),
317
        "measurement_source": optional_string("Adapter, filesystem probe, VCS trace, or estimate."),
318
    },
319
}
320
321
VCS_METRICS_SCHEMA: dict[str, Any] = {
322
    "type": "object",
323
    "additionalProperties": False,
324
    "required": [
325
        "commands_total",
326
        "status_commands",
327
        "diff_commands",
328
        "snapshot_commands",
329
        "branch_commands",
330
        "merge_commands",
331
        "cleanup_commands",
332
        "commits_created",
333
        "branches_created",
334
        "conflicts_total",
335
        "conflicts_resolved",
336
        "dirty_files_at_end",
337
        "cleanup_success",
338
    ],
339
    "properties": {
340
        "commands_total": optional_non_negative_integer(
341
            "Null when the agent transcript does not expose VCS command invocations."
342
        ),
343
        "status_commands": optional_non_negative_integer(),
344
        "diff_commands": optional_non_negative_integer(),
345
        "snapshot_commands": optional_non_negative_integer(),
346
        "branch_commands": optional_non_negative_integer(),
347
        "merge_commands": optional_non_negative_integer(),
348
        "cleanup_commands": optional_non_negative_integer(),
349
        "commits_created": non_negative_integer(),
350
        "branches_created": optional_non_negative_integer(),
351
        "conflicts_total": optional_non_negative_integer(
352
            "Null when conflict detection is not instrumented for the scenario."
353
        ),
354
        "conflicts_resolved": optional_non_negative_integer(),
355
        "dirty_files_at_end": optional_non_negative_integer(
356
            "Null when the final status command fails or its output cannot be interpreted."
357
        ),
358
        "cleanup_success": boolean(),
359
        "measurement_source": optional_string("How VCS command outcomes were observed."),
360
        # PATH-shim attribution fields (optional, additive). Null when the
361
        # --vcs-shim instrument was off or recorded nothing: unmeasured, not
362
        # zero (ADR-0002).
363
        "shim_sidecar_path": optional_string(
364
            "Artifact path of the shim sidecar JSONL ({command, argv, start_ms, end_ms, "
365
            "elapsed_ms, returncode, shim} per VCS call). Null when the shim was off."
366
        ),
367
        "shim_sidecar_path_unavailable_reason": optional_string(
368
            "Reason shim_sidecar_path is null."
369
        ),
370
        "shim_overhead": optional_non_negative_number(
371
            "Requested alias for shim_overhead_ms: median extra ms per call added by the PATH shim."
372
        ),
373
        "shim_overhead_ms": optional_non_negative_number(
374
            "Median extra ms per call added by the PATH shim, measured per run via a "
375
            "calibration pass. Recorded for interpretation, never subtracted from measurements."
376
        ),
377
        "shim_overhead_unavailable_reason": optional_string(
378
            "Reason shim_overhead/shim_overhead_ms is null."
379
        ),
380
        "vcs_ms": optional_non_negative_number(
381
            "Sum of shim-recorded VCS call wall time (includes shim overhead)."
382
        ),
383
        "vcs_ms_unavailable_reason": optional_string("Reason shim-recorded vcs_ms is null."),
384
        "vcs_call_count_shim": optional_non_negative_integer(
385
            "Number of VCS binary invocations recorded by the PATH shim."
386
        ),
387
        "vcs_call_count_shim_unavailable_reason": optional_string(
388
            "Reason vcs_call_count_shim is null."
389
        ),
390
        "agent_blocked_on_vcs_ms": optional_non_negative_number(
391
            "oakbench.thrash.agent_blocked_on_vcs_ms over the shim sidecar rows."
392
        ),
393
        "agent_blocked_on_vcs_ms_unavailable_reason": optional_string(
394
            "Reason agent_blocked_on_vcs_ms is null."
395
        ),
396
        "vcs_share_of_task_wall": optional_non_negative_number(
397
            "agent_blocked_on_vcs_ms divided by agent wall time."
398
        ),
399
        "vcs_share_of_task_wall_unavailable_reason": optional_string(
400
            "Reason vcs_share_of_task_wall is null."
401
        ),
402
        "thrash_events_count": optional_non_negative_integer(
403
            "Count of oakbench.thrash.thrash_events detected in the shim sidecar rows."
404
        ),
405
        "thrash_events_count_unavailable_reason": optional_string(
406
            "Reason thrash_events_count is null."
407
        ),
408
        "polling_loop_count": optional_non_negative_integer(
409
            "Count of shim-sidecar thrash events with type vcs_polling_loop."
410
        ),
411
        "polling_loop_count_unavailable_reason": optional_string(
412
            "Reason polling_loop_count is null."
413
        ),
414
    },
415
}
416
417
TEST_METRICS_SCHEMA: dict[str, Any] = {
418
    "type": "object",
419
    "additionalProperties": False,
420
    "required": [
421
        "commands_total",
422
        "passed_commands",
423
        "failed_commands",
424
        "assertions_total",
425
        "assertions_failed",
426
        "required_checks_passed",
427
    ],
428
    "properties": {
429
        "commands_total": optional_non_negative_integer(),
430
        "passed_commands": optional_non_negative_integer(),
431
        "failed_commands": optional_non_negative_integer(),
432
        "assertions_total": optional_non_negative_integer(),
433
        "assertions_failed": optional_non_negative_integer(),
434
        "required_checks_passed": optional_boolean(),
435
    },
436
}
437
438
PARALLEL_METRICS_SCHEMA: dict[str, Any] = {
439
    "type": "object",
440
    "additionalProperties": False,
441
    "required": [
442
        "tasks_total",
443
        "tasks_completed",
444
        "max_parallel_tasks",
445
        "overlapping_files_count",
446
        "lost_updates_detected",
447
    ],
448
    "properties": {
449
        "tasks_total": non_negative_integer(),
450
        "tasks_completed": non_negative_integer(),
451
        "max_parallel_tasks": non_negative_integer(),
452
        "overlapping_files_count": optional_non_negative_integer(
453
            "Null when overlap detection did not run (e.g. single-task scenarios)."
454
        ),
455
        "lost_updates_detected": optional_non_negative_integer(
456
            "Null when no integrity check ran. Zero means the check ran and found nothing lost."
457
        ),
458
        "lock_wait_ms": optional_non_negative_number(
459
            "Time spent blocked on or retrying VCS locks under contention."
460
        ),
461
        "commit_throughput_per_s": optional_non_negative_number(
462
            "Successful snapshots per second across all parallel workers."
463
        ),
464
        "integrity_check_passed": optional_boolean(
465
            "fsck-equivalent result after the parallel run. Null when no check ran."
466
        ),
467
        "measurement_source": optional_string("How parallel behavior was observed."),
468
    },
469
}
470
471
TURN_METRICS_SCHEMA: dict[str, Any] = {
472
    "type": "object",
473
    "additionalProperties": False,
474
    "required": [
475
        "turn_source",
476
        "assistant_turns_total",
477
        "tool_calls_per_turn_avg",
478
        "max_tool_calls_per_turn",
479
        "cumulative_input_tokens_reported",
480
        "peak_input_tokens_reported",
481
    ],
482
    "properties": {
483
        "turn_source": {"type": "string", "enum": TURN_SOURCES},
484
        "assistant_turns_total": optional_non_negative_integer(
485
            "Model inference round-trips. Each turn re-pays the transcript as input tokens, so "
486
            "turns dominate agent cost; tool-call counts alone hide this."
487
        ),
488
        "tool_calls_per_turn_avg": optional_non_negative_number(),
489
        "max_tool_calls_per_turn": optional_non_negative_integer(
490
            "Peak batching within one turn. Batched calls are far cheaper than sequential turns."
491
        ),
492
        "cumulative_input_tokens_reported": optional_non_negative_integer(
493
            "Sum of provider-reported input tokens (including cache reads) across all turns. "
494
            "Captures context-residency cost: early verbose output is re-paid every later turn."
495
        ),
496
        "peak_input_tokens_reported": optional_non_negative_integer(
497
            "Largest single-turn input, a proxy for peak context size."
498
        ),
499
        "turn_timeline_path": optional_string(
500
            "Artifact path of per-turn JSONL: turn index, input/output/cache tokens, tool calls."
501
        ),
502
    },
503
}
504
505
SCHEMA: dict[str, Any] = {
506
    "$schema": "https://json-schema.org/draft/2020-12/schema",
507
    "title": "Oak agent workflow benchmark result row",
508
    "schema_version": SCHEMA_VERSION,
509
    "type": "object",
510
    "additionalProperties": False,
511
    "required": [
512
        "schema_version",
513
        "bench_id",
514
        "timestamp_utc",
515
        "scenario",
516
        "run",
517
        "subject",
518
        "subject_kind",
519
        "vcs_mode",
520
        "agent",
521
        "fixture",
522
        "outcome",
523
        "success",
524
        "success_criteria",
525
        "token_metrics",
526
        "turn_metrics",
527
        "tool_call_metrics",
528
        "failed_command_retry_metrics",
529
        "wall_clock_metrics",
530
        "bytes_metrics",
531
        "vcs_metrics",
532
        "test_metrics",
533
        "parallel_metrics",
534
    ],
535
    "properties": {
536
        "schema_version": {"type": "integer", "const": SCHEMA_VERSION},
537
        "bench_id": string("Stable identifier for a benchmark invocation."),
538
        "timestamp_utc": string("UTC ISO-8601 timestamp for row completion."),
539
        "profile": optional_string("Run profile such as smoke, standard, or large."),
540
        "host": optional_string("Host name used for run provenance."),
541
        "platform": optional_string("Platform string used for run provenance."),
542
        "machine": optional_string("Machine architecture used for run provenance."),
543
        "scenario": string("Scenario id from scenarios/agent.yaml."),
544
        "scenario_version": optional_string("Fixture or prompt version for this scenario."),
545
        "randomized_subject_order": {"type": "boolean"},
546
        "subject_order": {"type": "array", "items": {"type": "string"}},
547
        "agent_launched": boolean("Whether an agent process or mock was invoked."),
548
        "outcome_attribution": {"type": "string", "enum": ["agent", "subject", "harness"]},
549
        "setup_returncode": {"type": "integer"},
550
        "skipped": boolean("Whether this trial lacks a required measurement instrument."),
551
        "runner_id": optional_string(),
552
        "runner_class": optional_string(),
553
        "runner_profile": {"type": "object"},
554
        "cache_state": optional_string(),
555
        "cache_state_reason": optional_string(),
556
        "calibration_version": optional_string(),
557
        "calibration_unmeasured_reason": optional_string(),
558
        "workflow_integrity_evidence_class": optional_string(),
559
        "workflow_integrity_committed_scope": {"type": ["array", "null"], "items": {"type": "string"},
560
            "description": "Committed paths read by the Git or Oak committed-tree check (same scope for both)."},
561
        "workflow_subject_binary_sha256": optional_string(),
562
        "env_isolation_version": optional_string("Version of environment isolation policy."),
563
        "oracle_version": optional_string("Version of task correctness evidence."),
564
        "workflow_integrity_source": optional_string("Actual worktree/committed-tree evidence source."),
565
        "workflow_integrity_passed": optional_boolean("Null when required correctness instrument is unavailable."),
566
        "workflow_integrity_failure_reason": optional_string("Observed incorrect task state."),
567
        "workflow_integrity_unmeasured_reason": optional_string("Why required correctness evidence is unavailable."),
568
        "skip_reason": optional_string("Explicit missing measurement capability."),
569
        "git_head_instrument_binary": optional_string("Resolved Git object-reading instrument."),
570
        "git_head_instrument_sha256": optional_string("Git object reader SHA256."),
571
        "git_head_instrument_version": optional_string("Git object reader reported version."),
572
        "oak_export_instrument_binary": optional_string("Subject-produced export binary."),
573
        "oak_export_instrument_sha256": optional_string("Subject-produced export binary SHA256."),
574
        "oak_reader_version": optional_string(),
575
"oak_reader_sha256": optional_string(),
576
"oak_reader_head": optional_string(),
577
"oak_reader_blake3_version": optional_string(),
578
"oak_reader_zstandard_version": optional_string(),
579
"oak_reader_backend": optional_string(),
580
"oak_reader_codec_sha256": optional_string(),
581
"oak_reader_python_sha256": optional_string(),
582
        "oak_export_head": optional_string("Oak HEAD checked unchanged across export."),
583
        "operation": optional_string("Optional report grouping operation, e.g. agent.workflow."),
584
        "benchmark_track": optional_string("Command semantics track such as agent-default or core-equivalent."),
585
        "command_semantics_version": optional_string("Version of the command semantics contract."),
586
        "elapsed_ms": optional_non_negative_number("Compatibility mirror of wall_clock_metrics.total_ms."),
587
        "returncode": optional_non_negative_integer("Compatibility mirror: 0 for success, non-zero for failure."),
588
        "raw_output_bytes": optional_non_negative_integer("Compatibility mirror of captured agent stdout+stderr bytes."),
589
        "stdout_bytes": optional_non_negative_integer("Captured agent stdout bytes."),
590
        "stderr_bytes": optional_non_negative_integer("Captured agent stderr bytes."),
591
        "output_truncated": optional_boolean("Whether captured output was truncated before artifact storage."),
592
        "task_id": optional_string("Subtask id for parallel scenarios, if this row is per subtask."),
593
        "task_prompt_id": optional_string("Stable prompt/template id."),
594
        "run": non_negative_integer("Zero-based repetition index."),
595
        "subject": string("Benchmark subject such as git, oak_installed, or oak_local."),
596
        "subject_kind": {"type": "string", "enum": SUBJECT_KINDS},
597
        "subject_label": optional_string("Human label for reports."),
598
        "vcs_mode": {"type": "string", "enum": VCS_MODES},
599
        "agent": {
600
            "type": "object",
601
            "additionalProperties": False,
602
            "required": ["name", "adapter"],
603
            "properties": {
604
                "name": string("Agent or harness adapter name."),
605
                "adapter": string("Runner integration name."),
606
                "provider": optional_string("Provider name, if applicable."),
607
                "model": optional_string("Model id, if applicable."),
608
                "version": optional_string("Agent binary, prompt, or adapter version."),
609
                "environment": optional_string("Agent environment mode, such as minimal or local-default."),
610
                "instruction_level": {
611
                    "type": ["string", "null"],
612
                    "enum": INSTRUCTION_LEVELS + [None],
613
                    "description": (
614
                        "How much VCS-specific instruction the prompt carried. zero-shot measures the "
615
                        "model-familiarity tax: Git is in pretraining data, a new VCS is not."
616
                    ),
617
                },
618
                "temperature": {"type": ["number", "null"]},
619
                "max_output_tokens": optional_non_negative_integer(),
620
            },
621
        },
622
        "fixture": {
623
            "type": "object",
624
            "additionalProperties": False,
625
            "required": ["id", "repo_shape", "fixture_version"],
626
            "properties": {
627
                "id": string("Fixture id used by the runner."),
628
                "repo_shape": string("Human-readable fixture shape."),
629
                "fixture_version": string("Stable fixture content version or hash."),
630
                "logical_file_count": optional_non_negative_integer(),
631
                "logical_bytes": optional_non_negative_integer(),
632
            },
633
        },
634
        "source": {
635
            "type": "object",
636
            "additionalProperties": True,
637
            "properties": {
638
                "oak_hash": optional_string(),
639
                "oak_status": optional_string(),
640
                "benchmark_repo_hash": optional_string(),
641
            },
642
        },
643
        "outcome": {"type": "string", "enum": OUTCOMES},
644
        "success": boolean("True only when deterministic scenario criteria passed."),
645
        "failure_reason": optional_string("Short stable failure reason for dashboards."),
646
        "success_criteria": {
647
            "type": "object",
648
            "additionalProperties": False,
649
            "required": ["passed", "checks"],
650
            "properties": {
651
                "passed": boolean(),
652
                "checks": {
653
                    "type": "object",
654
                    "additionalProperties": optional_boolean("Individual deterministic check result; null if unmeasured."),
655
                    "properties": {},
656
                },
657
                "notes": optional_string(),
658
            },
659
        },
660
        "token_metrics": TOKEN_METRICS_SCHEMA,
661
        "turn_metrics": TURN_METRICS_SCHEMA,
662
        "tool_call_metrics": TOOL_CALL_METRICS_SCHEMA,
663
        "failed_command_retry_metrics": FAILURE_RETRY_METRICS_SCHEMA,
664
        "wall_clock_metrics": WALL_CLOCK_METRICS_SCHEMA,
665
        "bytes_metrics": BYTES_METRICS_SCHEMA,
666
        "vcs_metrics": VCS_METRICS_SCHEMA,
667
        "test_metrics": TEST_METRICS_SCHEMA,
668
        "parallel_metrics": PARALLEL_METRICS_SCHEMA,
669
        "artifacts": {
670
            "type": "object",
671
            "additionalProperties": False,
672
            "properties": {
673
                "transcript_path": optional_string(),
674
                "command_log_path": optional_string(),
675
                "diff_path": optional_string(),
676
                "workspace_path": optional_string(),
677
                "extra_paths": string_array(),
678
            },
679
        },
680
        "notes": optional_string(),
681
    },
682
}
683
684
685
def type_matches(value: Any, expected_type: Any) -> bool:
686
    if isinstance(expected_type, list):
687
        return any(type_matches(value, item) for item in expected_type)
688
    if expected_type == "null":
689
        return value is None
690
    if expected_type == "boolean":
691
        return isinstance(value, bool)
692
    if expected_type == "integer":
693
        return isinstance(value, int) and not isinstance(value, bool)
694
    if expected_type == "number":
695
        return (isinstance(value, int) or isinstance(value, float)) and not isinstance(value, bool)
696
    if expected_type == "string":
697
        return isinstance(value, str)
698
    if expected_type == "object":
699
        return isinstance(value, dict)
700
    if expected_type == "array":
701
        return isinstance(value, list)
702
    return True
703
704
705
def display_path(path: str, key: str) -> str:
706
    if path == "$":
707
        return "$" + "." + key
708
    return path + "." + key
709
710
711
def validate_value(value: Any, schema: dict[str, Any], path: str) -> List[str]:
712
    errors: List[str] = []
713
    expected_type = schema.get("type")
714
    if expected_type is not None and not type_matches(value, expected_type):
715
        errors.append(f"{path}: expected {expected_type}, got {type(value).__name__}")
716
        return errors
717
718
    if "const" in schema and value != schema["const"]:
719
        errors.append(f"{path}: expected constant {schema['const']!r}, got {value!r}")
720
    if "enum" in schema and value not in schema["enum"]:
721
        errors.append(f"{path}: expected one of {schema['enum']!r}, got {value!r}")
722
    if "minimum" in schema and isinstance(value, (int, float)) and not isinstance(value, bool):
723
        if value < schema["minimum"]:
724
            errors.append(f"{path}: expected >= {schema['minimum']}, got {value!r}")
725
726
    if isinstance(value, dict):
727
        required = schema.get("required", [])
728
        for key in required:
729
            if key not in value:
730
                errors.append(f"{path}: missing required property {key!r}")
731
        properties = schema.get("properties", {})
732
        additional = schema.get("additionalProperties", True)
733
        for key, item in value.items():
734
            child_path = display_path(path, key)
735
            if key in properties:
736
                errors.extend(validate_value(item, properties[key], child_path))
737
            elif additional is False:
738
                errors.append(f"{child_path}: unexpected property")
739
            elif isinstance(additional, dict):
740
                errors.extend(validate_value(item, additional, child_path))
741
    elif isinstance(value, list) and "items" in schema:
742
        item_schema = schema["items"]
743
        for index, item in enumerate(value):
744
            errors.extend(validate_value(item, item_schema, f"{path}[{index}]"))
745
746
    return errors
747
748
749
def validate_schema_shape(schema: dict[str, Any]) -> List[str]:
750
    errors: List[str] = []
751
    for key in ["title", "type", "required", "properties"]:
752
        if key not in schema:
753
            errors.append(f"schema missing {key!r}")
754
    if schema.get("type") != "object":
755
        errors.append("schema root type must be object")
756
    properties = schema.get("properties", {})
757
    for required_key in schema.get("required", []):
758
        if required_key not in properties:
759
            errors.append(f"schema requires {required_key!r} but has no property definition")
760
    return errors
761
762
763
def validate_row(row: Any) -> List[str]:
764
    if not isinstance(row, dict):
765
        return [f"$: expected object row, got {type(row).__name__}"]
766
    return validate_value(row, SCHEMA, "$")
767
768
769
def iter_rows(path: Path) -> Iterable[Any]:
770
    text = path.read_text(encoding="utf-8")
771
    stripped = text.strip()
772
    if not stripped:
773
        return []
774
    if path.suffix == ".json" or stripped[0] == "[":
775
        data = json.loads(stripped)
776
        if isinstance(data, list):
777
            return data
778
        return [data]
779
    rows: List[Any] = []
780
    for line_number, line in enumerate(text.splitlines(), 1):
781
        if not line.strip():
782
            continue
783
        try:
784
            rows.append(json.loads(line))
785
        except json.JSONDecodeError as exc:
786
            rows.append({"__json_error__": f"line {line_number}: {exc}"})
787
    return rows
788
789
790
def validate_file(path: Path) -> dict[str, Any]:
791
    errors: List[str] = []
792
    rows = list(iter_rows(path))
793
    for index, row in enumerate(rows, 1):
794
        if isinstance(row, dict) and "__json_error__" in row:
795
            errors.append(row["__json_error__"])
796
            continue
797
        for error in validate_row(row):
798
            errors.append(f"row {index}: {error}")
799
    return {"ok": not errors, "rows": len(rows), "errors": errors}
800
801
802
def example_row() -> dict[str, Any]:
803
    return {
804
        "schema_version": SCHEMA_VERSION,
805
        "bench_id": "20260609T000000Z",
806
        "timestamp_utc": "2026-06-09T00:00:00Z",
807
        "profile": "smoke",
808
        "scenario": "one_line_bug_fix",
809
        "scenario_version": "v1",
810
        "task_id": None,
811
        "task_prompt_id": "agent-one-line-bug-fix-v1",
812
        "run": 0,
813
        "subject": "oak_local",
814
        "subject_kind": "oak",
815
        "subject_label": "Oak local",
816
        "vcs_mode": "oak_space_per_task",
817
        "agent": {
818
            "name": "example-agent",
819
            "adapter": "example-adapter",
820
            "provider": None,
821
            "model": None,
822
            "version": None,
823
            "environment": "minimal",
824
            "instruction_level": "cheat-sheet",
825
            "temperature": None,
826
            "max_output_tokens": None,
827
        },
828
        "fixture": {
829
            "id": "agent_one_line_bug_fix",
830
            "repo_shape": "small text application",
831
            "fixture_version": "v1",
832
            "logical_file_count": 42,
833
            "logical_bytes": 65536,
834
        },
835
        "source": {"oak_hash": None, "oak_status": None, "benchmark_repo_hash": None},
836
        "outcome": "pass",
837
        "success": True,
838
        "failure_reason": None,
839
        "success_criteria": {
840
            "passed": True,
841
            "checks": {"tests_passed": True, "expected_files_changed": True},
842
            "notes": None,
843
        },
844
        "token_metrics": {
845
            "token_source": "char_only",
846
            "tokenizer": None,
847
            "input_tokens_reported": None,
848
            "output_tokens_reported": None,
849
            "reasoning_tokens_reported": None,
850
            "cache_read_tokens_reported": None,
851
            "cache_write_tokens_reported": None,
852
            "total_tokens_reported": None,
853
            "normalized_input_chars": 12000,
854
            "normalized_output_chars": 1200,
855
            "transcript_chars": 13200,
856
            "context_truncation_events": None,
857
            "token_budget": None,
858
            "cost_weighted_total": 4500.0,
859
            "cost_weights": "emitted=5,ingested=1",
860
        },
861
        "turn_metrics": {
862
            "turn_source": "adapter_stream",
863
            "assistant_turns_total": 6,
864
            "tool_calls_per_turn_avg": 1.33,
865
            "max_tool_calls_per_turn": 2,
866
            "cumulative_input_tokens_reported": 48000,
867
            "peak_input_tokens_reported": 11000,
868
            "turn_timeline_path": None,
869
        },
870
        "tool_call_metrics": {
871
            "tool_calls_total": 8,
872
            "tool_calls_by_kind": {"terminal": 3, "read": 3, "edit": 1, "search": 1},
873
            "terminal_commands_total": 3,
874
            "file_read_calls": 3,
875
            "file_write_calls": 1,
876
            "search_calls": 1,
877
            "edit_calls": 1,
878
            "vcs_commands_total": 2,
879
            "test_commands_total": 1,
880
            "network_calls_total": 0,
881
            "privileged_calls_total": 0,
882
            "help_calls_total": 0,
883
            "unknown_command_failures_total": None,
884
            "parallel_batches_total": None,
885
            "max_concurrency": None,
886
            "measurement_source": "adapter_stream_events",
887
        },
888
        "failed_command_retry_metrics": {
889
            "failed_tool_calls_total": 0,
890
            "failed_commands_total": 0,
891
            "failed_vcs_commands_total": 0,
892
            "failed_test_commands_total": 0,
893
            "retry_attempts_total": None,
894
            "retry_chains_total": None,
895
            "same_command_retries_total": None,
896
            "retry_after_failure_ms": None,
897
            "recoverable_failures_total": None,
898
            "unrecoverable_failures_total": None,
899
            "turns_to_recovery": None,
900
            "tokens_to_recovery": None,
901
            "failed_command_examples": [],
902
            "measurement_source": "adapter_stream_events",
903
        },
904
        "wall_clock_metrics": {
905
            "total_ms": 15000.0,
906
            "setup_ms": 800.0,
907
            "agent_active_ms": 7000.0,
908
            "tool_wait_ms": None,
909
            "vcs_ms": 900.0,
910
            "test_ms": 2100.0,
911
            "merge_ms": None,
912
            "cleanup_ms": None,
913
            "idle_ms": None,
914
            "timeout_ms": None,
915
            "measurement_source": "outer_cli_wall_clock",
916
        },
917
        "bytes_metrics": {
918
            "workspace_logical_bytes": 65536,
919
            "workspace_materialized_bytes": 32768,
920
            "bytes_read_by_agent": 18000,
921
            "bytes_written_by_agent": 140,
922
            "bytes_hydrated": None,
923
            "files_hydrated": None,
924
            "hydration_events": None,
925
            "large_file_bytes_scanned": None,
926
            "binary_bytes_touched": None,
927
            "command_output_bytes": 3000,
928
            "measurement_source": "example-adapter",
929
        },
930
        "vcs_metrics": {
931
            "commands_total": 2,
932
            "status_commands": 1,
933
            "diff_commands": 0,
934
            "snapshot_commands": 1,
935
            "branch_commands": 0,
936
            "merge_commands": 0,
937
            "cleanup_commands": 0,
938
            "commits_created": 1,
939
            "branches_created": 1,
940
            "conflicts_total": None,
941
            "conflicts_resolved": None,
942
            "dirty_files_at_end": 0,
943
            "cleanup_success": True,
944
            "measurement_source": "transcript_command_classification",
945
        },
946
        "test_metrics": {
947
            "commands_total": 1,
948
            "passed_commands": 1,
949
            "failed_commands": 0,
950
            "assertions_total": None,
951
            "assertions_failed": None,
952
            "required_checks_passed": True,
953
        },
954
        "parallel_metrics": {
955
            "tasks_total": 1,
956
            "tasks_completed": 1,
957
            "max_parallel_tasks": 1,
958
            "overlapping_files_count": None,
959
            "lost_updates_detected": None,
960
            "lock_wait_ms": None,
961
            "commit_throughput_per_s": None,
962
            "integrity_check_passed": None,
963
            "measurement_source": "single_task_run",
964
        },
965
        "artifacts": {
966
            "transcript_path": None,
967
            "command_log_path": None,
968
            "diff_path": None,
969
            "workspace_path": None,
970
            "extra_paths": [],
971
        },
972
        "notes": None,
973
    }
974
975
976
def parse_args() -> argparse.Namespace:
977
    parser = argparse.ArgumentParser(description=__doc__)
978
    parser.add_argument("--validate-jsonl", type=Path, help="Validate a JSON or JSONL result file.")
979
    parser.add_argument("--example", action="store_true", help="Print an example valid result row.")
980
    parser.add_argument("--compact", action="store_true", help="Print compact JSON.")
981
    return parser.parse_args()
982
983
984
def print_json(value: Any, compact: bool) -> None:
985
    if compact:
986
        print(json.dumps(value, sort_keys=True, separators=(",", ":")))
987
    else:
988
        print(json.dumps(value, indent=2, sort_keys=True))
989
990
991
def main() -> int:
992
    args = parse_args()
993
    schema_errors = validate_schema_shape(SCHEMA)
994
    if schema_errors:
995
        print_json({"ok": False, "schema_errors": schema_errors}, args.compact)
996
        return 1
997
998
    if args.example:
999
        row = example_row()
1000
        errors = validate_row(row)
1001
        if errors:
1002
            print_json({"ok": False, "example_errors": errors}, args.compact)
1003
            return 1
1004
        print_json(row, args.compact)
1005
        return 0
1006
1007
    if args.validate_jsonl:
1008
        result = validate_file(args.validate_jsonl)
1009
        print_json(result, args.compact)
1010
        return 0 if result["ok"] else 1
1011
1012
    print_json(SCHEMA, args.compact)
1013
    return 0
1014
1015
1016
if __name__ == "__main__":
1017
    raise SystemExit(main())