/oak
Log in
benchmarks
scripts/agent_metrics_schema.py 1002 lines · 41.1 KB · python Blame
1
#!/usr/bin/env python3
2
"""Emit and validate Oak agent workflow benchmark result row schema.
3
4
The schema is JSON-schema-like but intentionally dependency-free. Running this
5
script with no arguments prints the schema as JSON. Use --validate-jsonl to check
6
JSON or JSONL result rows against the subset of schema rules implemented here.
7
"""
8
9
from __future__ import annotations
10
11
import argparse
12
import json
13
from pathlib import Path
14
from typing import Any, Iterable, List
15
16
SCHEMA_VERSION = 2
17
18
VCS_MODES = [
19
    "git_single_checkout",
20
    "git_worktree_per_task",
21
    "oak_single_mount",
22
    "oak_mount_per_task",
23
    "oak_space_per_task",
24
]
25
26
SUBJECT_KINDS = ["git", "oak", "other"]
27
OUTCOMES = ["pass", "fail", "timeout", "error", "abandoned"]
28
TOKEN_SOURCES = ["provider_reported", "adapter_estimated", "char_only", "mixed", "unknown"]
29
TURN_SOURCES = ["adapter_stream", "estimated", "unmeasured"]
30
INSTRUCTION_LEVELS = ["zero-shot", "cheat-sheet", "full-docs", "adapter-default"]
31
32
# Schema v2 measurement contract: a field that a runner cannot actually measure
33
# MUST be null, never a fabricated zero. Dashboards must render null as
34
# "unmeasured", not as a good result. Each metric group carries an optional
35
# measurement_source describing how its values were obtained.
36
37
38
def string(description: str = "") -> dict[str, Any]:
39
    schema: dict[str, Any] = {"type": "string"}
40
    if description:
41
        schema["description"] = description
42
    return schema
43
44
45
def optional_string(description: str = "") -> dict[str, Any]:
46
    schema: dict[str, Any] = {"type": ["string", "null"]}
47
    if description:
48
        schema["description"] = description
49
    return schema
50
51
52
def non_negative_integer(description: str = "") -> dict[str, Any]:
53
    schema: dict[str, Any] = {"type": "integer", "minimum": 0}
54
    if description:
55
        schema["description"] = description
56
    return schema
57
58
59
def optional_non_negative_integer(description: str = "") -> dict[str, Any]:
60
    schema: dict[str, Any] = {"type": ["integer", "null"], "minimum": 0}
61
    if description:
62
        schema["description"] = description
63
    return schema
64
65
66
def non_negative_number(description: str = "") -> dict[str, Any]:
67
    schema: dict[str, Any] = {"type": "number", "minimum": 0}
68
    if description:
69
        schema["description"] = description
70
    return schema
71
72
73
def optional_non_negative_number(description: str = "") -> dict[str, Any]:
74
    schema: dict[str, Any] = {"type": ["number", "null"], "minimum": 0}
75
    if description:
76
        schema["description"] = description
77
    return schema
78
79
80
def boolean(description: str = "") -> dict[str, Any]:
81
    schema: dict[str, Any] = {"type": "boolean"}
82
    if description:
83
        schema["description"] = description
84
    return schema
85
86
87
def optional_boolean(description: str = "") -> dict[str, Any]:
88
    schema: dict[str, Any] = {"type": ["boolean", "null"]}
89
    if description:
90
        schema["description"] = description
91
    return schema
92
93
94
def string_array(description: str = "") -> dict[str, Any]:
95
    schema: dict[str, Any] = {"type": "array", "items": {"type": "string"}}
96
    if description:
97
        schema["description"] = description
98
    return schema
99
100
101
TOKEN_METRICS_SCHEMA: dict[str, Any] = {
102
    "type": "object",
103
    "additionalProperties": False,
104
    "required": [
105
        "token_source",
106
        "input_tokens_reported",
107
        "output_tokens_reported",
108
        "reasoning_tokens_reported",
109
        "cache_read_tokens_reported",
110
        "cache_write_tokens_reported",
111
        "total_tokens_reported",
112
        "normalized_input_chars",
113
        "normalized_output_chars",
114
        "transcript_chars",
115
        "context_truncation_events",
116
        "token_budget",
117
        "cost_weighted_total",
118
    ],
119
    "properties": {
120
        "token_source": {"type": "string", "enum": TOKEN_SOURCES},
121
        "tokenizer": optional_string("Tokenizer or accounting method, when known."),
122
        "input_tokens_reported": optional_non_negative_integer("Provider-reported input tokens."),
123
        "output_tokens_reported": optional_non_negative_integer("Provider-reported output tokens."),
124
        "reasoning_tokens_reported": optional_non_negative_integer("Provider-reported reasoning tokens."),
125
        "cache_read_tokens_reported": optional_non_negative_integer("Provider-reported cache-read tokens."),
126
        "cache_write_tokens_reported": optional_non_negative_integer("Provider-reported cache-write tokens."),
127
        "total_tokens_reported": optional_non_negative_integer("Provider-reported total tokens."),
128
        "normalized_input_chars": non_negative_integer("Prompt and admitted tool-result characters."),
129
        "normalized_output_chars": non_negative_integer("Assistant output characters."),
130
        "transcript_chars": non_negative_integer("Full normalized transcript characters."),
131
        "context_truncation_events": optional_non_negative_integer(
132
            "Times context was compacted or truncated. Null when the adapter does not expose compaction events."
133
        ),
134
        "token_budget": optional_non_negative_integer("Configured token budget, if any."),
135
        "cost_weighted_total": optional_non_negative_number(
136
            "Billing-direction weighted token cost. Agent-emitted text (commands, assistant output) is "
137
            "model OUTPUT and weighted higher than agent-ingested tool results (model INPUT)."
138
        ),
139
        "cost_weights": optional_string("Weights used for cost_weighted_total, e.g. 'emitted=5,ingested=1'."),
140
    },
141
}
142
143
TOOL_CALL_METRICS_SCHEMA: dict[str, Any] = {
144
    "type": "object",
145
    "additionalProperties": False,
146
    "required": [
147
        "tool_calls_total",
148
        "tool_calls_by_kind",
149
        "terminal_commands_total",
150
        "file_read_calls",
151
        "file_write_calls",
152
        "search_calls",
153
        "edit_calls",
154
        "vcs_commands_total",
155
        "test_commands_total",
156
        "network_calls_total",
157
        "privileged_calls_total",
158
        "help_calls_total",
159
        "unknown_command_failures_total",
160
        "parallel_batches_total",
161
        "max_concurrency",
162
    ],
163
    "properties": {
164
        "tool_calls_total": non_negative_integer(),
165
        "tool_calls_by_kind": {
166
            "type": "object",
167
            "additionalProperties": non_negative_integer("Count by adapter-defined tool kind."),
168
            "properties": {},
169
        },
170
        "terminal_commands_total": non_negative_integer(),
171
        "file_read_calls": non_negative_integer(),
172
        "file_write_calls": non_negative_integer(),
173
        "search_calls": non_negative_integer(),
174
        "edit_calls": non_negative_integer(),
175
        "vcs_commands_total": non_negative_integer(),
176
        "test_commands_total": non_negative_integer(),
177
        "network_calls_total": non_negative_integer(),
178
        "privileged_calls_total": non_negative_integer(),
179
        "help_calls_total": non_negative_integer(
180
            "Discovery calls: --help/-h flags, help subcommands, and man pages. "
181
            "High counts signal the agent does not know the tool zero-shot."
182
        ),
183
        "unknown_command_failures_total": optional_non_negative_integer(
184
            "Commands rejected as unknown subcommand/flag (hallucinated usage). Null when exit "
185
            "status or error text is not adapter-visible."
186
        ),
187
        "vcs_info_followup_calls_total": optional_non_negative_integer(
188
            "Successful read-only VCS info commands immediately re-asked in the same family "
189
            "(e.g. status, then status with more flags). Each one means the first output did "
190
            "not carry enough information to act on — the hidden cost of compact output. "
191
            "Null when per-call outcomes are not adapter-visible."
192
        ),
193
        "parallel_batches_total": optional_non_negative_integer(
194
            "Tool batches issued in parallel. Null when the adapter does not expose batching."
195
        ),
196
        "max_concurrency": optional_non_negative_integer(
197
            "Peak concurrent tool calls. Null when the adapter does not expose concurrency."
198
        ),
199
        "measurement_source": optional_string("How tool calls were observed, e.g. adapter_stream_events."),
200
    },
201
}
202
203
FAILURE_RETRY_METRICS_SCHEMA: dict[str, Any] = {
204
    "type": "object",
205
    "additionalProperties": False,
206
    "required": [
207
        "failed_tool_calls_total",
208
        "failed_commands_total",
209
        "failed_vcs_commands_total",
210
        "failed_test_commands_total",
211
        "retry_attempts_total",
212
        "retry_chains_total",
213
        "same_command_retries_total",
214
        "retry_after_failure_ms",
215
        "recoverable_failures_total",
216
        "unrecoverable_failures_total",
217
        "turns_to_recovery",
218
        "tokens_to_recovery",
219
    ],
220
    "properties": {
221
        "failed_tool_calls_total": optional_non_negative_integer(
222
            "Null when per-call success is not adapter-visible."
223
        ),
224
        "failed_commands_total": optional_non_negative_integer(),
225
        "failed_vcs_commands_total": optional_non_negative_integer(),
226
        "failed_test_commands_total": optional_non_negative_integer(),
227
        "retry_attempts_total": optional_non_negative_integer(),
228
        "retry_chains_total": optional_non_negative_integer(),
229
        "same_command_retries_total": optional_non_negative_integer(),
230
        "retry_after_failure_ms": optional_non_negative_number(),
231
        "recoverable_failures_total": optional_non_negative_integer(),
232
        "unrecoverable_failures_total": optional_non_negative_integer(),
233
        "turns_to_recovery": optional_non_negative_integer(
234
            "Assistant turns between the first failed tool call and the next successful tool call. "
235
            "Measures error-message ergonomics: a good error resolves in one turn."
236
        ),
237
        "tokens_to_recovery": optional_non_negative_integer(
238
            "Model output tokens spent in the recovery window, when per-turn usage is available."
239
        ),
240
        "failed_command_examples": string_array("Small redacted examples for debugging."),
241
        "measurement_source": optional_string("How failures/retries were observed."),
242
    },
243
}
244
245
WALL_CLOCK_METRICS_SCHEMA: dict[str, Any] = {
246
    "type": "object",
247
    "additionalProperties": False,
248
    "required": [
249
        "total_ms",
250
        "setup_ms",
251
        "agent_active_ms",
252
        "tool_wait_ms",
253
        "vcs_ms",
254
        "test_ms",
255
        "merge_ms",
256
        "cleanup_ms",
257
        "idle_ms",
258
        "timeout_ms",
259
    ],
260
    "properties": {
261
        "total_ms": non_negative_number(),
262
        "setup_ms": non_negative_number(),
263
        "agent_active_ms": non_negative_number(),
264
        "tool_wait_ms": optional_non_negative_number(
265
            "Null when the adapter does not separate tool wait from model time."
266
        ),
267
        "vcs_ms": non_negative_number(),
268
        "test_ms": non_negative_number(),
269
        "merge_ms": optional_non_negative_number("Null when the scenario has no merge phase."),
270
        "cleanup_ms": optional_non_negative_number("Null when cleanup is not separately timed."),
271
        "idle_ms": optional_non_negative_number("Null when harness queueing is not separately timed."),
272
        "timeout_ms": optional_non_negative_number(),
273
        "measurement_source": optional_string("How phases were attributed."),
274
    },
275
}
276
277
BYTES_METRICS_SCHEMA: dict[str, Any] = {
278
    "type": "object",
279
    "additionalProperties": False,
280
    "required": [
281
        "workspace_logical_bytes",
282
        "workspace_materialized_bytes",
283
        "bytes_read_by_agent",
284
        "bytes_written_by_agent",
285
        "bytes_hydrated",
286
        "files_hydrated",
287
        "hydration_events",
288
        "large_file_bytes_scanned",
289
        "binary_bytes_touched",
290
        "command_output_bytes",
291
    ],
292
    "properties": {
293
        "workspace_logical_bytes": non_negative_integer("Fixture size if fully materialized."),
294
        "workspace_materialized_bytes": non_negative_integer("Bytes materialized before timed work."),
295
        "bytes_read_by_agent": optional_non_negative_integer(
296
            "File bytes read through agent-visible read tools. Null when read-tool payloads are "
297
            "not adapter-visible — never transcript output bytes, which live in command_output_bytes. "
298
            "Hydration/read-efficiency claims must not cite a proxy here."
299
        ),
300
        "bytes_written_by_agent": non_negative_integer("File bytes written through agent-visible tools."),
301
        "bytes_hydrated": optional_non_negative_integer(
302
            "Bytes loaded lazily by VCS or filesystem. Null when no hydration probe ran."
303
        ),
304
        "files_hydrated": optional_non_negative_integer(
305
            "Files loaded lazily by VCS or filesystem. Null when no hydration probe ran."
306
        ),
307
        "hydration_events": optional_non_negative_integer(
308
            "Hydration events observed by the harness. Null when no hydration probe ran."
309
        ),
310
        "large_file_bytes_scanned": optional_non_negative_integer(
311
            "Bytes scanned in known large-file fixtures. Null when not instrumented."
312
        ),
313
        "binary_bytes_touched": optional_non_negative_integer(
314
            "Binary bytes read or written. Null when not instrumented."
315
        ),
316
        "command_output_bytes": non_negative_integer("Terminal output bytes admitted to transcript."),
317
        "measurement_source": optional_string("Adapter, filesystem probe, VCS trace, or estimate."),
318
    },
319
}
320
321
VCS_METRICS_SCHEMA: dict[str, Any] = {
322
    "type": "object",
323
    "additionalProperties": False,
324
    "required": [
325
        "commands_total",
326
        "status_commands",
327
        "diff_commands",
328
        "snapshot_commands",
329
        "branch_commands",
330
        "merge_commands",
331
        "cleanup_commands",
332
        "commits_created",
333
        "branches_created",
334
        "conflicts_total",
335
        "conflicts_resolved",
336
        "dirty_files_at_end",
337
        "cleanup_success",
338
    ],
339
    "properties": {
340
        "commands_total": optional_non_negative_integer(
341
            "Null when the agent transcript does not expose VCS command invocations."
342
        ),
343
        "status_commands": optional_non_negative_integer(),
344
        "diff_commands": optional_non_negative_integer(),
345
        "snapshot_commands": optional_non_negative_integer(),
346
        "branch_commands": optional_non_negative_integer(),
347
        "merge_commands": optional_non_negative_integer(),
348
        "cleanup_commands": optional_non_negative_integer(),
349
        "commits_created": non_negative_integer(),
350
        "branches_created": optional_non_negative_integer(),
351
        "conflicts_total": optional_non_negative_integer(
352
            "Null when conflict detection is not instrumented for the scenario."
353
        ),
354
        "conflicts_resolved": optional_non_negative_integer(),
355
        "dirty_files_at_end": optional_non_negative_integer(
356
            "Null when the final status command fails or its output cannot be interpreted."
357
        ),
358
        "cleanup_success": boolean(),
359
        "measurement_source": optional_string("How VCS command outcomes were observed."),
360
        # PATH-shim attribution fields (optional, additive). Null when the
361
        # --vcs-shim instrument was off or recorded nothing: unmeasured, not
362
        # zero (ADR-0002).
363
        "shim_sidecar_path": optional_string(
364
            "Artifact path of the shim sidecar JSONL ({command, argv, start_ms, end_ms, "
365
            "elapsed_ms, returncode, shim} per VCS call). Null when the shim was off."
366
        ),
367
        "shim_sidecar_path_unavailable_reason": optional_string(
368
            "Reason shim_sidecar_path is null."
369
        ),
370
        "shim_overhead": optional_non_negative_number(
371
            "Requested alias for shim_overhead_ms: median extra ms per call added by the PATH shim."
372
        ),
373
        "shim_overhead_ms": optional_non_negative_number(
374
            "Median extra ms per call added by the PATH shim, measured per run via a "
375
            "calibration pass. Recorded for interpretation, never subtracted from measurements."
376
        ),
377
        "shim_overhead_unavailable_reason": optional_string(
378
            "Reason shim_overhead/shim_overhead_ms is null."
379
        ),
380
        "vcs_ms": optional_non_negative_number(
381
            "Sum of shim-recorded VCS call wall time (includes shim overhead)."
382
        ),
383
        "vcs_ms_unavailable_reason": optional_string("Reason shim-recorded vcs_ms is null."),
384
        "vcs_call_count_shim": optional_non_negative_integer(
385
            "Number of VCS binary invocations recorded by the PATH shim."
386
        ),
387
        "vcs_call_count_shim_unavailable_reason": optional_string(
388
            "Reason vcs_call_count_shim is null."
389
        ),
390
        "agent_blocked_on_vcs_ms": optional_non_negative_number(
391
            "oakbench.thrash.agent_blocked_on_vcs_ms over the shim sidecar rows."
392
        ),
393
        "agent_blocked_on_vcs_ms_unavailable_reason": optional_string(
394
            "Reason agent_blocked_on_vcs_ms is null."
395
        ),
396
        "vcs_share_of_task_wall": optional_non_negative_number(
397
            "agent_blocked_on_vcs_ms divided by agent wall time."
398
        ),
399
        "vcs_share_of_task_wall_unavailable_reason": optional_string(
400
            "Reason vcs_share_of_task_wall is null."
401
        ),
402
        "thrash_events_count": optional_non_negative_integer(
403
            "Count of oakbench.thrash.thrash_events detected in the shim sidecar rows."
404
        ),
405
        "thrash_events_count_unavailable_reason": optional_string(
406
            "Reason thrash_events_count is null."
407
        ),
408
        "polling_loop_count": optional_non_negative_integer(
409
            "Count of shim-sidecar thrash events with type vcs_polling_loop."
410
        ),
411
        "polling_loop_count_unavailable_reason": optional_string(
412
            "Reason polling_loop_count is null."
413
        ),
414
    },
415
}
416
417
TEST_METRICS_SCHEMA: dict[str, Any] = {
418
    "type": "object",
419
    "additionalProperties": False,
420
    "required": [
421
        "commands_total",
422
        "passed_commands",
423
        "failed_commands",
424
        "assertions_total",
425
        "assertions_failed",
426
        "required_checks_passed",
427
    ],
428
    "properties": {
429
        "commands_total": non_negative_integer(),
430
        "passed_commands": optional_non_negative_integer(),
431
        "failed_commands": optional_non_negative_integer(),
432
        "assertions_total": optional_non_negative_integer(),
433
        "assertions_failed": optional_non_negative_integer(),
434
        "required_checks_passed": optional_boolean(),
435
    },
436
}
437
438
PARALLEL_METRICS_SCHEMA: dict[str, Any] = {
439
    "type": "object",
440
    "additionalProperties": False,
441
    "required": [
442
        "tasks_total",
443
        "tasks_completed",
444
        "max_parallel_tasks",
445
        "overlapping_files_count",
446
        "lost_updates_detected",
447
    ],
448
    "properties": {
449
        "tasks_total": non_negative_integer(),
450
        "tasks_completed": non_negative_integer(),
451
        "max_parallel_tasks": non_negative_integer(),
452
        "overlapping_files_count": optional_non_negative_integer(
453
            "Null when overlap detection did not run (e.g. single-task scenarios)."
454
        ),
455
        "lost_updates_detected": optional_non_negative_integer(
456
            "Null when no integrity check ran. Zero means the check ran and found nothing lost."
457
        ),
458
        "lock_wait_ms": optional_non_negative_number(
459
            "Time spent blocked on or retrying VCS locks under contention."
460
        ),
461
        "commit_throughput_per_s": optional_non_negative_number(
462
            "Successful snapshots per second across all parallel workers."
463
        ),
464
        "integrity_check_passed": optional_boolean(
465
            "fsck-equivalent result after the parallel run. Null when no check ran."
466
        ),
467
        "measurement_source": optional_string("How parallel behavior was observed."),
468
    },
469
}
470
471
TURN_METRICS_SCHEMA: dict[str, Any] = {
472
    "type": "object",
473
    "additionalProperties": False,
474
    "required": [
475
        "turn_source",
476
        "assistant_turns_total",
477
        "tool_calls_per_turn_avg",
478
        "max_tool_calls_per_turn",
479
        "cumulative_input_tokens_reported",
480
        "peak_input_tokens_reported",
481
    ],
482
    "properties": {
483
        "turn_source": {"type": "string", "enum": TURN_SOURCES},
484
        "assistant_turns_total": optional_non_negative_integer(
485
            "Model inference round-trips. Each turn re-pays the transcript as input tokens, so "
486
            "turns dominate agent cost; tool-call counts alone hide this."
487
        ),
488
        "tool_calls_per_turn_avg": optional_non_negative_number(),
489
        "max_tool_calls_per_turn": optional_non_negative_integer(
490
            "Peak batching within one turn. Batched calls are far cheaper than sequential turns."
491
        ),
492
        "cumulative_input_tokens_reported": optional_non_negative_integer(
493
            "Sum of provider-reported input tokens (including cache reads) across all turns. "
494
            "Captures context-residency cost: early verbose output is re-paid every later turn."
495
        ),
496
        "peak_input_tokens_reported": optional_non_negative_integer(
497
            "Largest single-turn input, a proxy for peak context size."
498
        ),
499
        "turn_timeline_path": optional_string(
500
            "Artifact path of per-turn JSONL: turn index, input/output/cache tokens, tool calls."
501
        ),
502
    },
503
}
504
505
SCHEMA: dict[str, Any] = {
506
    "$schema": "https://json-schema.org/draft/2020-12/schema",
507
    "title": "Oak agent workflow benchmark result row",
508
    "schema_version": SCHEMA_VERSION,
509
    "type": "object",
510
    "additionalProperties": False,
511
    "required": [
512
        "schema_version",
513
        "bench_id",
514
        "timestamp_utc",
515
        "scenario",
516
        "run",
517
        "subject",
518
        "subject_kind",
519
        "vcs_mode",
520
        "agent",
521
        "fixture",
522
        "outcome",
523
        "success",
524
        "success_criteria",
525
        "token_metrics",
526
        "turn_metrics",
527
        "tool_call_metrics",
528
        "failed_command_retry_metrics",
529
        "wall_clock_metrics",
530
        "bytes_metrics",
531
        "vcs_metrics",
532
        "test_metrics",
533
        "parallel_metrics",
534
    ],
535
    "properties": {
536
        "schema_version": {"type": "integer", "const": SCHEMA_VERSION},
537
        "bench_id": string("Stable identifier for a benchmark invocation."),
538
        "timestamp_utc": string("UTC ISO-8601 timestamp for row completion."),
539
        "profile": optional_string("Run profile such as smoke, standard, or large."),
540
        "host": optional_string("Host name used for run provenance."),
541
        "platform": optional_string("Platform string used for run provenance."),
542
        "machine": optional_string("Machine architecture used for run provenance."),
543
        "scenario": string("Scenario id from scenarios/agent.yaml."),
544
        "scenario_version": optional_string("Fixture or prompt version for this scenario."),
545
        "skipped": boolean("Whether this trial lacks a required measurement instrument."),
546
        "runner_id": optional_string(),
547
        "runner_class": optional_string(),
548
        "runner_profile": {"type": "object"},
549
        "cache_state": optional_string(),
550
        "cache_state_reason": optional_string(),
551
        "calibration_version": optional_string(),
552
        "calibration_unmeasured_reason": optional_string(),
553
        "workflow_integrity_evidence_class": optional_string(),
554
        "workflow_subject_binary_sha256": optional_string(),
555
        "env_isolation_version": optional_string("Version of environment isolation policy."),
556
        "oracle_version": optional_string("Version of task correctness evidence."),
557
        "workflow_integrity_source": optional_string("Actual worktree/committed-tree evidence source."),
558
        "workflow_integrity_passed": optional_boolean("Null when required correctness instrument is unavailable."),
559
        "workflow_integrity_failure_reason": optional_string("Observed incorrect task state."),
560
        "workflow_integrity_unmeasured_reason": optional_string("Why required correctness evidence is unavailable."),
561
        "skip_reason": optional_string("Explicit missing measurement capability."),
562
        "git_head_instrument_binary": optional_string("Resolved Git object-reading instrument."),
563
        "git_head_instrument_sha256": optional_string("Git object reader SHA256."),
564
        "git_head_instrument_version": optional_string("Git object reader reported version."),
565
        "oak_export_instrument_binary": optional_string("Subject-produced export binary."),
566
        "oak_export_instrument_sha256": optional_string("Subject-produced export binary SHA256."),
567
        "oak_export_head": optional_string("Oak HEAD checked unchanged across export."),
568
        "operation": optional_string("Optional report grouping operation, e.g. agent.workflow."),
569
        "benchmark_track": optional_string("Command semantics track such as agent-default or core-equivalent."),
570
        "command_semantics_version": optional_string("Version of the command semantics contract."),
571
        "elapsed_ms": optional_non_negative_number("Compatibility mirror of wall_clock_metrics.total_ms."),
572
        "returncode": optional_non_negative_integer("Compatibility mirror: 0 for success, non-zero for failure."),
573
        "raw_output_bytes": optional_non_negative_integer("Compatibility mirror of captured agent stdout+stderr bytes."),
574
        "stdout_bytes": optional_non_negative_integer("Captured agent stdout bytes."),
575
        "stderr_bytes": optional_non_negative_integer("Captured agent stderr bytes."),
576
        "output_truncated": optional_boolean("Whether captured output was truncated before artifact storage."),
577
        "task_id": optional_string("Subtask id for parallel scenarios, if this row is per subtask."),
578
        "task_prompt_id": optional_string("Stable prompt/template id."),
579
        "run": non_negative_integer("Zero-based repetition index."),
580
        "subject": string("Benchmark subject such as git, oak_installed, or oak_local."),
581
        "subject_kind": {"type": "string", "enum": SUBJECT_KINDS},
582
        "subject_label": optional_string("Human label for reports."),
583
        "vcs_mode": {"type": "string", "enum": VCS_MODES},
584
        "agent": {
585
            "type": "object",
586
            "additionalProperties": False,
587
            "required": ["name", "adapter"],
588
            "properties": {
589
                "name": string("Agent or harness adapter name."),
590
                "adapter": string("Runner integration name."),
591
                "provider": optional_string("Provider name, if applicable."),
592
                "model": optional_string("Model id, if applicable."),
593
                "version": optional_string("Agent binary, prompt, or adapter version."),
594
                "environment": optional_string("Agent environment mode, such as minimal or local-default."),
595
                "instruction_level": {
596
                    "type": ["string", "null"],
597
                    "enum": INSTRUCTION_LEVELS + [None],
598
                    "description": (
599
                        "How much VCS-specific instruction the prompt carried. zero-shot measures the "
600
                        "model-familiarity tax: Git is in pretraining data, a new VCS is not."
601
                    ),
602
                },
603
                "temperature": {"type": ["number", "null"]},
604
                "max_output_tokens": optional_non_negative_integer(),
605
            },
606
        },
607
        "fixture": {
608
            "type": "object",
609
            "additionalProperties": False,
610
            "required": ["id", "repo_shape", "fixture_version"],
611
            "properties": {
612
                "id": string("Fixture id used by the runner."),
613
                "repo_shape": string("Human-readable fixture shape."),
614
                "fixture_version": string("Stable fixture content version or hash."),
615
                "logical_file_count": optional_non_negative_integer(),
616
                "logical_bytes": optional_non_negative_integer(),
617
            },
618
        },
619
        "source": {
620
            "type": "object",
621
            "additionalProperties": True,
622
            "properties": {
623
                "oak_hash": optional_string(),
624
                "oak_status": optional_string(),
625
                "benchmark_repo_hash": optional_string(),
626
            },
627
        },
628
        "outcome": {"type": "string", "enum": OUTCOMES},
629
        "success": boolean("True only when deterministic scenario criteria passed."),
630
        "failure_reason": optional_string("Short stable failure reason for dashboards."),
631
        "success_criteria": {
632
            "type": "object",
633
            "additionalProperties": False,
634
            "required": ["passed", "checks"],
635
            "properties": {
636
                "passed": boolean(),
637
                "checks": {
638
                    "type": "object",
639
                    "additionalProperties": optional_boolean("Individual deterministic check result; null if unmeasured."),
640
                    "properties": {},
641
                },
642
                "notes": optional_string(),
643
            },
644
        },
645
        "token_metrics": TOKEN_METRICS_SCHEMA,
646
        "turn_metrics": TURN_METRICS_SCHEMA,
647
        "tool_call_metrics": TOOL_CALL_METRICS_SCHEMA,
648
        "failed_command_retry_metrics": FAILURE_RETRY_METRICS_SCHEMA,
649
        "wall_clock_metrics": WALL_CLOCK_METRICS_SCHEMA,
650
        "bytes_metrics": BYTES_METRICS_SCHEMA,
651
        "vcs_metrics": VCS_METRICS_SCHEMA,
652
        "test_metrics": TEST_METRICS_SCHEMA,
653
        "parallel_metrics": PARALLEL_METRICS_SCHEMA,
654
        "artifacts": {
655
            "type": "object",
656
            "additionalProperties": False,
657
            "properties": {
658
                "transcript_path": optional_string(),
659
                "command_log_path": optional_string(),
660
                "diff_path": optional_string(),
661
                "workspace_path": optional_string(),
662
                "extra_paths": string_array(),
663
            },
664
        },
665
        "notes": optional_string(),
666
    },
667
}
668
669
670
def type_matches(value: Any, expected_type: Any) -> bool:
671
    if isinstance(expected_type, list):
672
        return any(type_matches(value, item) for item in expected_type)
673
    if expected_type == "null":
674
        return value is None
675
    if expected_type == "boolean":
676
        return isinstance(value, bool)
677
    if expected_type == "integer":
678
        return isinstance(value, int) and not isinstance(value, bool)
679
    if expected_type == "number":
680
        return (isinstance(value, int) or isinstance(value, float)) and not isinstance(value, bool)
681
    if expected_type == "string":
682
        return isinstance(value, str)
683
    if expected_type == "object":
684
        return isinstance(value, dict)
685
    if expected_type == "array":
686
        return isinstance(value, list)
687
    return True
688
689
690
def display_path(path: str, key: str) -> str:
691
    if path == "$":
692
        return "$" + "." + key
693
    return path + "." + key
694
695
696
def validate_value(value: Any, schema: dict[str, Any], path: str) -> List[str]:
697
    errors: List[str] = []
698
    expected_type = schema.get("type")
699
    if expected_type is not None and not type_matches(value, expected_type):
700
        errors.append(f"{path}: expected {expected_type}, got {type(value).__name__}")
701
        return errors
702
703
    if "const" in schema and value != schema["const"]:
704
        errors.append(f"{path}: expected constant {schema['const']!r}, got {value!r}")
705
    if "enum" in schema and value not in schema["enum"]:
706
        errors.append(f"{path}: expected one of {schema['enum']!r}, got {value!r}")
707
    if "minimum" in schema and isinstance(value, (int, float)) and not isinstance(value, bool):
708
        if value < schema["minimum"]:
709
            errors.append(f"{path}: expected >= {schema['minimum']}, got {value!r}")
710
711
    if isinstance(value, dict):
712
        required = schema.get("required", [])
713
        for key in required:
714
            if key not in value:
715
                errors.append(f"{path}: missing required property {key!r}")
716
        properties = schema.get("properties", {})
717
        additional = schema.get("additionalProperties", True)
718
        for key, item in value.items():
719
            child_path = display_path(path, key)
720
            if key in properties:
721
                errors.extend(validate_value(item, properties[key], child_path))
722
            elif additional is False:
723
                errors.append(f"{child_path}: unexpected property")
724
            elif isinstance(additional, dict):
725
                errors.extend(validate_value(item, additional, child_path))
726
    elif isinstance(value, list) and "items" in schema:
727
        item_schema = schema["items"]
728
        for index, item in enumerate(value):
729
            errors.extend(validate_value(item, item_schema, f"{path}[{index}]"))
730
731
    return errors
732
733
734
def validate_schema_shape(schema: dict[str, Any]) -> List[str]:
735
    errors: List[str] = []
736
    for key in ["title", "type", "required", "properties"]:
737
        if key not in schema:
738
            errors.append(f"schema missing {key!r}")
739
    if schema.get("type") != "object":
740
        errors.append("schema root type must be object")
741
    properties = schema.get("properties", {})
742
    for required_key in schema.get("required", []):
743
        if required_key not in properties:
744
            errors.append(f"schema requires {required_key!r} but has no property definition")
745
    return errors
746
747
748
def validate_row(row: Any) -> List[str]:
749
    if not isinstance(row, dict):
750
        return [f"$: expected object row, got {type(row).__name__}"]
751
    return validate_value(row, SCHEMA, "$")
752
753
754
def iter_rows(path: Path) -> Iterable[Any]:
755
    text = path.read_text(encoding="utf-8")
756
    stripped = text.strip()
757
    if not stripped:
758
        return []
759
    if path.suffix == ".json" or stripped[0] == "[":
760
        data = json.loads(stripped)
761
        if isinstance(data, list):
762
            return data
763
        return [data]
764
    rows: List[Any] = []
765
    for line_number, line in enumerate(text.splitlines(), 1):
766
        if not line.strip():
767
            continue
768
        try:
769
            rows.append(json.loads(line))
770
        except json.JSONDecodeError as exc:
771
            rows.append({"__json_error__": f"line {line_number}: {exc}"})
772
    return rows
773
774
775
def validate_file(path: Path) -> dict[str, Any]:
776
    errors: List[str] = []
777
    rows = list(iter_rows(path))
778
    for index, row in enumerate(rows, 1):
779
        if isinstance(row, dict) and "__json_error__" in row:
780
            errors.append(row["__json_error__"])
781
            continue
782
        for error in validate_row(row):
783
            errors.append(f"row {index}: {error}")
784
    return {"ok": not errors, "rows": len(rows), "errors": errors}
785
786
787
def example_row() -> dict[str, Any]:
788
    return {
789
        "schema_version": SCHEMA_VERSION,
790
        "bench_id": "20260609T000000Z",
791
        "timestamp_utc": "2026-06-09T00:00:00Z",
792
        "profile": "smoke",
793
        "scenario": "one_line_bug_fix",
794
        "scenario_version": "v1",
795
        "task_id": None,
796
        "task_prompt_id": "agent-one-line-bug-fix-v1",
797
        "run": 0,
798
        "subject": "oak_local",
799
        "subject_kind": "oak",
800
        "subject_label": "Oak local",
801
        "vcs_mode": "oak_space_per_task",
802
        "agent": {
803
            "name": "example-agent",
804
            "adapter": "example-adapter",
805
            "provider": None,
806
            "model": None,
807
            "version": None,
808
            "environment": "minimal",
809
            "instruction_level": "cheat-sheet",
810
            "temperature": None,
811
            "max_output_tokens": None,
812
        },
813
        "fixture": {
814
            "id": "agent_one_line_bug_fix",
815
            "repo_shape": "small text application",
816
            "fixture_version": "v1",
817
            "logical_file_count": 42,
818
            "logical_bytes": 65536,
819
        },
820
        "source": {"oak_hash": None, "oak_status": None, "benchmark_repo_hash": None},
821
        "outcome": "pass",
822
        "success": True,
823
        "failure_reason": None,
824
        "success_criteria": {
825
            "passed": True,
826
            "checks": {"tests_passed": True, "expected_files_changed": True},
827
            "notes": None,
828
        },
829
        "token_metrics": {
830
            "token_source": "char_only",
831
            "tokenizer": None,
832
            "input_tokens_reported": None,
833
            "output_tokens_reported": None,
834
            "reasoning_tokens_reported": None,
835
            "cache_read_tokens_reported": None,
836
            "cache_write_tokens_reported": None,
837
            "total_tokens_reported": None,
838
            "normalized_input_chars": 12000,
839
            "normalized_output_chars": 1200,
840
            "transcript_chars": 13200,
841
            "context_truncation_events": None,
842
            "token_budget": None,
843
            "cost_weighted_total": 4500.0,
844
            "cost_weights": "emitted=5,ingested=1",
845
        },
846
        "turn_metrics": {
847
            "turn_source": "adapter_stream",
848
            "assistant_turns_total": 6,
849
            "tool_calls_per_turn_avg": 1.33,
850
            "max_tool_calls_per_turn": 2,
851
            "cumulative_input_tokens_reported": 48000,
852
            "peak_input_tokens_reported": 11000,
853
            "turn_timeline_path": None,
854
        },
855
        "tool_call_metrics": {
856
            "tool_calls_total": 8,
857
            "tool_calls_by_kind": {"terminal": 3, "read": 3, "edit": 1, "search": 1},
858
            "terminal_commands_total": 3,
859
            "file_read_calls": 3,
860
            "file_write_calls": 1,
861
            "search_calls": 1,
862
            "edit_calls": 1,
863
            "vcs_commands_total": 2,
864
            "test_commands_total": 1,
865
            "network_calls_total": 0,
866
            "privileged_calls_total": 0,
867
            "help_calls_total": 0,
868
            "unknown_command_failures_total": None,
869
            "parallel_batches_total": None,
870
            "max_concurrency": None,
871
            "measurement_source": "adapter_stream_events",
872
        },
873
        "failed_command_retry_metrics": {
874
            "failed_tool_calls_total": 0,
875
            "failed_commands_total": 0,
876
            "failed_vcs_commands_total": 0,
877
            "failed_test_commands_total": 0,
878
            "retry_attempts_total": None,
879
            "retry_chains_total": None,
880
            "same_command_retries_total": None,
881
            "retry_after_failure_ms": None,
882
            "recoverable_failures_total": None,
883
            "unrecoverable_failures_total": None,
884
            "turns_to_recovery": None,
885
            "tokens_to_recovery": None,
886
            "failed_command_examples": [],
887
            "measurement_source": "adapter_stream_events",
888
        },
889
        "wall_clock_metrics": {
890
            "total_ms": 15000.0,
891
            "setup_ms": 800.0,
892
            "agent_active_ms": 7000.0,
893
            "tool_wait_ms": None,
894
            "vcs_ms": 900.0,
895
            "test_ms": 2100.0,
896
            "merge_ms": None,
897
            "cleanup_ms": None,
898
            "idle_ms": None,
899
            "timeout_ms": None,
900
            "measurement_source": "outer_cli_wall_clock",
901
        },
902
        "bytes_metrics": {
903
            "workspace_logical_bytes": 65536,
904
            "workspace_materialized_bytes": 32768,
905
            "bytes_read_by_agent": 18000,
906
            "bytes_written_by_agent": 140,
907
            "bytes_hydrated": None,
908
            "files_hydrated": None,
909
            "hydration_events": None,
910
            "large_file_bytes_scanned": None,
911
            "binary_bytes_touched": None,
912
            "command_output_bytes": 3000,
913
            "measurement_source": "example-adapter",
914
        },
915
        "vcs_metrics": {
916
            "commands_total": 2,
917
            "status_commands": 1,
918
            "diff_commands": 0,
919
            "snapshot_commands": 1,
920
            "branch_commands": 0,
921
            "merge_commands": 0,
922
            "cleanup_commands": 0,
923
            "commits_created": 1,
924
            "branches_created": 1,
925
            "conflicts_total": None,
926
            "conflicts_resolved": None,
927
            "dirty_files_at_end": 0,
928
            "cleanup_success": True,
929
            "measurement_source": "transcript_command_classification",
930
        },
931
        "test_metrics": {
932
            "commands_total": 1,
933
            "passed_commands": 1,
934
            "failed_commands": 0,
935
            "assertions_total": None,
936
            "assertions_failed": None,
937
            "required_checks_passed": True,
938
        },
939
        "parallel_metrics": {
940
            "tasks_total": 1,
941
            "tasks_completed": 1,
942
            "max_parallel_tasks": 1,
943
            "overlapping_files_count": None,
944
            "lost_updates_detected": None,
945
            "lock_wait_ms": None,
946
            "commit_throughput_per_s": None,
947
            "integrity_check_passed": None,
948
            "measurement_source": "single_task_run",
949
        },
950
        "artifacts": {
951
            "transcript_path": None,
952
            "command_log_path": None,
953
            "diff_path": None,
954
            "workspace_path": None,
955
            "extra_paths": [],
956
        },
957
        "notes": None,
958
    }
959
960
961
def parse_args() -> argparse.Namespace:
962
    parser = argparse.ArgumentParser(description=__doc__)
963
    parser.add_argument("--validate-jsonl", type=Path, help="Validate a JSON or JSONL result file.")
964
    parser.add_argument("--example", action="store_true", help="Print an example valid result row.")
965
    parser.add_argument("--compact", action="store_true", help="Print compact JSON.")
966
    return parser.parse_args()
967
968
969
def print_json(value: Any, compact: bool) -> None:
970
    if compact:
971
        print(json.dumps(value, sort_keys=True, separators=(",", ":")))
972
    else:
973
        print(json.dumps(value, indent=2, sort_keys=True))
974
975
976
def main() -> int:
977
    args = parse_args()
978
    schema_errors = validate_schema_shape(SCHEMA)
979
    if schema_errors:
980
        print_json({"ok": False, "schema_errors": schema_errors}, args.compact)
981
        return 1
982
983
    if args.example:
984
        row = example_row()
985
        errors = validate_row(row)
986
        if errors:
987
            print_json({"ok": False, "example_errors": errors}, args.compact)
988
            return 1
989
        print_json(row, args.compact)
990
        return 0
991
992
    if args.validate_jsonl:
993
        result = validate_file(args.validate_jsonl)
994
        print_json(result, args.compact)
995
        return 0 if result["ok"] else 1
996
997
    print_json(SCHEMA, args.compact)
998
    return 0
999
1000
1001
if __name__ == "__main__":
1002
    raise SystemExit(main())