Log in
benchmarksPublic

Viewing files at commit 538751fb07e6Commit detailsView latest

Browse files
scripts/agent_metrics_schema.py 1015 lines · 42.0 KB · python
1
#!/usr/bin/env python3
2
"""Emit and validate Oak agent workflow benchmark result row schema.
3
4
The schema is JSON-schema-like but intentionally dependency-free. Running this
5
script with no arguments prints the schema as JSON. Use --validate-jsonl to check
6
JSON or JSONL result rows against the subset of schema rules implemented here.
7
"""
8
9
from __future__ import annotations
10
11
import argparse
12
import json
13
from pathlib import Path
14
from typing import Any, Iterable, List
15
16
SCHEMA_VERSION = 2
17
18
VCS_MODES = [
19
    "git_single_checkout",
20
    "git_worktree_per_task",
21
    "oak_single_mount",
22
    "oak_mount_per_task",
23
    "oak_space_per_task",
24
]
25
26
SUBJECT_KINDS = ["git", "oak", "other"]
27
OUTCOMES = ["pass", "fail", "timeout", "error", "abandoned"]
28
TOKEN_SOURCES = ["provider_reported", "adapter_estimated", "char_only", "mixed", "unknown"]
29
TURN_SOURCES = ["adapter_stream", "estimated", "unmeasured"]
30
INSTRUCTION_LEVELS = ["zero-shot", "cheat-sheet", "full-docs", "adapter-default"]
31
32
# Schema v2 measurement contract: a field that a runner cannot actually measure
33
# MUST be null, never a fabricated zero. Dashboards must render null as
34
# "unmeasured", not as a good result. Each metric group carries an optional
35
# measurement_source describing how its values were obtained.
36
37
38
def string(description: str = "") -> dict[str, Any]:
39
    schema: dict[str, Any] = {"type": "string"}
40
    if description:
41
        schema["description"] = description
42
    return schema
43
44
45
def optional_string(description: str = "") -> dict[str, Any]:
46
    schema: dict[str, Any] = {"type": ["string", "null"]}
47
    if description:
48
        schema["description"] = description
49
    return schema
50
51
52
def non_negative_integer(description: str = "") -> dict[str, Any]:
53
    schema: dict[str, Any] = {"type": "integer", "minimum": 0}
54
    if description:
55
        schema["description"] = description
56
    return schema
57
58
59
def optional_non_negative_integer(description: str = "") -> dict[str, Any]:
60
    schema: dict[str, Any] = {"type": ["integer", "null"], "minimum": 0}
61
    if description:
62
        schema["description"] = description
63
    return schema
64
65
66
def non_negative_number(description: str = "") -> dict[str, Any]:
67
    schema: dict[str, Any] = {"type": "number", "minimum": 0}
68
    if description:
69
        schema["description"] = description
70
    return schema
71
72
73
def optional_non_negative_number(description: str = "") -> dict[str, Any]:
74
    schema: dict[str, Any] = {"type": ["number", "null"], "minimum": 0}
75
    if description:
76
        schema["description"] = description
77
    return schema
78
79
80
def boolean(description: str = "") -> dict[str, Any]:
81
    schema: dict[str, Any] = {"type": "boolean"}
82
    if description:
83
        schema["description"] = description
84
    return schema
85
86
87
def optional_boolean(description: str = "") -> dict[str, Any]:
88
    schema: dict[str, Any] = {"type": ["boolean", "null"]}
89
    if description:
90
        schema["description"] = description
91
    return schema
92
93
94
def string_array(description: str = "") -> dict[str, Any]:
95
    schema: dict[str, Any] = {"type": "array", "items": {"type": "string"}}
96
    if description:
97
        schema["description"] = description
98
    return schema
99
100
101
TOKEN_METRICS_SCHEMA: dict[str, Any] = {
102
    "type": "object",
103
    "additionalProperties": False,
104
    "required": [
105
        "token_source",
106
        "input_tokens_reported",
107
        "output_tokens_reported",
108
        "reasoning_tokens_reported",
109
        "cache_read_tokens_reported",
110
        "cache_write_tokens_reported",
111
        "total_tokens_reported",
112
        "normalized_input_chars",
113
        "normalized_output_chars",
114
        "transcript_chars",
115
        "context_truncation_events",
116
        "token_budget",
117
        "cost_weighted_total",
118
    ],
119
    "properties": {
120
        "token_source": {"type": "string", "enum": TOKEN_SOURCES},
121
        "tokenizer": optional_string("Tokenizer or accounting method, when known."),
122
        "input_tokens_reported": optional_non_negative_integer("Provider-reported input tokens."),
123
        "output_tokens_reported": optional_non_negative_integer("Provider-reported output tokens."),
124
        "reasoning_tokens_reported": optional_non_negative_integer("Provider-reported reasoning tokens."),
125
        "cache_read_tokens_reported": optional_non_negative_integer("Provider-reported cache-read tokens."),
126
        "cache_write_tokens_reported": optional_non_negative_integer("Provider-reported cache-write tokens."),
127
        "total_tokens_reported": optional_non_negative_integer("Provider-reported total tokens."),
128
        "normalized_input_chars": non_negative_integer("Prompt and admitted tool-result characters."),
129
        "normalized_output_chars": optional_non_negative_integer("Assistant output characters."),
130
        "transcript_chars": optional_non_negative_integer("Full normalized transcript characters."),
131
        "context_truncation_events": optional_non_negative_integer(
132
            "Times context was compacted or truncated. Null when the adapter does not expose compaction events."
133
        ),
134
        "token_budget": optional_non_negative_integer("Configured token budget, if any."),
135
        "cost_weighted_total": optional_non_negative_number(
136
            "Billing-direction weighted token cost. Agent-emitted text (commands, assistant output) is "
137
            "model OUTPUT and weighted higher than agent-ingested tool results (model INPUT)."
138
        ),
139
        "cost_weights": optional_string("Weights used for cost_weighted_total, e.g. 'emitted=5,ingested=1'."),
140
    },
141
}
142
143
TOOL_CALL_METRICS_SCHEMA: dict[str, Any] = {
144
    "type": "object",
145
    "additionalProperties": False,
146
    "required": [
147
        "tool_calls_total",
148
        "tool_calls_by_kind",
149
        "terminal_commands_total",
150
        "file_read_calls",
151
        "file_write_calls",
152
        "search_calls",
153
        "edit_calls",
154
        "vcs_commands_total",
155
        "test_commands_total",
156
        "network_calls_total",
157
        "privileged_calls_total",
158
        "help_calls_total",
159
        "unknown_command_failures_total",
160
        "parallel_batches_total",
161
        "max_concurrency",
162
    ],
163
    "properties": {
164
        "tool_calls_total": optional_non_negative_integer(),
165
        "tool_calls_by_kind": {
166
            "type": ["object", "null"],
167
            "additionalProperties": optional_non_negative_integer("Count by adapter-defined tool kind."),
168
            "properties": {},
169
        },
170
        "terminal_commands_total": optional_non_negative_integer(),
171
        "file_read_calls": optional_non_negative_integer(),
172
        "file_write_calls": optional_non_negative_integer(),
173
        "search_calls": optional_non_negative_integer(),
174
        "edit_calls": optional_non_negative_integer(),
175
        "vcs_commands_total": optional_non_negative_integer(),
176
        "test_commands_total": optional_non_negative_integer(),
177
        "network_calls_total": optional_non_negative_integer(),
178
        "privileged_calls_total": optional_non_negative_integer(),
179
        "help_calls_total": optional_non_negative_integer(
180
            "Discovery calls: --help/-h flags, help subcommands, and man pages. "
181
            "High counts signal the agent does not know the tool zero-shot."
182
        ),
183
        "unknown_command_failures_total": optional_non_negative_integer(
184
            "Commands rejected as unknown subcommand/flag (hallucinated usage). Null when exit "
185
            "status or error text is not adapter-visible."
186
        ),
187
        "vcs_info_followup_calls_total": optional_non_negative_integer(
188
            "Successful read-only VCS info commands immediately re-asked in the same family "
189
            "(e.g. status, then status with more flags). Each one means the first output did "
190
            "not carry enough information to act on — the hidden cost of compact output. "
191
            "Null when per-call outcomes are not adapter-visible."
192
        ),
193
        "parallel_batches_total": optional_non_negative_integer(
194
            "Tool batches issued in parallel. Null when the adapter does not expose batching."
195
        ),
196
        "max_concurrency": optional_non_negative_integer(
197
            "Peak concurrent tool calls. Null when the adapter does not expose concurrency."
198
        ),
199
        "measurement_source": optional_string("How tool calls were observed, e.g. adapter_stream_events."),
200
    },
201
}
202
203
FAILURE_RETRY_METRICS_SCHEMA: dict[str, Any] = {
204
    "type": "object",
205
    "additionalProperties": False,
206
    "required": [
207
        "failed_tool_calls_total",
208
        "failed_commands_total",
209
        "failed_vcs_commands_total",
210
        "failed_test_commands_total",
211
        "retry_attempts_total",
212
        "retry_chains_total",
213
        "same_command_retries_total",
214
        "retry_after_failure_ms",
215
        "recoverable_failures_total",
216
        "unrecoverable_failures_total",
217
        "turns_to_recovery",
218
        "tokens_to_recovery",
219
    ],
220
    "properties": {
221
        "failed_tool_calls_total": optional_non_negative_integer(
222
            "Null when per-call success is not adapter-visible."
223
        ),
224
        "failed_commands_total": optional_non_negative_integer(),
225
        "failed_vcs_commands_total": optional_non_negative_integer(),
226
        "failed_test_commands_total": optional_non_negative_integer(),
227
        "retry_attempts_total": optional_non_negative_integer(),
228
        "retry_chains_total": optional_non_negative_integer(),
229
        "same_command_retries_total": optional_non_negative_integer(),
230
        "retry_after_failure_ms": optional_non_negative_number(),
231
        "recoverable_failures_total": optional_non_negative_integer(),
232
        "unrecoverable_failures_total": optional_non_negative_integer(),
233
        "turns_to_recovery": optional_non_negative_integer(
234
            "Assistant turns between the first failed tool call and the next successful tool call. "
235
            "Measures error-message ergonomics: a good error resolves in one turn."
236
        ),
237
        "tokens_to_recovery": optional_non_negative_integer(
238
            "Model output tokens spent in the recovery window, when per-turn usage is available."
239
        ),
240
        "failed_command_examples": string_array("Small redacted examples for debugging."),
241
        "measurement_source": optional_string("How failures/retries were observed."),
242
    },
243
}
244
245
WALL_CLOCK_METRICS_SCHEMA: dict[str, Any] = {
246
    "type": "object",
247
    "additionalProperties": False,
248
    "required": [
249
        "total_ms",
250
        "setup_ms",
251
        "agent_active_ms",
252
        "tool_wait_ms",
253
        "vcs_ms",
254
        "test_ms",
255
        "merge_ms",
256
        "cleanup_ms",
257
        "idle_ms",
258
        "timeout_ms",
259
    ],
260
    "properties": {
261
        "total_ms": non_negative_number(),
262
        "setup_ms": non_negative_number(),
263
        "agent_active_ms": optional_non_negative_number(),
264
        "tool_wait_ms": optional_non_negative_number(
265
            "Null when the adapter does not separate tool wait from model time."
266
        ),
267
        "vcs_ms": non_negative_number(),
268
        "test_ms": non_negative_number(),
269
        "merge_ms": optional_non_negative_number("Null when the scenario has no merge phase."),
270
        "cleanup_ms": optional_non_negative_number("Null when cleanup is not separately timed."),
271
        "idle_ms": optional_non_negative_number("Null when harness queueing is not separately timed."),
272
        "timeout_ms": optional_non_negative_number(),
273
        "measurement_source": optional_string("How phases were attributed."),
274
    },
275
}
276
277
BYTES_METRICS_SCHEMA: dict[str, Any] = {
278
    "type": "object",
279
    "additionalProperties": False,
280
    "required": [
281
        "workspace_logical_bytes",
282
        "workspace_materialized_bytes",
283
        "bytes_read_by_agent",
284
        "bytes_written_by_agent",
285
        "bytes_hydrated",
286
        "files_hydrated",
287
        "hydration_events",
288
        "large_file_bytes_scanned",
289
        "binary_bytes_touched",
290
        "command_output_bytes",
291
    ],
292
    "properties": {
293
        "workspace_logical_bytes": non_negative_integer("Fixture size if fully materialized."),
294
        "workspace_materialized_bytes": non_negative_integer("Bytes materialized before timed work."),
295
        "bytes_read_by_agent": optional_non_negative_integer(
296
            "File bytes read through agent-visible read tools. Null when read-tool payloads are "
297
            "not adapter-visible — never transcript output bytes, which live in command_output_bytes. "
298
            "Hydration/read-efficiency claims must not cite a proxy here."
299
        ),
300
        "bytes_written_by_agent": non_negative_integer("File bytes written through agent-visible tools."),
301
        "bytes_hydrated": optional_non_negative_integer(
302
            "Bytes loaded lazily by VCS or filesystem. Null when no hydration probe ran."
303
        ),
304
        "files_hydrated": optional_non_negative_integer(
305
            "Files loaded lazily by VCS or filesystem. Null when no hydration probe ran."
306
        ),
307
        "hydration_events": optional_non_negative_integer(
308
            "Hydration events observed by the harness. Null when no hydration probe ran."
309
        ),
310
        "large_file_bytes_scanned": optional_non_negative_integer(
311
            "Bytes scanned in known large-file fixtures. Null when not instrumented."
312
        ),
313
        "binary_bytes_touched": optional_non_negative_integer(
314
            "Binary bytes read or written. Null when not instrumented."
315
        ),
316
        "command_output_bytes": non_negative_integer("Terminal output bytes admitted to transcript."),
317
        "measurement_source": optional_string("Adapter, filesystem probe, VCS trace, or estimate."),
318
    },
319
}
320
321
VCS_METRICS_SCHEMA: dict[str, Any] = {
322
    "type": "object",
323
    "additionalProperties": False,
324
    "required": [
325
        "commands_total",
326
        "status_commands",
327
        "diff_commands",
328
        "snapshot_commands",
329
        "branch_commands",
330
        "merge_commands",
331
        "cleanup_commands",
332
        "commits_created",
333
        "branches_created",
334
        "conflicts_total",
335
        "conflicts_resolved",
336
        "dirty_files_at_end",
337
        "cleanup_success",
338
    ],
339
    "properties": {
340
        "commands_total": optional_non_negative_integer(
341
            "Null when the agent transcript does not expose VCS command invocations."
342
        ),
343
        "status_commands": optional_non_negative_integer(),
344
        "diff_commands": optional_non_negative_integer(),
345
        "snapshot_commands": optional_non_negative_integer(),
346
        "branch_commands": optional_non_negative_integer(),
347
        "merge_commands": optional_non_negative_integer(),
348
        "cleanup_commands": optional_non_negative_integer(),
349
        "commits_created": non_negative_integer(),
350
        "branches_created": optional_non_negative_integer(),
351
        "conflicts_total": optional_non_negative_integer(
352
            "Null when conflict detection is not instrumented for the scenario."
353
        ),
354
        "conflicts_resolved": optional_non_negative_integer(),
355
        "dirty_files_at_end": optional_non_negative_integer(
356
            "Null when the final status command fails or its output cannot be interpreted."
357
        ),
358
        "cleanup_success": boolean(),
359
        "measurement_source": optional_string("How VCS command outcomes were observed."),
360
        # PATH-shim attribution fields (optional, additive). Null when the
361
        # --vcs-shim instrument was off or recorded nothing: unmeasured, not
362
        # zero (ADR-0002).
363
        "shim_sidecar_path": optional_string(
364
            "Artifact path of the shim sidecar JSONL ({command, argv, start_ms, end_ms, "
365
            "elapsed_ms, returncode, shim} per VCS call). Null when the shim was off."
366
        ),
367
        "shim_sidecar_path_unavailable_reason": optional_string(
368
            "Reason shim_sidecar_path is null."
369
        ),
370
        "shim_overhead": optional_non_negative_number(
371
            "Requested alias for shim_overhead_ms: median extra ms per call added by the PATH shim."
372
        ),
373
        "shim_overhead_ms": optional_non_negative_number(
374
            "Median extra ms per call added by the PATH shim, measured per run via a "
375
            "calibration pass. Recorded for interpretation, never subtracted from measurements."
376
        ),
377
        "shim_overhead_unavailable_reason": optional_string(
378
            "Reason shim_overhead/shim_overhead_ms is null."
379
        ),
380
        "vcs_ms": optional_non_negative_number(
381
            "Sum of shim-recorded VCS call wall time (includes shim overhead)."
382
        ),
383
        "vcs_ms_unavailable_reason": optional_string("Reason shim-recorded vcs_ms is null."),
384
        "vcs_call_count_shim": optional_non_negative_integer(
385
            "Number of VCS binary invocations recorded by the PATH shim."
386
        ),
387
        "vcs_call_count_shim_unavailable_reason": optional_string(
388
            "Reason vcs_call_count_shim is null."
389
        ),
390
        "agent_blocked_on_vcs_ms": optional_non_negative_number(
391
            "oakbench.thrash.agent_blocked_on_vcs_ms over the shim sidecar rows."
392
        ),
393
        "agent_blocked_on_vcs_ms_unavailable_reason": optional_string(
394
            "Reason agent_blocked_on_vcs_ms is null."
395
        ),
396
        "vcs_share_of_task_wall": optional_non_negative_number(
397
            "agent_blocked_on_vcs_ms divided by agent wall time."
398
        ),
399
        "vcs_share_of_task_wall_unavailable_reason": optional_string(
400
            "Reason vcs_share_of_task_wall is null."
401
        ),
402
        "thrash_events_count": optional_non_negative_integer(
403
            "Count of oakbench.thrash.thrash_events detected in the shim sidecar rows."
404
        ),
405
        "thrash_events_count_unavailable_reason": optional_string(
406
            "Reason thrash_events_count is null."
407
        ),
408
        "polling_loop_count": optional_non_negative_integer(
409
            "Count of shim-sidecar thrash events with type vcs_polling_loop."
410
        ),
411
        "polling_loop_count_unavailable_reason": optional_string(
412
            "Reason polling_loop_count is null."
413
        ),
414
    },
415
}
416
417
TEST_METRICS_SCHEMA: dict[str, Any] = {
418
    "type": "object",
419
    "additionalProperties": False,
420
    "required": [
421
        "commands_total",
422
        "passed_commands",
423
        "failed_commands",
424
        "assertions_total",
425
        "assertions_failed",
426
        "required_checks_passed",
427
    ],
428
    "properties": {
429
        "commands_total": optional_non_negative_integer(),
430
        "passed_commands": optional_non_negative_integer(),
431
        "failed_commands": optional_non_negative_integer(),
432
        "assertions_total": optional_non_negative_integer(),
433
        "assertions_failed": optional_non_negative_integer(),
434
        "required_checks_passed": optional_boolean(),
435
    },
436
}
437
438
PARALLEL_METRICS_SCHEMA: dict[str, Any] = {
439
    "type": "object",
440
    "additionalProperties": False,
441
    "required": [
442
        "tasks_total",
443
        "tasks_completed",
444
        "max_parallel_tasks",
445
        "overlapping_files_count",
446
        "lost_updates_detected",
447
    ],
448
    "properties": {
449
        "tasks_total": non_negative_integer(),
450
        "tasks_completed": non_negative_integer(),
451
        "max_parallel_tasks": non_negative_integer(),
452
        "overlapping_files_count": optional_non_negative_integer(
453
            "Null when overlap detection did not run (e.g. single-task scenarios)."
454
        ),
455
        "lost_updates_detected": optional_non_negative_integer(
456
            "Null when no integrity check ran. Zero means the check ran and found nothing lost."
457
        ),
458
        "lock_wait_ms": optional_non_negative_number(
459
            "Time spent blocked on or retrying VCS locks under contention."
460
        ),
461
        "commit_throughput_per_s": optional_non_negative_number(
462
            "Successful snapshots per second across all parallel workers."
463
        ),
464
        "integrity_check_passed": optional_boolean(
465
            "fsck-equivalent result after the parallel run. Null when no check ran."
466
        ),
467
        "measurement_source": optional_string("How parallel behavior was observed."),
468
    },
469
}
470
471
TURN_METRICS_SCHEMA: dict[str, Any] = {
472
    "type": "object",
473
    "additionalProperties": False,
474
    "required": [
475
        "turn_source",
476
        "assistant_turns_total",
477
        "tool_calls_per_turn_avg",
478
        "max_tool_calls_per_turn",
479
        "cumulative_input_tokens_reported",
480
        "peak_input_tokens_reported",
481
    ],
482
    "properties": {
483
        "turn_source": {"type": "string", "enum": TURN_SOURCES},
484
        "assistant_turns_total": optional_non_negative_integer(
485
            "Model inference round-trips. Each turn re-pays the transcript as input tokens, so "
486
            "turns dominate agent cost; tool-call counts alone hide this."
487
        ),
488
        "tool_calls_per_turn_avg": optional_non_negative_number(),
489
        "max_tool_calls_per_turn": optional_non_negative_integer(
490
            "Peak batching within one turn. Batched calls are far cheaper than sequential turns."
491
        ),
492
        "cumulative_input_tokens_reported": optional_non_negative_integer(
493
            "Sum of provider-reported input tokens (including cache reads) across all turns. "
494
            "Captures context-residency cost: early verbose output is re-paid every later turn."
495
        ),
496
        "peak_input_tokens_reported": optional_non_negative_integer(
497
            "Largest single-turn input, a proxy for peak context size."
498
        ),
499
        "turn_timeline_path": optional_string(
500
            "Artifact path of per-turn JSONL: turn index, input/output/cache tokens, tool calls."
501
        ),
502
    },
503
}
504
505
SCHEMA: dict[str, Any] = {
506
    "$schema": "https://json-schema.org/draft/2020-12/schema",
507
    "title": "Oak agent workflow benchmark result row",
508
    "schema_version": SCHEMA_VERSION,
509
    "type": "object",
510
    "additionalProperties": False,
511
    "required": [
512
        "schema_version",
513
        "bench_id",
514
        "timestamp_utc",
515
        "scenario",
516
        "run",
517
        "subject",
518
        "subject_kind",
519
        "vcs_mode",
520
        "agent",
521
        "fixture",
522
        "outcome",
523
        "success",
524
        "success_criteria",
525
        "token_metrics",
526
        "turn_metrics",
527
        "tool_call_metrics",
528
        "failed_command_retry_metrics",
529
        "wall_clock_metrics",
530
        "bytes_metrics",
531
        "vcs_metrics",
532
        "test_metrics",
533
        "parallel_metrics",
534
    ],
535
    "properties": {
536
        "schema_version": {"type": "integer", "const": SCHEMA_VERSION},
537
        "bench_id": string("Stable identifier for a benchmark invocation."),
538
        "timestamp_utc": string("UTC ISO-8601 timestamp for row completion."),
539
        "profile": optional_string("Run profile such as smoke, standard, or large."),
540
        "host": optional_string("Host name used for run provenance."),
541
        "platform": optional_string("Platform string used for run provenance."),
542
        "machine": optional_string("Machine architecture used for run provenance."),
543
        "scenario": string("Scenario id from scenarios/agent.yaml."),
544
        "scenario_version": optional_string("Fixture or prompt version for this scenario."),
545
        "randomized_subject_order": {"type": "boolean"},
546
        "subject_order": {"type": "array", "items": {"type": "string"}},
547
        "agent_launched": boolean("Whether an agent process or mock was invoked."),
548
        "outcome_attribution": {"type": "string", "enum": ["agent", "subject", "harness"]},
549
        "setup_returncode": {"type": "integer"},
550
        "skipped": boolean("Whether this trial lacks a required measurement instrument."),
551
        "runner_id": optional_string(),
552
        "runner_class": optional_string(),
553
        "runner_profile": {"type": "object"},
554
        "cache_state": optional_string(),
555
        "cache_state_reason": optional_string(),
556
        "calibration_version": optional_string(),
557
        "calibration_unmeasured_reason": optional_string(),
558
        "workflow_integrity_evidence_class": optional_string(),
559
        "workflow_integrity_committed_scope": {"type": ["array", "null"], "items": {"type": "string"},
560
            "description": "Committed paths read by the Git or Oak committed-tree check (same scope for both)."},
561
        "workflow_subject_binary_sha256": optional_string(),
562
        "env_isolation_version": optional_string("Version of environment isolation policy."),
563
        "oracle_version": optional_string("Version of task correctness evidence."),
564
        "workflow_integrity_source": optional_string("Actual worktree/committed-tree evidence source."),
565
        "workflow_integrity_passed": optional_boolean("Null when required correctness instrument is unavailable."),
566
        "workflow_integrity_failure_reason": optional_string("Observed incorrect task state."),
567
        "workflow_integrity_unmeasured_reason": optional_string("Why required correctness evidence is unavailable."),
568
        "skip_reason": optional_string("Explicit missing measurement capability."),
569
        "git_head_instrument_binary": optional_string("Resolved Git object-reading instrument."),
570
        "git_head_instrument_sha256": optional_string("Git object reader SHA256."),
571
        "git_head_instrument_version": optional_string("Git object reader reported version."),
572
        "oak_export_instrument_binary": optional_string("Subject-produced export binary."),
573
        "oak_export_instrument_sha256": optional_string("Subject-produced export binary SHA256."),
574
        "oak_reader_version": optional_string(),
575
"oak_reader_sha256": optional_string(),
576
"oak_reader_head": optional_string(),
577
"oak_reader_blake3_version": optional_string(),
578
"oak_reader_zstandard_version": optional_string(),
579
"oak_reader_python_sha256": optional_string(),
580
        "oak_export_head": optional_string("Oak HEAD checked unchanged across export."),
581
        "operation": optional_string("Optional report grouping operation, e.g. agent.workflow."),
582
        "benchmark_track": optional_string("Command semantics track such as agent-default or core-equivalent."),
583
        "command_semantics_version": optional_string("Version of the command semantics contract."),
584
        "elapsed_ms": optional_non_negative_number("Compatibility mirror of wall_clock_metrics.total_ms."),
585
        "returncode": optional_non_negative_integer("Compatibility mirror: 0 for success, non-zero for failure."),
586
        "raw_output_bytes": optional_non_negative_integer("Compatibility mirror of captured agent stdout+stderr bytes."),
587
        "stdout_bytes": optional_non_negative_integer("Captured agent stdout bytes."),
588
        "stderr_bytes": optional_non_negative_integer("Captured agent stderr bytes."),
589
        "output_truncated": optional_boolean("Whether captured output was truncated before artifact storage."),
590
        "task_id": optional_string("Subtask id for parallel scenarios, if this row is per subtask."),
591
        "task_prompt_id": optional_string("Stable prompt/template id."),
592
        "run": non_negative_integer("Zero-based repetition index."),
593
        "subject": string("Benchmark subject such as git, oak_installed, or oak_local."),
594
        "subject_kind": {"type": "string", "enum": SUBJECT_KINDS},
595
        "subject_label": optional_string("Human label for reports."),
596
        "vcs_mode": {"type": "string", "enum": VCS_MODES},
597
        "agent": {
598
            "type": "object",
599
            "additionalProperties": False,
600
            "required": ["name", "adapter"],
601
            "properties": {
602
                "name": string("Agent or harness adapter name."),
603
                "adapter": string("Runner integration name."),
604
                "provider": optional_string("Provider name, if applicable."),
605
                "model": optional_string("Model id, if applicable."),
606
                "version": optional_string("Agent binary, prompt, or adapter version."),
607
                "environment": optional_string("Agent environment mode, such as minimal or local-default."),
608
                "instruction_level": {
609
                    "type": ["string", "null"],
610
                    "enum": INSTRUCTION_LEVELS + [None],
611
                    "description": (
612
                        "How much VCS-specific instruction the prompt carried. zero-shot measures the "
613
                        "model-familiarity tax: Git is in pretraining data, a new VCS is not."
614
                    ),
615
                },
616
                "temperature": {"type": ["number", "null"]},
617
                "max_output_tokens": optional_non_negative_integer(),
618
            },
619
        },
620
        "fixture": {
621
            "type": "object",
622
            "additionalProperties": False,
623
            "required": ["id", "repo_shape", "fixture_version"],
624
            "properties": {
625
                "id": string("Fixture id used by the runner."),
626
                "repo_shape": string("Human-readable fixture shape."),
627
                "fixture_version": string("Stable fixture content version or hash."),
628
                "logical_file_count": optional_non_negative_integer(),
629
                "logical_bytes": optional_non_negative_integer(),
630
            },
631
        },
632
        "source": {
633
            "type": "object",
634
            "additionalProperties": True,
635
            "properties": {
636
                "oak_hash": optional_string(),
637
                "oak_status": optional_string(),
638
                "benchmark_repo_hash": optional_string(),
639
            },
640
        },
641
        "outcome": {"type": "string", "enum": OUTCOMES},
642
        "success": boolean("True only when deterministic scenario criteria passed."),
643
        "failure_reason": optional_string("Short stable failure reason for dashboards."),
644
        "success_criteria": {
645
            "type": "object",
646
            "additionalProperties": False,
647
            "required": ["passed", "checks"],
648
            "properties": {
649
                "passed": boolean(),
650
                "checks": {
651
                    "type": "object",
652
                    "additionalProperties": optional_boolean("Individual deterministic check result; null if unmeasured."),
653
                    "properties": {},
654
                },
655
                "notes": optional_string(),
656
            },
657
        },
658
        "token_metrics": TOKEN_METRICS_SCHEMA,
659
        "turn_metrics": TURN_METRICS_SCHEMA,
660
        "tool_call_metrics": TOOL_CALL_METRICS_SCHEMA,
661
        "failed_command_retry_metrics": FAILURE_RETRY_METRICS_SCHEMA,
662
        "wall_clock_metrics": WALL_CLOCK_METRICS_SCHEMA,
663
        "bytes_metrics": BYTES_METRICS_SCHEMA,
664
        "vcs_metrics": VCS_METRICS_SCHEMA,
665
        "test_metrics": TEST_METRICS_SCHEMA,
666
        "parallel_metrics": PARALLEL_METRICS_SCHEMA,
667
        "artifacts": {
668
            "type": "object",
669
            "additionalProperties": False,
670
            "properties": {
671
                "transcript_path": optional_string(),
672
                "command_log_path": optional_string(),
673
                "diff_path": optional_string(),
674
                "workspace_path": optional_string(),
675
                "extra_paths": string_array(),
676
            },
677
        },
678
        "notes": optional_string(),
679
    },
680
}
681
682
683
def type_matches(value: Any, expected_type: Any) -> bool:
684
    if isinstance(expected_type, list):
685
        return any(type_matches(value, item) for item in expected_type)
686
    if expected_type == "null":
687
        return value is None
688
    if expected_type == "boolean":
689
        return isinstance(value, bool)
690
    if expected_type == "integer":
691
        return isinstance(value, int) and not isinstance(value, bool)
692
    if expected_type == "number":
693
        return (isinstance(value, int) or isinstance(value, float)) and not isinstance(value, bool)
694
    if expected_type == "string":
695
        return isinstance(value, str)
696
    if expected_type == "object":
697
        return isinstance(value, dict)
698
    if expected_type == "array":
699
        return isinstance(value, list)
700
    return True
701
702
703
def display_path(path: str, key: str) -> str:
704
    if path == "$":
705
        return "$" + "." + key
706
    return path + "." + key
707
708
709
def validate_value(value: Any, schema: dict[str, Any], path: str) -> List[str]:
710
    errors: List[str] = []
711
    expected_type = schema.get("type")
712
    if expected_type is not None and not type_matches(value, expected_type):
713
        errors.append(f"{path}: expected {expected_type}, got {type(value).__name__}")
714
        return errors
715
716
    if "const" in schema and value != schema["const"]:
717
        errors.append(f"{path}: expected constant {schema['const']!r}, got {value!r}")
718
    if "enum" in schema and value not in schema["enum"]:
719
        errors.append(f"{path}: expected one of {schema['enum']!r}, got {value!r}")
720
    if "minimum" in schema and isinstance(value, (int, float)) and not isinstance(value, bool):
721
        if value < schema["minimum"]:
722
            errors.append(f"{path}: expected >= {schema['minimum']}, got {value!r}")
723
724
    if isinstance(value, dict):
725
        required = schema.get("required", [])
726
        for key in required:
727
            if key not in value:
728
                errors.append(f"{path}: missing required property {key!r}")
729
        properties = schema.get("properties", {})
730
        additional = schema.get("additionalProperties", True)
731
        for key, item in value.items():
732
            child_path = display_path(path, key)
733
            if key in properties:
734
                errors.extend(validate_value(item, properties[key], child_path))
735
            elif additional is False:
736
                errors.append(f"{child_path}: unexpected property")
737
            elif isinstance(additional, dict):
738
                errors.extend(validate_value(item, additional, child_path))
739
    elif isinstance(value, list) and "items" in schema:
740
        item_schema = schema["items"]
741
        for index, item in enumerate(value):
742
            errors.extend(validate_value(item, item_schema, f"{path}[{index}]"))
743
744
    return errors
745
746
747
def validate_schema_shape(schema: dict[str, Any]) -> List[str]:
748
    errors: List[str] = []
749
    for key in ["title", "type", "required", "properties"]:
750
        if key not in schema:
751
            errors.append(f"schema missing {key!r}")
752
    if schema.get("type") != "object":
753
        errors.append("schema root type must be object")
754
    properties = schema.get("properties", {})
755
    for required_key in schema.get("required", []):
756
        if required_key not in properties:
757
            errors.append(f"schema requires {required_key!r} but has no property definition")
758
    return errors
759
760
761
def validate_row(row: Any) -> List[str]:
762
    if not isinstance(row, dict):
763
        return [f"$: expected object row, got {type(row).__name__}"]
764
    return validate_value(row, SCHEMA, "$")
765
766
767
def iter_rows(path: Path) -> Iterable[Any]:
768
    text = path.read_text(encoding="utf-8")
769
    stripped = text.strip()
770
    if not stripped:
771
        return []
772
    if path.suffix == ".json" or stripped[0] == "[":
773
        data = json.loads(stripped)
774
        if isinstance(data, list):
775
            return data
776
        return [data]
777
    rows: List[Any] = []
778
    for line_number, line in enumerate(text.splitlines(), 1):
779
        if not line.strip():
780
            continue
781
        try:
782
            rows.append(json.loads(line))
783
        except json.JSONDecodeError as exc:
784
            rows.append({"__json_error__": f"line {line_number}: {exc}"})
785
    return rows
786
787
788
def validate_file(path: Path) -> dict[str, Any]:
789
    errors: List[str] = []
790
    rows = list(iter_rows(path))
791
    for index, row in enumerate(rows, 1):
792
        if isinstance(row, dict) and "__json_error__" in row:
793
            errors.append(row["__json_error__"])
794
            continue
795
        for error in validate_row(row):
796
            errors.append(f"row {index}: {error}")
797
    return {"ok": not errors, "rows": len(rows), "errors": errors}
798
799
800
def example_row() -> dict[str, Any]:
801
    return {
802
        "schema_version": SCHEMA_VERSION,
803
        "bench_id": "20260609T000000Z",
804
        "timestamp_utc": "2026-06-09T00:00:00Z",
805
        "profile": "smoke",
806
        "scenario": "one_line_bug_fix",
807
        "scenario_version": "v1",
808
        "task_id": None,
809
        "task_prompt_id": "agent-one-line-bug-fix-v1",
810
        "run": 0,
811
        "subject": "oak_local",
812
        "subject_kind": "oak",
813
        "subject_label": "Oak local",
814
        "vcs_mode": "oak_space_per_task",
815
        "agent": {
816
            "name": "example-agent",
817
            "adapter": "example-adapter",
818
            "provider": None,
819
            "model": None,
820
            "version": None,
821
            "environment": "minimal",
822
            "instruction_level": "cheat-sheet",
823
            "temperature": None,
824
            "max_output_tokens": None,
825
        },
826
        "fixture": {
827
            "id": "agent_one_line_bug_fix",
828
            "repo_shape": "small text application",
829
            "fixture_version": "v1",
830
            "logical_file_count": 42,
831
            "logical_bytes": 65536,
832
        },
833
        "source": {"oak_hash": None, "oak_status": None, "benchmark_repo_hash": None},
834
        "outcome": "pass",
835
        "success": True,
836
        "failure_reason": None,
837
        "success_criteria": {
838
            "passed": True,
839
            "checks": {"tests_passed": True, "expected_files_changed": True},
840
            "notes": None,
841
        },
842
        "token_metrics": {
843
            "token_source": "char_only",
844
            "tokenizer": None,
845
            "input_tokens_reported": None,
846
            "output_tokens_reported": None,
847
            "reasoning_tokens_reported": None,
848
            "cache_read_tokens_reported": None,
849
            "cache_write_tokens_reported": None,
850
            "total_tokens_reported": None,
851
            "normalized_input_chars": 12000,
852
            "normalized_output_chars": 1200,
853
            "transcript_chars": 13200,
854
            "context_truncation_events": None,
855
            "token_budget": None,
856
            "cost_weighted_total": 4500.0,
857
            "cost_weights": "emitted=5,ingested=1",
858
        },
859
        "turn_metrics": {
860
            "turn_source": "adapter_stream",
861
            "assistant_turns_total": 6,
862
            "tool_calls_per_turn_avg": 1.33,
863
            "max_tool_calls_per_turn": 2,
864
            "cumulative_input_tokens_reported": 48000,
865
            "peak_input_tokens_reported": 11000,
866
            "turn_timeline_path": None,
867
        },
868
        "tool_call_metrics": {
869
            "tool_calls_total": 8,
870
            "tool_calls_by_kind": {"terminal": 3, "read": 3, "edit": 1, "search": 1},
871
            "terminal_commands_total": 3,
872
            "file_read_calls": 3,
873
            "file_write_calls": 1,
874
            "search_calls": 1,
875
            "edit_calls": 1,
876
            "vcs_commands_total": 2,
877
            "test_commands_total": 1,
878
            "network_calls_total": 0,
879
            "privileged_calls_total": 0,
880
            "help_calls_total": 0,
881
            "unknown_command_failures_total": None,
882
            "parallel_batches_total": None,
883
            "max_concurrency": None,
884
            "measurement_source": "adapter_stream_events",
885
        },
886
        "failed_command_retry_metrics": {
887
            "failed_tool_calls_total": 0,
888
            "failed_commands_total": 0,
889
            "failed_vcs_commands_total": 0,
890
            "failed_test_commands_total": 0,
891
            "retry_attempts_total": None,
892
            "retry_chains_total": None,
893
            "same_command_retries_total": None,
894
            "retry_after_failure_ms": None,
895
            "recoverable_failures_total": None,
896
            "unrecoverable_failures_total": None,
897
            "turns_to_recovery": None,
898
            "tokens_to_recovery": None,
899
            "failed_command_examples": [],
900
            "measurement_source": "adapter_stream_events",
901
        },
902
        "wall_clock_metrics": {
903
            "total_ms": 15000.0,
904
            "setup_ms": 800.0,
905
            "agent_active_ms": 7000.0,
906
            "tool_wait_ms": None,
907
            "vcs_ms": 900.0,
908
            "test_ms": 2100.0,
909
            "merge_ms": None,
910
            "cleanup_ms": None,
911
            "idle_ms": None,
912
            "timeout_ms": None,
913
            "measurement_source": "outer_cli_wall_clock",
914
        },
915
        "bytes_metrics": {
916
            "workspace_logical_bytes": 65536,
917
            "workspace_materialized_bytes": 32768,
918
            "bytes_read_by_agent": 18000,
919
            "bytes_written_by_agent": 140,
920
            "bytes_hydrated": None,
921
            "files_hydrated": None,
922
            "hydration_events": None,
923
            "large_file_bytes_scanned": None,
924
            "binary_bytes_touched": None,
925
            "command_output_bytes": 3000,
926
            "measurement_source": "example-adapter",
927
        },
928
        "vcs_metrics": {
929
            "commands_total": 2,
930
            "status_commands": 1,
931
            "diff_commands": 0,
932
            "snapshot_commands": 1,
933
            "branch_commands": 0,
934
            "merge_commands": 0,
935
            "cleanup_commands": 0,
936
            "commits_created": 1,
937
            "branches_created": 1,
938
            "conflicts_total": None,
939
            "conflicts_resolved": None,
940
            "dirty_files_at_end": 0,
941
            "cleanup_success": True,
942
            "measurement_source": "transcript_command_classification",
943
        },
944
        "test_metrics": {
945
            "commands_total": 1,
946
            "passed_commands": 1,
947
            "failed_commands": 0,
948
            "assertions_total": None,
949
            "assertions_failed": None,
950
            "required_checks_passed": True,
951
        },
952
        "parallel_metrics": {
953
            "tasks_total": 1,
954
            "tasks_completed": 1,
955
            "max_parallel_tasks": 1,
956
            "overlapping_files_count": None,
957
            "lost_updates_detected": None,
958
            "lock_wait_ms": None,
959
            "commit_throughput_per_s": None,
960
            "integrity_check_passed": None,
961
            "measurement_source": "single_task_run",
962
        },
963
        "artifacts": {
964
            "transcript_path": None,
965
            "command_log_path": None,
966
            "diff_path": None,
967
            "workspace_path": None,
968
            "extra_paths": [],
969
        },
970
        "notes": None,
971
    }
972
973
974
def parse_args() -> argparse.Namespace:
975
    parser = argparse.ArgumentParser(description=__doc__)
976
    parser.add_argument("--validate-jsonl", type=Path, help="Validate a JSON or JSONL result file.")
977
    parser.add_argument("--example", action="store_true", help="Print an example valid result row.")
978
    parser.add_argument("--compact", action="store_true", help="Print compact JSON.")
979
    return parser.parse_args()
980
981
982
def print_json(value: Any, compact: bool) -> None:
983
    if compact:
984
        print(json.dumps(value, sort_keys=True, separators=(",", ":")))
985
    else:
986
        print(json.dumps(value, indent=2, sort_keys=True))
987
988
989
def main() -> int:
990
    args = parse_args()
991
    schema_errors = validate_schema_shape(SCHEMA)
992
    if schema_errors:
993
        print_json({"ok": False, "schema_errors": schema_errors}, args.compact)
994
        return 1
995
996
    if args.example:
997
        row = example_row()
998
        errors = validate_row(row)
999
        if errors:
1000
            print_json({"ok": False, "example_errors": errors}, args.compact)
1001
            return 1
1002
        print_json(row, args.compact)
1003
        return 0
1004
1005
    if args.validate_jsonl:
1006
        result = validate_file(args.validate_jsonl)
1007
        print_json(result, args.compact)
1008
        return 0 if result["ok"] else 1
1009
1010
    print_json(SCHEMA, args.compact)
1011
    return 0
1012
1013
1014
if __name__ == "__main__":
1015
    raise SystemExit(main())