version: 1

suite: agent_workflow

suite_version: 2026-06-09

description: >-

  End-to-end agent workflow benchmark definitions for comparing VCS-backed

  coding tasks across Git and Oak without assuming a specific model provider.

runner_contract:

  row_schema: scripts/agent_metrics_schema.py

  row_granularity: one result row per scenario, agent, subject, vcs_mode, and run

  comparison_policy: >-

    Compare the same agent adapter and task prompt across VCS modes first. Cross-agent

    comparisons must report provider-reported tokens separately from normalized text

    and tool I/O metrics.

  setup_excluded_from_timing:

    - fixture_generation

    - subject_binary_build

    - dependency_cache_population

  timed_phases:

    - task_setup

    - prompt_delivery

    - agent_work

    - tool_wait

    - vcs_operations

    - tests

    - merge_or_cleanup

vcs_modes:

  git_single_checkout:

    subject_kind: git

    description: One mutable checkout per run. Parallel scenarios must run serially in this mode.

  git_worktree_per_task:

    subject_kind: git

    description: One Git worktree or clone per task branch, with merge back to the run root.

  oak_single_mount:

    subject_kind: oak

    description: One Oak mount for the run, with branches created inside that mount.

  oak_mount_per_task:

    subject_kind: oak

    description: One Oak mount per task branch, without the higher-level Oak space template flow.

  oak_space_per_task:

    subject_kind: oak

    description: One Oak space subdirectory per task, using the agent-oriented mount lifecycle.

mode_sets:

  single_task_modes: &single_task_modes

    - git_single_checkout

    - git_worktree_per_task

    - oak_single_mount

    - oak_mount_per_task

    - oak_space_per_task

  isolated_parallel_modes: &isolated_parallel_modes

    - git_worktree_per_task

    - oak_mount_per_task

    - oak_space_per_task

  cleanup_modes: &cleanup_modes

    - git_single_checkout

    - git_worktree_per_task

    - oak_single_mount

    - oak_mount_per_task

    - oak_space_per_task

metric_groups:

  token_metrics: &token_metrics

    - input_tokens_reported

    - output_tokens_reported

    - reasoning_tokens_reported

    - cache_read_tokens_reported

    - cache_write_tokens_reported

    - total_tokens_reported

    - normalized_input_chars

    - normalized_output_chars

    - transcript_chars

    - context_truncation_events

    - token_budget

    - token_source

  tool_call_metrics: &tool_call_metrics

    - tool_calls_total

    - tool_calls_by_kind

    - terminal_commands_total

    - file_read_calls

    - file_write_calls

    - search_calls

    - edit_calls

    - vcs_commands_total

    - test_commands_total

    - network_calls_total

    - privileged_calls_total

    - parallel_batches_total

    - max_concurrency

  failed_command_retry_metrics: &failed_command_retry_metrics

    - failed_tool_calls_total

    - failed_commands_total

    - failed_vcs_commands_total

    - failed_test_commands_total

    - retry_attempts_total

    - retry_chains_total

    - same_command_retries_total

    - retry_after_failure_ms

    - recoverable_failures_total

    - unrecoverable_failures_total

  wall_clock_metrics: &wall_clock_metrics

    - total_ms

    - setup_ms

    - agent_active_ms

    - tool_wait_ms

    - vcs_ms

    - test_ms

    - merge_ms

    - cleanup_ms

    - idle_ms

    - timeout_ms

  bytes_read_hydrated_metrics: &bytes_read_hydrated_metrics

    - workspace_logical_bytes

    - workspace_materialized_bytes

    - bytes_read_by_agent

    - bytes_written_by_agent

    - bytes_hydrated

    - files_hydrated

    - hydration_events

    - large_file_bytes_scanned

    - binary_bytes_touched

    - command_output_bytes

scenarios:

  - id: one_line_bug_fix

    title: One-line bug fix

    category: small_edit

    prompt_shape: >-

      Fix the failing unit test by making the smallest reasonable code change.

    expected_fixtures:

      fixture_id: agent_one_line_bug_fix

      repo_shape: Small text application with one intentionally wrong conditional or operator.

      required_files:

        - src/calculator.py

        - tests/test_calculator.py

        - README.md

      seeded_state:

        - Exactly one deterministic failing test identifies the behavior.

        - Neighboring files contain plausible but irrelevant code.

      expected_change_shape:

        files_changed_min: 1

        files_changed_max: 2

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - The targeted test suite passes.

      - The broken behavior is fixed for the asserted edge case.

      - No unrelated source, docs, lockfile, generated, or asset files change.

      - The final VCS state is clean and snapshotable.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Baseline overhead for tiny tasks where VCS setup can dominate agent work.

      - Ratio of useful file reads to exploratory reads.

  - id: docs_edit

    title: Docs edit

    category: documentation

    prompt_shape: >-

      Update the user-facing documentation for a changed command or option without

      changing executable code.

    expected_fixtures:

      fixture_id: agent_docs_edit

      repo_shape: Medium repo with README, docs pages, and nearby stale examples.

      required_files:

        - README.md

        - docs/usage.md

        - docs/reference.md

        - src/cli/options.py

      seeded_state:

        - Source code already implements the behavior.

        - At least two docs files contain outdated wording.

      expected_change_shape:

        files_changed_min: 1

        files_changed_max: 3

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - Documentation reflects the fixture's current command behavior.

      - Code and generated files remain unchanged.

      - Markdown formatting checks pass when the fixture provides them.

      - Final status reports no dirty files after snapshot.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Cost of inspecting source only enough to verify docs facts.

      - Avoiding broad repo scans after the relevant docs are found.

  - id: cross_file_refactor

    title: Cross-file refactor

    category: refactor

    prompt_shape: >-

      Rename or reshape a public helper used across implementation, tests, and docs

      while preserving behavior.

    expected_fixtures:

      fixture_id: agent_cross_file_refactor

      repo_shape: Text-heavy application with shared modules, tests, examples, and docs references.

      required_files:

        - src/core/serializer.py

        - src/api/client.py

        - src/cli/main.py

        - tests/test_serializer.py

        - docs/api.md

      seeded_state:

        - The old symbol appears in code, tests, comments, and docs.

        - Similar symbol names exist and must not be changed.

      expected_change_shape:

        files_changed_min: 4

        files_changed_max: 12

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - All runtime references use the new API name or shape.

      - Tests and type checks pass for the fixture language.

      - Similar but unrelated symbols are untouched.

      - Docs examples compile or match the new public API.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Search and edit breadth needed for a correct multi-file change.

      - Retry behavior after tests expose a missed reference.

  - id: dependency_upgrade

    title: Dependency upgrade

    category: dependency_management

    prompt_shape: >-

      Upgrade one pinned dependency to the requested version and fix any API or lockfile fallout.

    expected_fixtures:

      fixture_id: agent_dependency_upgrade

      repo_shape: Small service with a package manifest, lockfile, cached dependency artifacts, and tests.

      required_files:

        - package.json

        - package-lock.json

        - src/server.js

        - tests/server.test.js

      seeded_state:

        - A local fixture cache or registry provides the target dependency version.

        - The upgraded dependency requires a small source or test adjustment.

      expected_change_shape:

        files_changed_min: 2

        files_changed_max: 5

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - Manifest and lockfile agree on the requested dependency version.

      - The package manager runs without reaching an uncontrolled network source.

      - Tests pass after any required API adjustment.

      - Unrelated dependencies are not upgraded.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Lockfile churn and command retries caused by package-manager behavior.

      - Fair treatment of network-disabled or cache-only runners.

  - id: generated_file_churn

    title: Generated-file churn

    category: generated_outputs

    prompt_shape: >-

      Change a schema or template, run the generator, and commit the resulting deterministic output.

    expected_fixtures:

      fixture_id: agent_generated_file_churn

      repo_shape: Project with source schema, generator script, and hundreds of generated text files.

      required_files:

        - schema/events.yaml

        - scripts/generate_events.py

        - src/generated/events/

        - tests/test_generated_events.py

      seeded_state:

        - Generated files are stale relative to the requested schema change.

        - A fixture test fails until generation is rerun.

      expected_change_shape:

        files_changed_min: 20

        files_changed_max: 400

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - Source schema reflects the requested change.

      - Generated files match a clean generator run.

      - Tests pass without hand-editing generated output to hide generator drift.

      - VCS snapshot handles high file-count churn correctly.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - File-status and snapshot cost under many changed files.

      - Distinguishing agent reads from generator-written bytes.

  - id: binary_asset_update

    title: Binary asset update

    category: binary_assets

    prompt_shape: >-

      Replace or update a binary asset and adjust the metadata that validates it.

    expected_fixtures:

      fixture_id: agent_binary_asset_update

      repo_shape: App with small source files, binary images, and checksum or dimension metadata.

      required_files:

        - assets/logo.png

        - assets/[email protected]

        - src/assets_manifest.json

        - tests/test_assets.py

      seeded_state:

        - Replacement binary assets are provided inside the fixture inputs directory.

        - Metadata is stale until the asset update is applied.

      expected_change_shape:

        files_changed_min: 2

        files_changed_max: 4

        binary_assets: true

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - The requested binary asset bytes are installed exactly.

      - Checksums, dimensions, or manifest entries are correct.

      - Binary files are not corrupted by text editing tools.

      - Asset validation tests pass.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Bytes hydrated and copied for binary-heavy changes.

      - Whether the agent avoids dumping binary content into context.

  - id: large_file_inspection

    title: Large-file inspection

    category: large_files

    prompt_shape: >-

      Inspect a large structured data file to answer a targeted question, then update

      a small verification file or test expectation.

    expected_fixtures:

      fixture_id: agent_large_file_inspection

      repo_shape: Repo containing a large NDJSON or CSV file plus small indexes and tests.

      required_files:

        - data/events.ndjson

        - data/events.index.json

        - tests/test_event_fixture.py

        - docs/event-fixture-notes.md

      seeded_state:

        - The target record is discoverable by a stable key.

        - The large file is too large to read fully into the agent transcript.

      expected_change_shape:

        files_changed_min: 1

        files_changed_max: 3

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - The answer or expectation matches the targeted record in the large file.

      - The agent does not modify the large fixture unless the task explicitly requires it.

      - Tests or verification commands pass.

      - The transcript does not contain a bulk dump of the large file.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Logical bytes read versus bytes hydrated from storage.

      - Effectiveness of range reads, search, and indexes.

  - id: find_the_right_file_exploration

    title: Find-the-right-file exploration

    category: navigation

    prompt_shape: >-

      Resolve a vague bug report by locating the responsible file in a broad repo.

    expected_fixtures:

      fixture_id: agent_find_right_file

      repo_shape: Wide source tree with many plausible modules and one localized bug.

      required_files:

        - src/features/

        - src/shared/

        - tests/integration/test_user_flow.py

        - docs/architecture.md

      seeded_state:

        - The failing behavior is described at product level, not by filename.

        - Several files contain similar terms to create realistic search noise.

      expected_change_shape:

        files_changed_min: 1

        files_changed_max: 4

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - The correct module is changed and the reported behavior is fixed.

      - Search noise files remain untouched.

      - Integration and targeted tests pass.

      - The agent records enough evidence in tool traces to justify the file choice.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Search calls, directory listings, and file reads before first edit.

      - Whether lazy hydration reduces cost for broad exploration.

  - id: parallel_independent_tasks

    title: Parallel independent tasks

    category: parallel_work

    prompt_shape: >-

      Run multiple independent agent tasks against the same starting revision and merge them.

    expected_fixtures:

      fixture_id: agent_parallel_independent

      repo_shape: App with three unrelated issues in separate modules and tests.

      required_files:

        - src/payments/

        - src/search/

        - docs/cli.md

        - tests/

      seeded_state:

        - Task A is a source bug fix.

        - Task B is a docs edit.

        - Task C is a small test expectation update.

      expected_change_shape:

        files_changed_min: 3

        files_changed_max: 9

        binary_assets: false

    allowed_vcs_modes: *isolated_parallel_modes

    task_success_criteria:

      - Every subtask succeeds against the shared starting revision.

      - All accepted changes are present after merge or squash.

      - No subtask observes another subtask's unmerged dirty work.

      - Final combined tests pass and the run root is clean.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Branch or mount creation overhead under concurrent work.

      - Wall-clock speedup versus serialized task duration.

  - id: parallel_overlapping_tasks

    title: Parallel overlapping tasks

    category: parallel_work

    prompt_shape: >-

      Run multiple tasks that touch nearby files or non-conflicting hunks, then merge them.

    expected_fixtures:

      fixture_id: agent_parallel_overlapping

      repo_shape: App with two feature requests touching the same module family.

      required_files:

        - src/config/defaults.py

        - src/config/validation.py

        - tests/test_config.py

        - docs/config.md

      seeded_state:

        - Subtasks overlap in file set but target different symbols or sections.

        - The correct combined result should merge without semantic loss.

      expected_change_shape:

        files_changed_min: 2

        files_changed_max: 8

        binary_assets: false

    allowed_vcs_modes: *isolated_parallel_modes

    task_success_criteria:

      - Both overlapping task intents survive the final merge.

      - Any auto-merge is verified by tests and direct inspection.

      - No last-writer-wins overwrite is accepted.

      - Final status is clean after snapshot or merge.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Conflict-free overlap handling and merge verification cost.

      - Extra reads needed to confirm both task intents remain present.

  - id: conflict_resolution

    title: Conflict resolution

    category: merge_conflicts

    prompt_shape: >-

      Merge two completed task branches that intentionally edit the same lines, resolve

      the conflict, and preserve both behavioral intents.

    expected_fixtures:

      fixture_id: agent_conflict_resolution

      repo_shape: Small service with two branches or mounts that change the same function.

      required_files:

        - src/routing/rules.py

        - tests/test_routing_rules.py

        - docs/routing.md

      seeded_state:

        - Branch A changes behavior for one route class.

        - Branch B changes behavior for a second route class in the same hunk.

        - The naive conflict choices each break one test.

      expected_change_shape:

        files_changed_min: 1

        files_changed_max: 5

        binary_assets: false

    allowed_vcs_modes: *isolated_parallel_modes

    task_success_criteria:

      - The conflict is detected and resolved deliberately.

      - Tests for both branch intents pass after resolution.

      - Conflict markers are absent from tracked files.

      - Merge metadata or snapshot state is clean and auditable.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Failed merge commands, conflict-resolution retries, and verification tests.

      - Whether VCS output gives agents enough context without excessive file reads.

  - id: failed_test_then_fix

    title: Failed-test-then-fix

    category: recovery

    prompt_shape: >-

      Make an apparently obvious fix, observe the failing test output, then diagnose

      and apply the correct fix.

    expected_fixtures:

      fixture_id: agent_failed_test_then_fix

      repo_shape: Small application where the first plausible edit leaves a hidden edge case broken.

      required_files:

        - src/parser.py

        - tests/test_parser.py

        - tests/fixtures/parser_cases.json

      seeded_state:

        - The direct symptom points to one line, but the correct fix requires another helper.

        - Test output includes enough detail for recovery.

      expected_change_shape:

        files_changed_min: 1

        files_changed_max: 4

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - At least one failing test command is observed before the final successful run.

      - The final fix handles both the original symptom and the edge case.

      - Tests pass after the recovery edit.

      - Failed intermediate edits are not left as unrelated churn.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Command failure handling and useful retries after test output.

      - Additional context cost required for recovery.

  - id: abandon_and_cleanup

    title: Abandon and cleanup

    category: lifecycle

    prompt_shape: >-

      Start a task, discover from repository evidence that the requested change should

      not be made, then abandon the branch or mount cleanly.

    expected_fixtures:

      fixture_id: agent_abandon_cleanup

      repo_shape: Project with a misleading request contradicted by policy docs or tests.

      required_files:

        - docs/policy.md

        - src/feature_flags.py

        - tests/test_feature_flags.py

      seeded_state:

        - The requested feature is explicitly disallowed by repository policy.

        - A partial local edit would be easy but incorrect.

      expected_change_shape:

        files_changed_min: 0

        files_changed_max: 0

        binary_assets: false

    allowed_vcs_modes: *cleanup_modes

    task_success_criteria:

      - The agent identifies repository evidence for abandoning the task.

      - No partial edits, commits, branches, mounts, or worktrees are leaked after cleanup.

      - The original checkout or space is clean.

      - The final result records an intentional abandoned outcome rather than a task failure.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Lifecycle cleanup cost and correctness.

      - Distinguishing productive abandonment from timeout or crash failure.

  - id: history_archaeology

    title: History archaeology

    category: history_read

    prompt_shape: >-

      Answer "what changed recently, what did the last commit touch, when did this

      file last change, and which commit introduced this term" using history

      commands only.

    expected_fixtures:

      fixture_id: agent_history_archaeology

      repo_shape: Small application with several seeded commits carrying meaningful messages.

      required_files:

        - app/pricing.py

        - app/totals.py

        - README.md

      seeded_state:

        - At least three commits after the initial snapshot, each editing a known file.

        - Commit messages name the behavior change so log output is answerable.

      expected_change_shape:

        files_changed_min: 0

        files_changed_max: 0

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - The history questions are answered from command output, not guesses.

      - No working-tree files change.

      - Subjects lacking a capability (file-scoped log, pickaxe, show) run the

        closest available command; the gap is recorded, not hidden.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Token cost per answered history question, by subject.

      - Capability gaps where one subject needs extra commands or full-history dumps.

    implemented_by: scripts/workflow_ab.py --workflows history_archaeology

  - id: vcs_error_recovery

    title: VCS error recovery

    category: failure_recovery

    prompt_shape: >-

      Recover from common VCS failures: a snapshot with nothing to record and a

      mistyped subcommand, then complete a real change and snapshot it.

    expected_fixtures:

      fixture_id: agent_vcs_error_recovery

      repo_shape: Small clean repository where the first snapshot attempt has nothing to commit.

      required_files:

        - app/pricing.py

        - README.md

      seeded_state:

        - The working tree is clean so the first commit attempt fails or no-ops.

        - The scripted lane also issues one intentionally unknown subcommand.

      expected_change_shape:

        files_changed_min: 1

        files_changed_max: 1

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - The run ends with the real change snapshotted and a clean tree.

      - Error output cost (bytes, estimated tokens) is recorded per failure step.

      - For real-agent rows, turns_to_recovery and tokens_to_recovery are populated

        when the adapter exposes per-call outcomes.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Error-message ergonomics, measured as recovery turns and tokens; an error

        message is a prompt.

      - Exit-code clarity and whether a follow-up status call is needed to learn state.

    implemented_by: scripts/workflow_ab.py --workflows vcs_error_recovery

  - id: model_familiarity_tax

    title: Model familiarity tax

    category: zero_shot_competence

    prompt_shape: >-

      Run an identical task at three instruction levels (zero-shot, cheat-sheet,

      full-docs) and compare success rate, help/discovery calls, hallucinated

      commands, and tokens to reach parity.

    expected_fixtures:

      fixture_id: agent_one_line_bug_fix

      repo_shape: Reuses an existing small-task fixture; only the instructions vary.

      required_files:

        - app/pricing.py

        - tests/test_pricing.py

      seeded_state:

        - Git is in every model's pretraining data; a new VCS is not. The fixed

          per-session instruction cost and the guessing cost are the measurement.

      expected_change_shape:

        files_changed_min: 1

        files_changed_max: 2

        binary_assets: false

    allowed_vcs_modes: *single_task_modes

    task_success_criteria:

      - Rows record agent.instruction_level and are never aggregated across levels.

      - Zero-shot rows count help_calls_total and unknown_command_failures_total.

      - The headline metric is instruction tokens required to reach git-parity

        success rate, per scenario.

    token_metrics: *token_metrics

    tool_call_metrics: *tool_call_metrics

    failed_command_retry_metrics: *failed_command_retry_metrics

    wall_clock_metrics: *wall_clock_metrics

    bytes_read_hydrated_metrics: *bytes_read_hydrated_metrics

    metric_focus:

      - Success rate and retry chains per instruction level, per subject.

      - Help/discovery tool calls and hallucinated-command failures at zero-shot.

    implemented_by: scripts/agent_workflow.py --instruction-level {zero-shot,cheat-sheet,full-docs}