| 1 | version: 1
|
| 2 | suite: platform_integration_lifecycle
|
| 3 | description: GitHub-the-platform vs oak hosted integration workflows, measured from "task branch exists locally" to "change merged in main, present in local checkout", including the async waits agents actually experience.
|
| 4 |
|
| 5 | # This lane runs against real hosted services and is excluded from devloop:
|
| 6 | # external variance must never poison changeset verdicts. Repos are configured
|
| 7 | # per platform at the CLI (--repo) and must be DISPOSABLE benchmark repos.
|
| 8 | # Scenario and operation names below are immutable identity (ADR-0005);
|
| 9 | # the polling cadence is encoded in the scenario name (poll_until_merged_i10s)
|
| 10 | # so a different cadence is a different scenario, never a silent change.
|
| 11 |
|
| 12 | defaults:
|
| 13 | runs: 1
|
| 14 | timeout_seconds: 120
|
| 15 | settle_interval_s: 2
|
| 16 | settle_timeout_s: 300
|
| 17 | admitted_output_chars: 20000
|
| 18 |
|
| 19 | metrics:
|
| 20 | - elapsed_ms
|
| 21 | - settle_elapsed_ms
|
| 22 | - observed_wait_ms
|
| 23 | - poll_iterations
|
| 24 | - poll_interval_s
|
| 25 | - poll_quantization_ms
|
| 26 | - api_metrics
|
| 27 | - tool_call_count
|
| 28 | - estimated_tokens_total
|
| 29 | - branches_reviewed
|
| 30 | - branches_closed
|
| 31 | - merge_order_correct
|
| 32 | - stale_detected_count
|
| 33 | - duplicate_detected_count
|
| 34 | - conflict_detected_count
|
| 35 |
|
| 36 | scenarios:
|
| 37 | - name: platform_capability_probe
|
| 38 | intent: Report whether this runner can talk to the platform at all (CLI present, token usable, repo reachable). Emits skip rows instead of failures when credentials are absent.
|
| 39 | requires_remote: false
|
| 40 | operations:
|
| 41 | - capability.probe
|
| 42 |
|
| 43 | - name: pr_single_anatomy
|
| 44 | intent: "Anatomy of one integration: publish the local task branch, open the integration, wait for it to become visible and mergeable (GitHub computes mergeability asynchronously - that settling is a measured operation), request the merge, wait for main to advance, and sync the merged result into the local checkout."
|
| 45 | requires_remote: true
|
| 46 | operations:
|
| 47 | - branch.publish
|
| 48 | - integration.open
|
| 49 | - integration.visible.settle
|
| 50 | - integration.mergeable.settle
|
| 51 | - integration.merge.request
|
| 52 | - integration.main.settle
|
| 53 | - integration.sync.local
|
| 54 | - integration.total
|
| 55 |
|
| 56 | - name: poll_until_merged_i10s
|
| 57 | intent: "The agent-experienced wait: the same integration anatomy, but every settle polls at a fixed 10-second cadence (the i10s in the name), the interval an agent loop typically uses. Adds integration.merged.settle - polling the integration object itself until it reports merged."
|
| 58 | requires_remote: true
|
| 59 | poll_interval_s: 10
|
| 60 | operations:
|
| 61 | - branch.publish
|
| 62 | - integration.open
|
| 63 | - integration.visible.settle
|
| 64 | - integration.mergeable.settle
|
| 65 | - integration.merge.request
|
| 66 | - integration.merged.settle
|
| 67 | - integration.main.settle
|
| 68 | - integration.sync.local
|
| 69 | - integration.total
|
| 70 |
|
| 71 | # integration_race_nN: the flagship race family. N task branches each go
|
| 72 | # through publish -> PR -> merge, landing serially or as fast as
|
| 73 | # RateLimitBudgeter pacing allows. Per-branch rows reuse the anatomy
|
| 74 | # vocabulary above (integration.checks.green.settle is structural under
|
| 75 | # protection-none and a measured settle under check-instant, where the
|
| 76 | # harness posts the required status itself); race.* rows measure the whole
|
| 77 | # race. race.pacing.injected is the sum of budgeter-injected waits -
|
| 78 | # REPORTED, never subtracted from any operation duration. N is encoded in
|
| 79 | # the scenario name (ADR-0005): a different N is a different scenario,
|
| 80 | # never a silent change.
|
| 81 | - name: integration_race_n1
|
| 82 | intent: "Race family baseline (N=1): one task branch through publish -> PR -> merge with full per-branch anatomy, plus the race-level windows (race.publish.window = first publish start to last publish end; race.integrate.window = first integration.open to last integration.main.settle; race.pacing.injected = budgeter-injected waits, reported never subtracted; race.total = whole-race wall clock)."
|
| 83 | requires_remote: true
|
| 84 | branch_count: 1
|
| 85 | operations:
|
| 86 | - branch.publish
|
| 87 | - integration.open
|
| 88 | - integration.visible.settle
|
| 89 | - integration.mergeable.settle
|
| 90 | - integration.checks.green.settle
|
| 91 | - integration.merge.request
|
| 92 | - integration.merged.settle
|
| 93 | - integration.main.settle
|
| 94 | - race.publish.window
|
| 95 | - race.integrate.window
|
| 96 | - race.pacing.injected
|
| 97 | - race.total
|
| 98 |
|
| 99 | - name: integration_race_n10
|
| 100 | intent: "Race family at N=10: ten task branches each through publish -> PR -> merge, landing serially or as fast as RateLimitBudgeter pacing allows. Same per-branch anatomy vocabulary and race-level rows as integration_race_n1."
|
| 101 | requires_remote: true
|
| 102 | branch_count: 10
|
| 103 | operations:
|
| 104 | - branch.publish
|
| 105 | - integration.open
|
| 106 | - integration.visible.settle
|
| 107 | - integration.mergeable.settle
|
| 108 | - integration.checks.green.settle
|
| 109 | - integration.merge.request
|
| 110 | - integration.merged.settle
|
| 111 | - integration.main.settle
|
| 112 | - race.publish.window
|
| 113 | - race.integrate.window
|
| 114 | - race.pacing.injected
|
| 115 | - race.total
|
| 116 |
|
| 117 | - name: branch_triage_n4
|
| 118 | intent: "Minimal branch/PR triage lane: four seeded branches represent clean, stale, duplicate, and conflicting review outcomes. The oracle checks the action plan and final survivors. Real platform drivers currently emit explicit skip rows; fake-provider exercises the decision logic without network access."
|
| 119 | requires_remote: true
|
| 120 | branch_count: 4
|
| 121 | operations:
|
| 122 | - triage.seed
|
| 123 | - triage.review
|
| 124 | - triage.plan
|
| 125 | - triage.apply
|
| 126 | - triage.oracle
|
| 127 | - triage.total
|
| 128 |
|
| 129 | # branch_fleet_nN: the launch-story branch review workflow. N open branches
|
| 130 | # are seeded, about 30% are real merge conflicts, and the measured path is
|
| 131 | # classify -> plan -> merge safe branches -> leave conflicts open -> sync /
|
| 132 | # observe final main -> oracle. This is intentionally separate from the
|
| 133 | # older fake-only branch_triage_n4 so we can scale to 10/100/1000 without
|
| 134 | # changing that contract.
|
| 135 | - name: branch_fleet_n10
|
| 136 | intent: "Ten open branches: seven clean and three conflicting. Measures the full branch-fleet workflow agents actually perform: classify, plan, merge safe branches, leave conflicts open, sync/observe final main, and verify final survivors."
|
| 137 | requires_remote: true
|
| 138 | branch_count: 10
|
| 139 | operations:
|
| 140 | - fleet.seed
|
| 141 | - fleet.classify
|
| 142 | - fleet.plan
|
| 143 | - fleet.apply
|
| 144 | - fleet.sync
|
| 145 | - fleet.oracle
|
| 146 | - fleet.total
|
| 147 |
|
| 148 | - name: branch_fleet_n100
|
| 149 | intent: "One hundred open branches with the same clean/conflict ratio as branch_fleet_n10, used to expose API/tool-call/tail-latency scaling."
|
| 150 | requires_remote: true
|
| 151 | branch_count: 100
|
| 152 | operations:
|
| 153 | - fleet.seed
|
| 154 | - fleet.classify
|
| 155 | - fleet.plan
|
| 156 | - fleet.apply
|
| 157 | - fleet.sync
|
| 158 | - fleet.oracle
|
| 159 | - fleet.total
|
| 160 |
|
| 161 | - name: branch_fleet_n1000
|
| 162 | intent: "One thousand open branches with the same clean/conflict ratio as branch_fleet_n10, used for the agent-scale launch claim. Fake-provider is always allowed; live GitHub/Oak providers require OAK_BENCH_BRANCH_FLEET_LIVE_LARGE=1 and a disposable benchmark repo."
|
| 163 | requires_remote: true
|
| 164 | branch_count: 1000
|
| 165 | operations:
|
| 166 | - fleet.seed
|
| 167 | - fleet.classify
|
| 168 | - fleet.plan
|
| 169 | - fleet.apply
|
| 170 | - fleet.sync
|
| 171 | - fleet.oracle
|
| 172 | - fleet.total
|