Viewing files at commit 3d84c25e26dbCommit detailsView latest

Browse files
scripts/stale_land.py 1110 lines · 50.9 KB · python
1
#!/usr/bin/env python3
2
"""stale_branch_land_v1: land N agent branches that are k commits behind main.
3
4
Baseline instrument for "landing a stale branch" before any server-side land
5
primitive exists. Each trial creates a fresh repository whose main has moved k
6
commits past the point where N ordinary agent branches (disjoint files) plus
7
one seeded semantic-conflict pair (green alone, red combined) were forked, then
8
releases every agent at once and measures how each arm lands them.
9
10
Arms (``contention_mode`` on every row):
11
12
- ``oak_safe``   documented safe flow against a local oakspace server: shallow
13
                 worker -> ``oak pull`` converge (fresh shallow worker + reapply
14
                 on failure) -> local check on the combined tree -> push ->
15
                 exact-commit CI on the branch tip -> main-unchanged recheck ->
16
                 hosted merge -> verify.
17
- ``oak_naive``  ``oak merge`` on the stale branch as-is (server 3-way; the CI
18
                 gate only knows the branch tip).
19
- ``git_competent`` bare origin + clone per agent over file transport; fetch ->
20
                 rebase -> CI on the combined tree -> push; non-fast-forward
21
                 rejection retries the loop.
22
- ``git_fifo``   the same flow serialized through one FIFO lock (a merge queue
23
                 of depth 1); no push races.
24
25
CI is a real check script (``python3 ci/check.py``) plus a fixed, labeled
26
``--ci-added-seconds`` delay. Oak arms run it through oakspace's real dispatch
27
+ merge gate with ``oakbench.ci_stub`` as the loopback execution backend; Git
28
arms run the same script as a local subprocess with the same delay.
29
30
Independent oracles (never the agent's own report):
31
- every main head created during the trial is materialized (Oak: the server's
32
  token-scoped git export of that exact commit; Git: ``git archive`` from the
33
  bare origin) and checked -> ``red_main_events``;
34
- the final main tree must equal base + k mover files + all N agent patches +
35
  exactly one member of the pair -> ``intended_changes_exact``;
36
- ``combined_tree_tested_before_land``: the landed main commit's tree digest
37
  was among the tree digests this agent checked (locally or in CI) before its
38
  landing call.
39
40
Rows use the ``contention`` lane contract. Null means unmeasured (ADR-0002).
41
Scenario/operation names are immutable identity (ADR-0005).
42
"""
43
44
from __future__ import annotations
45
46
import argparse
47
import hashlib
48
import hmac
49
import json
50
import os
51
import platform
52
import random
53
import re
54
import shutil
55
import statistics
56
import string
57
import subprocess
58
import sys
59
import tempfile
60
import threading
61
import time
62
from dataclasses import dataclass, field
63
from pathlib import Path
64
from typing import Any, Callable, Optional
65
66
sys.path.insert(0, str(Path(__file__).resolve().parent))
67
68
from oakbench.ci_stub import CiStub, read_ledger, tree_digest  # noqa: E402
69
from oakbench.results import ResultsStore  # noqa: E402
70
from oakbench.runlock import measurement_lock  # noqa: E402
71
from oakbench.subjects import binary_sha256  # noqa: E402
72
73
SCENARIO = "stale_branch_land_v1"
74
SCENARIO_VERSION = 1
75
FIXTURE_VERSION = "stale_land_fixture_v1"
76
PROFILE = "stale-land"
77
ARMS = ("oak_safe", "oak_naive", "git_competent", "git_fifo")
78
OAK_ARMS = ("oak_safe", "oak_naive")
79
MAX_ATTEMPTS = 8
80
CI_WAIT_TIMEOUT_S = 300
81
PAIR = ("pair_rename", "pair_consumer")
82
CI_TRIGGER_NOTE = (
83
    "explicit exact-commit `oak ci trigger` after each push: the local server's publication-"
84
    "triggered path reads workflows with local_chunks=None (oakspace ci.rs:2687), so push/"
85
    "branch.created runs conclude 'no readable blob' on a local-chunk server; hosted (R2) "
86
    "auto-triggers. tool_call_count_hosted_equivalent subtracts these trigger calls."
87
)
88
89
# ---------------------------------------------------------------------------
90
# Fixture (pure; unit tested)
91
# ---------------------------------------------------------------------------
92
93
CHECK_PY = '''import importlib
94
import pathlib
95
import sys
96
97
sys.dont_write_bytecode = True
98
ROOT = pathlib.Path(__file__).resolve().parent.parent
99
sys.path.insert(0, str(ROOT))
100
failures = 0
101
for test in sorted((ROOT / "tests").glob("test_*.py")):
102
    module = importlib.import_module("tests." + test.stem)
103
    for name in sorted(dir(module)):
104
        if name.startswith("test_"):
105
            try:
106
                getattr(module, name)()
107
            except Exception as exc:  # noqa: BLE001
108
                failures += 1
109
                print("FAIL %s::%s: %r" % (test.name, name, exc))
110
print("ok" if not failures else "%d failed" % failures)
111
sys.exit(1 if failures else 0)
112
'''
113
114
WORKFLOW_YML = """on: [push, branch.created, merge, manual]
115
jobs:
116
  check:
117
    steps:
118
      - name: check
119
        run: PYTHONDONTWRITEBYTECODE=1 python3 ci/check.py
120
"""
121
122
123
def base_files() -> dict[str, str]:
124
    return {
125
        ".oak/workflows/ci.yml": WORKFLOW_YML,
126
        "ci/check.py": CHECK_PY,
127
        "pkg/__init__.py": "",
128
        "pkg/config.py": "LIMIT = 10\n",
129
        "pkg/core.py": "from pkg import config\n\n\ndef scale(x):\n    return x * config.LIMIT\n",
130
        "tests/__init__.py": "",
131
        "tests/test_core.py": "from pkg.core import scale\n\n\ndef test_scale():\n    assert scale(2) == 20\n",
132
        "README.md": f"fixture {FIXTURE_VERSION}\n",
133
    }
134
135
136
def agent_names(n: int) -> list[str]:
137
    return [f"agent{i:02d}" for i in range(n)] + list(PAIR)
138
139
140
def agent_patch(name: str) -> dict[str, str]:
141
    """Files each agent writes. Ordinary agents and the pair touch disjoint paths."""
142
    if name == "pair_rename":
143
        return {
144
            "pkg/config.py": "MAX_LIMIT = 10\n",
145
            "pkg/core.py": "from pkg import config\n\n\ndef scale(x):\n    return x * config.MAX_LIMIT\n",
146
        }
147
    if name == "pair_consumer":
148
        return {
149
            "pkg/feature_b.py": "from pkg import config\n\n\ndef limit():\n    return config.LIMIT\n",
150
            "tests/test_feature_b.py": "from pkg.feature_b import limit\n\n\ndef test_limit():\n    assert limit() == 10\n",
151
        }
152
    return {
153
        f"pkg/{name}.py": f"def value():\n    return {name!r}\n",
154
        f"tests/test_{name}.py": f"from pkg.{name} import value\n\n\ndef test_value():\n    assert value() == {name!r}\n",
155
    }
156
157
158
def mover_file(j: int) -> tuple[str, str]:
159
    return f"hist/h{j:03d}.txt", f"main moved {j}\n"
160
161
162
def expected_final_files(n: int, k: int, pair_winner: str) -> dict[str, str]:
163
    files = dict(base_files())
164
    for j in range(k):
165
        path, content = mover_file(j)
166
        files[path] = content
167
    for name in agent_names(n)[:n]:
168
        files.update(agent_patch(name))
169
    files.update(agent_patch(pair_winner))
170
    return files
171
172
173
def write_files(root: Path, files: dict[str, str]) -> None:
174
    for rel, content in files.items():
175
        path = root / rel
176
        path.parent.mkdir(parents=True, exist_ok=True)
177
        path.write_text(content)
178
179
180
def files_digest(files: dict[str, str]) -> str:
181
    """tree_digest() of a tree that holds exactly ``files``."""
182
    with tempfile.TemporaryDirectory(prefix="sl-digest-") as tmp:
183
        write_files(Path(tmp), files)
184
        return tree_digest(Path(tmp))
185
186
187
def read_tree(root: Path) -> dict[str, str]:
188
    out: dict[str, str] = {}
189
    for path in sorted(root.rglob("*")):
190
        rel = path.relative_to(root).as_posix()
191
        if not path.is_file() or rel.startswith(".git/") or "__pycache__" in rel:
192
            continue
193
        if rel.startswith(".oak/") and not rel.startswith(".oak/workflows/"):
194
            continue
195
        out[rel] = path.read_text(errors="replace")
196
    return out
197
198
199
def run_check(root: Path) -> tuple[bool, str]:
200
    env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
201
    proc = subprocess.run(
202
        [sys.executable, "ci/check.py"], cwd=root, env=env, capture_output=True, text=True, timeout=120
203
    )
204
    return proc.returncode == 0, (proc.stdout + proc.stderr)[-400:]
205
206
207
def classify_final(files: dict[str, str], n: int, k: int) -> dict[str, Any]:
208
    """Final-state oracle: exactly the intended changes, one pair member."""
209
    rename_in = files.get("pkg/config.py") == agent_patch("pair_rename")["pkg/config.py"]
210
    consumer_in = "pkg/feature_b.py" in files
211
    winners = [name for name, present in (("pair_rename", rename_in), ("pair_consumer", consumer_in)) if present]
212
    missing_agents = [
213
        name for name in agent_names(n)[:n] if any(files.get(p) != c for p, c in agent_patch(name).items())
214
    ]
215
    exact = False
216
    if len(winners) == 1:
217
        exact = files == expected_final_files(n, k, winners[0])
218
    return {
219
        "pair_landed": winners,
220
        "both_pair_landed": len(winners) == 2,
221
        "missing_agents": missing_agents,
222
        "intended_changes_exact": exact,
223
    }
224
225
226
# ---------------------------------------------------------------------------
227
# Agent tool-call accounting
228
# ---------------------------------------------------------------------------
229
230
231
@dataclass
232
class AgentRun:
233
    name: str
234
    arm: str
235
    tool_calls: int = 0
236
    ci_trigger_calls: int = 0
237
    output_bytes: int = 0
238
    attempts: int = 0
239
    ci_runs_local: int = 0
240
    outcome: str = "pending"
241
    error: Optional[str] = None
242
    landed_commit: Optional[str] = None
243
    landed_at: Optional[float] = None
244
    finished_at: Optional[float] = None
245
    tested_digests: list[str] = field(default_factory=list)
246
    local_checks: int = 0
247
    converge_path: list[str] = field(default_factory=list)
248
    trace: list[dict[str, Any]] = field(default_factory=list)
249
    branch: Optional[str] = None
250
    abandoned_branches: list[str] = field(default_factory=list)
251
    rate_limited: int = 0
252
    conflict_retries: int = 0
253
254
    def call(self, argv: list[str], cwd: Path, env: dict[str, str], timeout: float = 600) -> subprocess.CompletedProcess[str]:
255
        start = time.perf_counter()
256
        try:
257
            proc = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, text=True, timeout=timeout)
258
        except subprocess.TimeoutExpired as exc:
259
            proc = subprocess.CompletedProcess(argv, 124, exc.stdout or "", (exc.stderr or "") + "\n[timeout]")
260
        self.tool_calls += 1
261
        out = len((proc.stdout or "").encode()) + len((proc.stderr or "").encode())
262
        self.output_bytes += out
263
        self.trace.append(
264
            {
265
                "argv": [Path(argv[0]).name] + argv[1:],
266
                "rc": proc.returncode,
267
                "ms": round((time.perf_counter() - start) * 1000, 1),
268
                "bytes": out,
269
                "tail": ((proc.stdout or "") + (proc.stderr or ""))[-300:],
270
            }
271
        )
272
        return proc
273
274
    def call_retrying(
275
        self, argv: list[str], cwd: Path, env: dict[str, str], timeout: float = 600, retry_conflicts: int = 0
276
    ) -> subprocess.CompletedProcess[str]:
277
        """A competent agent honours HTTP 429 retry-after and, where the caller
278
        allows it, explicitly retryable 409s. Every retry is a tool call."""
279
        conflicts = 0
280
        for _ in range(30):
281
            proc = self.call(argv, cwd, env, timeout)
282
            wait = retry_after(proc)
283
            if wait is not None:
284
                self.rate_limited += 1
285
                time.sleep(wait)
286
                continue
287
            if proc.returncode != 0 and conflicts < retry_conflicts and retryable_conflict(proc):
288
                conflicts += 1
289
                self.conflict_retries += 1
290
                time.sleep(0.5)
291
                continue
292
            return proc
293
        return proc
294
295
    def local_check(self, root: Path, added_seconds: float = 0.0) -> bool:
296
        """One tool call: run the check script on the working tree."""
297
        start = time.perf_counter()
298
        self.tested_digests.append(tree_digest(root))
299
        ok, tail = run_check(root)
300
        if added_seconds > 0:
301
            time.sleep(added_seconds)
302
        self.tool_calls += 1
303
        self.local_checks += 1
304
        self.output_bytes += len(tail.encode())
305
        self.trace.append({"argv": ["check"], "rc": 0 if ok else 1, "ms": round((time.perf_counter() - start) * 1000, 1), "bytes": len(tail.encode()), "tail": tail})
306
        return ok
307
308
    def apply_patch(self, root: Path, files: dict[str, str]) -> None:
309
        """One tool call: the agent re-applies its intended edit (fallback path)."""
310
        write_files(root, files)
311
        self.tool_calls += 1
312
        self.trace.append({"argv": ["apply_patch"], "rc": 0, "ms": 0.0, "bytes": 0, "tail": ""})
313
314
315
def last_json(text: str) -> Optional[dict[str, Any]]:
316
    """Last JSON object in stdout (some clients print human lines first, fb-479)."""
317
    for line in reversed((text or "").strip().splitlines()):
318
        line = line.strip()
319
        if line.startswith("{"):
320
            try:
321
                value = json.loads(line)
322
            except ValueError:
323
                continue
324
            if isinstance(value, dict):
325
                return value
326
    try:
327
        value = json.loads(text)
328
        return value if isinstance(value, dict) else None
329
    except ValueError:
330
        return None
331
332
333
# ---------------------------------------------------------------------------
334
# Oak arms (local oakspace server)
335
# ---------------------------------------------------------------------------
336
337
338
@dataclass
339
class OakServer:
340
    url: str
341
    token: str
342
    secret: str
343
    owner: str
344
    psql: list[str]
345
    home: Path
346
347
    def env(self) -> dict[str, str]:
348
        env = dict(os.environ)
349
        env.update(
350
            {
351
                "HOME": str(self.home),
352
                "OAK_API_KEY": self.token,
353
                "OAK_URL": self.url,
354
                "OAK_REMOTE": self.url,
355
                "OAK_AUTHOR": self.owner,
356
                "OAK_TRUSTED_REMOTES": self.url,
357
                "PYTHONDONTWRITEBYTECODE": "1",
358
                "NO_COLOR": "1",
359
            }
360
        )
361
        return env
362
363
    def sql(self, query: str) -> list[list[str]]:
364
        proc = subprocess.run(self.psql + ["-qAt", "-F", "\t", "-c", query], capture_output=True, text=True, timeout=60)
365
        if proc.returncode != 0:
366
            raise RuntimeError(f"psql failed: {proc.stderr.strip()[:300]}")
367
        return [line.split("\t") for line in proc.stdout.splitlines() if line.strip()]
368
369
    def repo_id(self, repo: str) -> int:
370
        rows = self.sql(f"select id from repos where name='{repo}'")
371
        return int(rows[0][0])
372
373
    def main_heads(self, repo_id: int) -> list[tuple[int, str]]:
374
        rows = self.sql(
375
            f"select transition_generation, head_hash from publication_events where repo_id={repo_id} "
376
            "and branch_name='main' order by transition_generation"
377
        )
378
        return [(int(gen), head) for gen, head in rows]
379
380
    def fetch_token(self, repo_id: int, commit: str) -> str:
381
        payload = f"{repo_id}.{commit}.{int(time.time()) + 900}"
382
        return payload + "." + hmac.new(self.secret.encode(), payload.encode(), hashlib.sha256).hexdigest()
383
384
    def materialize(self, repo_id: int, commit: str, dest: Path) -> Path:
385
        env = dict(os.environ)
386
        env.update(
387
            {
388
                "GIT_TERMINAL_PROMPT": "0",
389
                "GIT_CONFIG_COUNT": "1",
390
                "GIT_CONFIG_KEY_0": "http.extraHeader",
391
                "GIT_CONFIG_VALUE_0": f"x-oak-ci-fetch-token: {self.fetch_token(repo_id, commit)}",
392
            }
393
        )
394
        shutil.rmtree(dest, ignore_errors=True)
395
        proc = subprocess.run(
396
            ["git", "clone", "-q", "--depth", "1", "--no-tags", f"{self.url}/api/ci/fetch/{repo_id}", str(dest)],
397
            env=env, capture_output=True, text=True, timeout=120,
398
        )
399
        if proc.returncode != 0:
400
            raise RuntimeError(f"materialize {commit[:12]} failed: {proc.stderr.strip()[:300]}")
401
        return dest
402
403
404
RETRY_AFTER_RE = re.compile(r"429 Too Many Requests.*?retry after (\d+) second")
405
406
407
def retry_after(proc: subprocess.CompletedProcess[str]) -> Optional[float]:
408
    """Seconds the server asked us to wait (HTTP 429), else None."""
409
    match = RETRY_AFTER_RE.search((proc.stderr or "") + (proc.stdout or ""))
410
    return float(match.group(1)) if match else None
411
412
413
def retryable_conflict(proc: subprocess.CompletedProcess[str]) -> bool:
414
    """A conclusive server refusal that explicitly says it is safe to retry.
415
416
    Oak prints e.g. ``HTTP 409 Conflict: a verified blob mapping changed before
417
    branch publication; retry`` (concurrent pushes) or a ``main_moved`` /
418
    ``branch_moved`` code with ``retryable: true`` (concurrent merges). Nothing
419
    was written, so retrying is not replaying an ambiguous mutation.
420
    """
421
    text = (proc.stderr or "") + (proc.stdout or "")
422
    return proc.returncode != 0 and "409" in text and ("; retry" in text or '"retryable":true' in text or "retryable: true" in text)
423
424
425
def oak_run(oak: Path, args: list[str], cwd: Path, env: dict[str, str], timeout: float = 300) -> subprocess.CompletedProcess[str]:
426
    """Untimed setup call; honours the server's 429 retry-after hint."""
427
    for _ in range(30):
428
        proc = subprocess.run([str(oak)] + args, cwd=cwd, env=env, capture_output=True, text=True, timeout=timeout)
429
        wait = retry_after(proc)
430
        if wait is None and retryable_conflict(proc):
431
            wait = 0.5
432
        if proc.returncode == 0 or wait is None:
433
            return proc
434
        time.sleep(wait)
435
    return proc
436
437
438
def must(proc: subprocess.CompletedProcess[str], what: str) -> subprocess.CompletedProcess[str]:
439
    if proc.returncode != 0:
440
        raise RuntimeError(f"setup step failed ({what}): rc={proc.returncode} {(proc.stderr or proc.stdout).strip()[-300:]}")
441
    return proc
442
443
444
def oak_head(oak: Path, cwd: Path, env: dict[str, str]) -> str:
445
    return must(oak_run(oak, ["hash"], cwd, env), "hash").stdout.strip()
446
447
448
def oak_branch(oak: Path, cwd: Path, env: dict[str, str]) -> str:
449
    return json.loads(must(oak_run(oak, ["status", "--json"], cwd, env), "status").stdout)["branch"]
450
451
452
def oak_trigger_and_wait(agent: Optional[AgentRun], oak: Path, cwd: Path, env: dict[str, str], key: str) -> tuple[bool, Optional[str]]:
453
    """Exact-commit CI: trigger on the current head and wait for its conclusion."""
454
    call = agent.call_retrying if agent else (lambda argv, cwd, env, timeout=600: oak_run(Path(argv[0]), argv[1:], cwd, env, timeout))
455
    head = oak_head(oak, cwd, env) if agent is None else None
456
    if agent is not None:
457
        proc = call([str(oak), "hash"], cwd, env)
458
        head = proc.stdout.strip()
459
    trig = call([str(oak), "ci", "trigger", "--expected-commit", head, "--idempotency-key", key, "--json"], cwd, env)
460
    if agent is not None:
461
        agent.ci_trigger_calls += 1
462
    doc = last_json(trig.stdout) or {}
463
    run_ids = [str(r) for r in doc.get("run_ids") or []]
464
    if trig.returncode != 0 or not run_ids:
465
        return False, f"ci trigger failed: {(trig.stderr or trig.stdout).strip()[-200:]}"
466
    wait = call([str(oak), "ci", "wait", *run_ids, "--commit", head, "--timeout", str(CI_WAIT_TIMEOUT_S), "--json"], cwd, env, timeout=CI_WAIT_TIMEOUT_S + 30)
467
    wdoc = last_json(wait.stdout) or {}
468
    text = json.dumps(wdoc)
469
    ok = wait.returncode == 0 and '"failure"' not in text and '"cancelled"' not in text and '"timed_out"' not in text
470
    return ok, None if ok else f"ci not green: rc={wait.returncode} {text[:200]}"
471
472
473
@dataclass
474
class OakTrial:
475
    server: OakServer
476
    oak: Path
477
    setup_oak: Path
478
    root: Path
479
    repo: str
480
    n: int
481
    k: int
482
    arm: str
483
    stub: CiStub
484
    repo_id: int = 0
485
    workers: dict[str, Path] = field(default_factory=dict)
486
    branches: dict[str, str] = field(default_factory=dict)
487
    setup_ledger: Path = Path()
488
    trial_ledger: Path = Path()
489
    base_generation: int = 0
490
491
    def setup(self) -> dict[str, Any]:
492
        env = self.server.env()
493
        t0 = time.perf_counter()
494
        seed = self.root / "seed"
495
        seed.mkdir(parents=True)
496
        must(oak_run(self.setup_oak, ["init"], seed, env), "init")
497
        write_files(seed, base_files())
498
        must(oak_run(self.setup_oak, ["commit"], seed, env), "seed commit")
499
        must(oak_run(self.setup_oak, ["push", "-r", self.server.url, "--repo", f"{self.server.owner}/{self.repo}"], seed, env), "seed push")
500
        # Setup-only: bypass the gate (the auto-triggered run cannot read the workflow locally).
501
        must(oak_run(self.setup_oak, ["merge", "--force", "--json"], seed, env), "seed merge")
502
        self.repo_id = self.server.repo_id(self.repo)
503
        self.stub.ledger = self.setup_ledger
504
505
        errors: list[str] = []
506
507
        def prep(name: str) -> None:
508
            try:
509
                wdir = self.root / f"w-{name}"
510
                must(oak_run(self.oak, ["clone", "--shallow", f"{self.server.owner}/{self.repo}", str(wdir)], self.root, env), f"clone {name}")
511
                write_files(wdir, agent_patch(name))
512
                must(oak_run(self.oak, ["commit"], wdir, env), f"commit {name}")
513
                must(oak_run(self.oak, ["push"], wdir, env), f"push {name}")
514
                ok, err = oak_trigger_and_wait(None, self.oak, wdir, env, f"setup-{name}")
515
                if not ok:
516
                    raise RuntimeError(f"setup CI for {name}: {err}")
517
                self.workers[name] = wdir
518
                self.branches[name] = oak_branch(self.oak, wdir, env)
519
            except Exception as exc:  # noqa: BLE001
520
                errors.append(f"{name}: {exc}")
521
522
        threads = [threading.Thread(target=prep, args=(name,)) for name in agent_names(self.n)]
523
        for thread in threads:
524
            thread.start()
525
        for thread in threads:
526
            thread.join()
527
        if errors:
528
            raise RuntimeError("; ".join(errors))
529
530
        mover = self.root / "mover"
531
        must(oak_run(self.setup_oak, ["clone", "--shallow", f"{self.server.owner}/{self.repo}", str(mover)], self.root, env), "mover clone")
532
        for j in range(self.k):
533
            path, content = mover_file(j)
534
            write_files(mover, {path: content})
535
            must(oak_run(self.setup_oak, ["commit"], mover, env), "mover commit")
536
            must(oak_run(self.setup_oak, ["push"], mover, env), "mover push")
537
            must(oak_run(self.setup_oak, ["merge", "--force", "--json"], mover, env), "mover merge")
538
        heads = self.server.main_heads(self.repo_id)
539
        self.base_generation = heads[-1][0]
540
        self.stub.ledger = self.trial_ledger
541
        return {"setup_ms": (time.perf_counter() - t0) * 1000, "main_head_before": heads[-1][1], "main_commits_before": len(heads)}
542
543
    # -- timed flows -----------------------------------------------------------
544
    def main_head(self, agent: AgentRun, cwd: Path, env: dict[str, str]) -> Optional[str]:
545
        proc = agent.call_retrying([str(self.oak), "branch", "show", "main", "--remote", "--json"], cwd, env)
546
        doc = last_json(proc.stdout) or {}
547
        return doc.get("head")
548
549
    def flow_naive(self, agent: AgentRun) -> None:
550
        env = self.server.env()
551
        wdir = self.workers[agent.name]
552
        agent.attempts = 1
553
        proc = agent.call_retrying([str(self.oak), "merge", "--json"], wdir, env, retry_conflicts=3)
554
        doc = last_json(proc.stdout) or {}
555
        if proc.returncode == 0 and doc.get("merged"):
556
            agent.landed_commit = doc.get("commit_hash")
557
            agent.landed_at = time.perf_counter()
558
            agent.outcome = "landed"
559
        else:
560
            agent.outcome = "failed"
561
            agent.error = f"merge rc={proc.returncode} {(proc.stderr or proc.stdout).strip()[-200:]}"
562
563
    def flow_safe(self, agent: AgentRun) -> None:
564
        env = self.server.env()
565
        wdir = self.workers[agent.name]
566
        while agent.attempts < MAX_ATTEMPTS:
567
            agent.attempts += 1
568
            seen_main = self.main_head(agent, wdir, env)
569
            pull = agent.call_retrying([str(self.oak), "pull"], wdir, env)
570
            if pull.returncode != 0:
571
                # Documented fallback: fresh shallow worker from current main + reapply.
572
                agent.converge_path.append("fresh_reapply")
573
                if agent.branch:
574
                    agent.abandoned_branches.append(agent.branch)
575
                fresh = self.root / f"w-{agent.name}-fresh{agent.attempts}"
576
                clone = agent.call_retrying([str(self.oak), "clone", "--shallow", f"{self.server.owner}/{self.repo}", str(fresh)], self.root, env)
577
                if clone.returncode != 0:
578
                    agent.outcome, agent.error = "failed", f"fresh clone failed: {clone.stderr.strip()[-200:]}"
579
                    return
580
                seen_main = self.main_head(agent, fresh, env)
581
                wdir = fresh
582
                agent.apply_patch(wdir, agent_patch(agent.name))
583
                commit = agent.call_retrying([str(self.oak), "commit"], wdir, env)
584
                if commit.returncode != 0:
585
                    agent.outcome, agent.error = "failed", f"commit failed: {commit.stderr.strip()[-200:]}"
586
                    return
587
            else:
588
                agent.converge_path.append("pull")
589
            if not agent.local_check(wdir):
590
                agent.outcome = "refused_red_combined"
591
                return
592
            push = agent.call_retrying([str(self.oak), "push", "--json"], wdir, env, retry_conflicts=3)
593
            if push.returncode != 0:
594
                agent.outcome, agent.error = "failed", f"push failed: {(push.stderr or push.stdout).strip()[-200:]}"
595
                return
596
            agent.branch = (last_json(push.stdout) or {}).get("branch") or agent.branch
597
            ok, err = oak_trigger_and_wait(agent, self.oak, wdir, env, f"{agent.name}-a{agent.attempts}")
598
            if not ok:
599
                agent.outcome, agent.error = "refused_ci_red", err
600
                return
601
            now_main = self.main_head(agent, wdir, env)
602
            if seen_main is None or now_main != seen_main:
603
                continue  # main moved since converge: the tested tree is stale; re-converge
604
            merge = agent.call_retrying([str(self.oak), "merge", "--json"], wdir, env)
605
            doc = last_json(merge.stdout) or {}
606
            if merge.returncode == 0 and doc.get("merged"):
607
                agent.landed_commit = doc.get("commit_hash")
608
                agent.landed_at = time.perf_counter()
609
                verify = self.main_head(agent, wdir, env)
610
                agent.outcome = "landed"
611
                if verify is None:
612
                    agent.error = "post-merge verify could not read main"
613
                return
614
            text = (merge.stderr or "") + (merge.stdout or "")
615
            if "main_moved" in text or "branch_moved" in text or "409" in text:
616
                continue
617
            agent.outcome, agent.error = "failed", f"merge rc={merge.returncode} {text.strip()[-200:]}"
618
            return
619
        agent.outcome, agent.error = "failed", "attempt budget exhausted"
620
621
    # -- oracles ---------------------------------------------------------------
622
    def oracle(self, agents: list[AgentRun]) -> dict[str, Any]:
623
        heads = [(g, h) for g, h in self.server.main_heads(self.repo_id) if g > self.base_generation]
624
        red = 0
625
        digests: dict[str, str] = {}
626
        final_files: dict[str, str] = {}
627
        for index, (_, head) in enumerate(heads):
628
            tree = self.server.materialize(self.repo_id, head, self.root / f"oracle-{index}")
629
            ok, _ = run_check(tree)
630
            red += 0 if ok else 1
631
            digests[head] = tree_digest(tree)
632
            final_files = read_tree(tree)
633
        if not heads:
634
            tree = self.server.materialize(self.repo_id, self.server.main_heads(self.repo_id)[-1][1], self.root / "oracle-final")
635
            final_files = read_tree(tree)
636
        ci_trial = read_ledger(self.trial_ledger)
637
        ci_setup = read_ledger(self.setup_ledger)
638
        return {"heads": heads, "red": red, "digests": digests, "final_files": final_files, "ci_trial": ci_trial, "ci_setup": ci_setup}
639
640
641
# ---------------------------------------------------------------------------
642
# Git arms (bare origin, file transport)
643
# ---------------------------------------------------------------------------
644
645
646
def git(args: list[str], cwd: Path, env: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess[str]:
647
    return subprocess.run(["git"] + args, cwd=cwd, env=env, capture_output=True, text=True, timeout=300)
648
649
650
def git_env() -> dict[str, str]:
651
    env = dict(os.environ)
652
    env.update(
653
        {
654
            "GIT_AUTHOR_NAME": "bench", "GIT_AUTHOR_EMAIL": "bench@localhost", "GIT_COMMITTER_NAME": "bench",
655
            "GIT_COMMITTER_EMAIL": "bench@localhost", "GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0",
656
            "PYTHONDONTWRITEBYTECODE": "1", "HOME": env.get("OAK_BENCH_GIT_HOME", tempfile.gettempdir()),
657
        }
658
    )
659
    return env
660
661
662
@dataclass
663
class GitTrial:
664
    root: Path
665
    n: int
666
    k: int
667
    arm: str
668
    ci_added_seconds: float
669
    workers: dict[str, Path] = field(default_factory=dict)
670
    base_main: str = ""
671
    queue: threading.Lock = field(default_factory=threading.Lock)
672
673
    @property
674
    def origin(self) -> Path:
675
        return self.root / "origin.git"
676
677
    def setup(self) -> dict[str, Any]:
678
        env = git_env()
679
        t0 = time.perf_counter()
680
        must(git(["init", "-q", "--bare", "-b", "main", str(self.origin)], self.root, env), "bare init")
681
        seed = self.root / "seed"
682
        must(git(["clone", "-q", str(self.origin), str(seed)], self.root, env), "seed clone")
683
        write_files(seed, base_files())
684
        must(git(["checkout", "-q", "-b", "main"], seed, env), "seed branch")
685
        must(git(["add", "-A"], seed, env), "seed add")
686
        must(git(["commit", "-q", "-m", "base"], seed, env), "seed commit")
687
        must(git(["push", "-q", "origin", "main"], seed, env), "seed push")
688
        for name in agent_names(self.n):
689
            wdir = self.root / f"w-{name}"
690
            must(git(["clone", "-q", str(self.origin), str(wdir)], self.root, env), f"clone {name}")
691
            must(git(["checkout", "-q", "-b", name], wdir, env), f"branch {name}")
692
            write_files(wdir, agent_patch(name))
693
            must(git(["add", "-A"], wdir, env), f"add {name}")
694
            must(git(["commit", "-q", "-m", name], wdir, env), f"commit {name}")
695
            self.workers[name] = wdir
696
        for j in range(self.k):
697
            path, content = mover_file(j)
698
            write_files(seed, {path: content})
699
            must(git(["add", "-A"], seed, env), "mover add")
700
            must(git(["commit", "-q", "-m", f"move {j}"], seed, env), "mover commit")
701
        must(git(["push", "-q", "origin", "main"], seed, env), "mover push")
702
        self.base_main = must(git(["rev-parse", "main"], self.origin, env), "rev-parse").stdout.strip()
703
        return {"setup_ms": (time.perf_counter() - t0) * 1000, "main_head_before": self.base_main, "main_commits_before": self.k + 1}
704
705
    def flow(self, agent: AgentRun) -> None:
706
        env = git_env()
707
        wdir = self.workers[agent.name]
708
        g = lambda args: agent.call(["git"] + args, wdir, env)  # noqa: E731
709
        while agent.attempts < MAX_ATTEMPTS * 3:
710
            ctx = self.queue if self.arm == "git_fifo" else _NullLock()
711
            with ctx:
712
                agent.attempts += 1
713
                if g(["fetch", "-q", "origin"]).returncode != 0:
714
                    continue
715
                if g(["rebase", "-q", "origin/main"]).returncode != 0:
716
                    g(["rebase", "--abort"])
717
                    agent.outcome, agent.error = "failed", "rebase conflict"
718
                    return
719
                agent.ci_runs_local += 1
720
                if not agent.local_check(wdir, added_seconds=self.ci_added_seconds):
721
                    agent.outcome = "refused_red_combined"
722
                    return
723
                push = g(["push", "-q", "origin", "HEAD:main"])
724
                if push.returncode == 0:
725
                    agent.landed_commit = must(git(["rev-parse", "HEAD"], wdir, env), "rev-parse").stdout.strip()
726
                    agent.landed_at = time.perf_counter()
727
                    agent.outcome = "landed"
728
                    return
729
                if "rejected" not in (push.stderr or "") and "fetch first" not in (push.stderr or ""):
730
                    agent.outcome, agent.error = "failed", f"push failed: {push.stderr.strip()[-200:]}"
731
                    return
732
        agent.outcome, agent.error = "failed", "attempt budget exhausted"
733
734
    def oracle(self, agents: list[AgentRun]) -> dict[str, Any]:
735
        env = git_env()
736
        revs = must(git(["rev-list", "--reverse", f"{self.base_main}..main"], self.origin, env), "rev-list").stdout.split()
737
        red = 0
738
        digests: dict[str, str] = {}
739
        final_files: dict[str, str] = {}
740
        targets = revs or [self.base_main]
741
        for index, rev in enumerate(targets):
742
            dest = self.root / f"oracle-{index}"
743
            dest.mkdir()
744
            archive = subprocess.run(["git", "archive", rev], cwd=self.origin, capture_output=True, timeout=120)
745
            subprocess.run(["tar", "-x", "-C", str(dest)], input=archive.stdout, check=True, timeout=120)
746
            if revs:
747
                ok, _ = run_check(dest)
748
                red += 0 if ok else 1
749
                digests[rev] = tree_digest(dest)
750
            final_files = read_tree(dest)
751
        return {"heads": [(i, r) for i, r in enumerate(revs)], "red": red, "digests": digests, "final_files": final_files, "ci_trial": [], "ci_setup": []}
752
753
754
class _NullLock:
755
    def __enter__(self) -> None:
756
        return None
757
758
    def __exit__(self, *exc: Any) -> None:
759
        return None
760
761
762
# ---------------------------------------------------------------------------
763
# Trial driver + rows
764
# ---------------------------------------------------------------------------
765
766
767
def percentile(values: list[float], p: float) -> Optional[float]:
768
    if not values:
769
        return None
770
    ordered = sorted(values)
771
    index = min(len(ordered) - 1, max(0, int(round(p / 100 * (len(ordered) - 1)))))
772
    return ordered[index]
773
774
775
def run_agents(agents: list[AgentRun], flow: Callable[[AgentRun], None]) -> tuple[float, float]:
776
    barrier = threading.Barrier(len(agents) + 1)
777
778
    def body(agent: AgentRun) -> None:
779
        barrier.wait()
780
        try:
781
            flow(agent)
782
        except Exception as exc:  # noqa: BLE001 - recorded as a failed agent, never hidden
783
            agent.outcome, agent.error = "failed", f"harness exception: {exc}"
784
        agent.finished_at = time.perf_counter()
785
786
    threads = [threading.Thread(target=body, args=(agent,)) for agent in agents]
787
    for thread in threads:
788
        thread.start()
789
    barrier.wait()
790
    start = time.perf_counter()
791
    for thread in threads:
792
        thread.join()
793
    return start, time.perf_counter()
794
795
796
def build_rows(
797
    *,
798
    meta: dict[str, Any],
799
    arm: str,
800
    n: int,
801
    k: int,
802
    run_index: int,
803
    agents: list[AgentRun],
804
    start: float,
805
    end: float,
806
    setup: dict[str, Any],
807
    oracle: dict[str, Any],
808
    subject: str,
809
    subject_kind: str,
810
    binary_hash: Optional[str],
811
) -> list[dict[str, Any]]:
812
    is_oak = arm in OAK_ARMS
813
    ci_by_commit: dict[str, list[dict[str, Any]]] = {}
814
    for record in oracle["ci_trial"]:
815
        ci_by_commit.setdefault(record.get("ref") or "", []).append(record)
816
    setup_by_ref: dict[str, list[dict[str, Any]]] = {}
817
    for record in oracle["ci_setup"]:
818
        setup_by_ref.setdefault(record.get("ref") or "", []).append(record)
819
    common = {
820
        **meta,
821
        "profile": PROFILE,
822
        "scenario": SCENARIO,
823
        "scenario_version": SCENARIO_VERSION,
824
        "fixture_version": FIXTURE_VERSION,
825
        "contention_mode": arm,
826
        "workers": n,
827
        "agents_total": n + len(PAIR),
828
        "behind_k": k,
829
        "run": run_index,
830
        "subject": subject,
831
        "subject_kind": subject_kind,
832
        "binary_sha256": binary_hash,
833
        "transport": "loopback" if is_oak else "local-file",
834
        "hosted_equivalence": "hosted-equivalent-local" if is_oak else "not-applicable-local-git",
835
        "ci_executor": "oakspace_dispatch+oakbench.ci_stub(loopback)" if is_oak else "local_subprocess",
836
        "ci_trigger_mode": CI_TRIGGER_NOTE if is_oak else "local_pre_push",
837
    }
838
    rows: list[dict[str, Any]] = []
839
    classification = classify_final(oracle["final_files"], n, k)
840
    for agent in agents:
841
        refs = [agent.branch or ""] + list(agent.abandoned_branches)
842
        ci_runs_trial = [r for ref in refs for r in ci_by_commit.get(ref, [])]
843
        ci_digests = [r.get("tree_digest") for r in ci_runs_trial if r.get("conclusion") == "success"]
844
        if arm == "oak_naive":
845
            ci_digests += [r.get("tree_digest") for r in setup_by_ref.get(agent.branch or "", []) if r.get("conclusion") == "success"]
846
        landed_digest = oracle["digests"].get(agent.landed_commit or "")
847
        tested = None
848
        local_tested = ci_tested = None
849
        if agent.outcome == "landed":
850
            local_tested = landed_digest is not None and landed_digest in agent.tested_digests
851
            ci_tested = landed_digest is not None and landed_digest in ci_digests
852
            tested = bool(local_tested or ci_tested)
853
        ci_runs = len(ci_runs_trial) if is_oak else agent.ci_runs_local
854
        rows.append(
855
            {
856
                **common,
857
                "operation": "land.agent",
858
                "agent": agent.name,
859
                "agent_role": "pair" if agent.name in PAIR else "ordinary",
860
                "elapsed_ms": round(((agent.landed_at or agent.finished_at or end) - start) * 1000, 1),
861
                "time_to_landed_ms": round((agent.landed_at - start) * 1000, 1) if agent.landed_at else None,
862
                "returncode": 0 if agent.outcome in ("landed", "refused_red_combined", "refused_ci_red") else 1,
863
                "outcome": agent.outcome,
864
                "error": agent.error,
865
                "tool_call_count": agent.tool_calls,
866
                "tool_call_count_hosted_equivalent": agent.tool_calls - agent.ci_trigger_calls if is_oak else agent.tool_calls,
867
                "ci_trigger_calls": agent.ci_trigger_calls if is_oak else None,
868
                "output_bytes": agent.output_bytes,
869
                "attempts": agent.attempts,
870
                "rate_limited_retries": agent.rate_limited if is_oak else None,
871
                "conflict_retries": agent.conflict_retries if is_oak else None,
872
                "retries": max(0, agent.attempts - 1),
873
                "ci_runs": ci_runs,
874
                "local_checks": agent.local_checks,
875
                "converge_path": agent.converge_path or None,
876
                "landed_commit": agent.landed_commit,
877
                "combined_tree_tested_before_land": tested,
878
                "combined_tree_ci_tested_before_land": ci_tested,
879
                "combined_tree_locally_tested_before_land": local_tested,
880
                "command": " ; ".join(" ".join(t["argv"][:3]) for t in agent.trace)[:2000],
881
                "trace": agent.trace,
882
            }
883
        )
884
    landed = [a for a in agents if a.outcome == "landed"]
885
    ordinary = [a for a in agents if a.name not in PAIR]
886
    times = [(a.landed_at - start) * 1000 for a in ordinary if a.landed_at]
887
    tested_flags = [r["combined_tree_tested_before_land"] for r in rows if r["outcome"] == "landed"]
888
    total_ci = len(oracle["ci_trial"]) if is_oak else sum(a.ci_runs_local for a in agents)
889
    final_green = None
890
    if oracle["final_files"]:
891
        with tempfile.TemporaryDirectory(prefix="sl-final-") as tmp:
892
            write_files(Path(tmp), oracle["final_files"])
893
            final_green = run_check(Path(tmp))[0]
894
    lost = [a.name for a in ordinary if a.outcome != "landed"]
895
    rows.append(
896
        {
897
            **common,
898
            "operation": "land.trial",
899
            "elapsed_ms": round((end - start) * 1000, 1),
900
            "returncode": 0 if (classification["intended_changes_exact"] and oracle["red"] == 0 and not lost) else 1,
901
            "setup_ms": round(setup["setup_ms"], 1),
902
            "main_commits_before": setup["main_commits_before"],
903
            "main_heads_created": len(oracle["heads"]),
904
            "red_main_events": oracle["red"],
905
            "final_main_green": final_green,
906
            **classification,
907
            "ordinary_landed": len(times),
908
            "ordinary_lost": lost,
909
            "failed_agents": [a.name for a in agents if a.outcome == "failed"],
910
            "retries_total": sum(max(0, a.attempts - 1) for a in agents),
911
            "rate_limited_retries_total": sum(a.rate_limited for a in agents) if is_oak else None,
912
            "conflict_retries_total": sum(a.conflict_retries for a in agents) if is_oak else None,
913
            "tool_call_count": sum(a.tool_calls for a in agents),
914
            "output_bytes": sum(a.output_bytes for a in agents),
915
            "ci_runs_total": total_ci,
916
            "time_to_landed_ms_p50": percentile(times, 50),
917
            "time_to_landed_ms_max": max(times) if times else None,
918
            "combined_tested_fraction": (sum(1 for f in tested_flags if f) / len(tested_flags)) if tested_flags else None,
919
            "landed_total": len(landed),
920
            "human_interventions": 0,
921
            "human_interventions_source": "scripted_harness_has_no_human_step",
922
            "command": arm,
923
        }
924
    )
925
    return rows
926
927
928
def rand_suffix() -> str:
929
    return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(6))
930
931
932
def run_trial(args: argparse.Namespace, arm: str, n: int, k: int, run_index: int, stub: Optional[CiStub], meta: dict[str, Any], subject: tuple[str, str, Optional[Path]]) -> list[dict[str, Any]]:
933
    subject_name, subject_kind, oak_bin = subject
934
    root = Path(tempfile.mkdtemp(prefix=f"sl-{arm}-n{n}-k{k}-r{run_index}-", dir=args.scratch))
935
    agents = [AgentRun(name=name, arm=arm) for name in agent_names(n)]
936
    if arm in OAK_ARMS:
937
        assert stub is not None and oak_bin is not None
938
        server = OakServer(url=args.oak_url, token=Path(args.token_file).read_text().strip(), secret=os.environ[args.secret_env],
939
                           owner=args.owner, psql=args.psql.split(), home=Path(args.oak_home))
940
        trial = OakTrial(server=server, oak=oak_bin, setup_oak=Path(args.setup_oak), root=root,
941
                         repo=f"sl-{arm.replace('_', '')}-{subject_name.replace('_', '')}-n{n}-k{k}-r{run_index}-{rand_suffix()}",
942
                         n=n, k=k, arm=arm, stub=stub, setup_ledger=root / "ci-setup.jsonl", trial_ledger=root / "ci-trial.jsonl")
943
        setup = trial.setup()
944
        for agent in agents:
945
            agent.branch = trial.branches.get(agent.name)
946
        flow = trial.flow_naive if arm == "oak_naive" else trial.flow_safe
947
        with measurement_lock(f"stale_land:{arm}:n{n}:k{k}:r{run_index}") as lock:
948
            start, end = run_agents(agents, flow)
949
            stub.join_runs(timeout=60)
950
        oracle = trial.oracle(agents)
951
        binary_hash = binary_sha256(oak_bin)
952
    else:
953
        trial_g = GitTrial(root=root, n=n, k=k, arm=arm, ci_added_seconds=args.ci_added_seconds)
954
        setup = trial_g.setup()
955
        with measurement_lock(f"stale_land:{arm}:n{n}:k{k}:r{run_index}") as lock:
956
            start, end = run_agents(agents, trial_g.flow)
957
        oracle = trial_g.oracle(agents)
958
        binary_hash = binary_sha256(Path(shutil.which("git") or "git"))
959
    rows = build_rows(meta={**meta, "lock_wait_ms": lock.wait_ms, "lock_enabled": lock.enabled, "trial_root": str(root)}, arm=arm, n=n, k=k,
960
                      run_index=run_index, agents=agents, start=start, end=end, setup=setup, oracle=oracle,
961
                      subject=subject_name, subject_kind=subject_kind, binary_hash=binary_hash)
962
    if not args.keep_trials:
963
        shutil.rmtree(root, ignore_errors=True)
964
    return rows
965
966
967
def summary_text(rows: list[dict[str, Any]]) -> str:
968
    trials = [r for r in rows if r["operation"] == "land.trial"]
969
    groups: dict[tuple[Any, ...], list[dict[str, Any]]] = {}
970
    for row in trials:
971
        groups.setdefault((row["contention_mode"], row["subject"], row["workers"], row["behind_k"], row.get("aa_group")), []).append(row)
972
    lines = [
973
        f"# {SCENARIO} baseline",
974
        "",
975
        f"CI added delay: {trials[0].get('ci_added_seconds') if trials else None} s (labeled). Oak rows: loopback, hosted-equivalent-local. Git rows: local-file.",
976
        "",
977
        "| arm | subject | N | k | trials | median p50 time-to-landed s | median makespan s | CI runs (median) | tool calls (median) | combined-tested % | red-main events | exact final | failures/lost | retries (median) |",
978
        "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|",
979
    ]
980
981
    def med(values: list[Any]) -> Optional[float]:
982
        clean = [v for v in values if v is not None]
983
        return statistics.median(clean) if clean else None
984
985
    def fmt(value: Optional[float], scale: float = 1.0, digits: int = 1) -> str:
986
        return "n/a" if value is None else f"{value / scale:.{digits}f}"
987
988
    for (arm, subject, n, k, aa), group in sorted(groups.items(), key=lambda kv: [str(x) for x in kv[0]]):
989
        tested = [r["combined_tested_fraction"] for r in group if r["combined_tested_fraction"] is not None]
990
        lines.append(
991
            "| {arm}{aa} | {subject} | {n} | {k} | {t} | {p50} | {mk} | {ci} | {tc} | {tested} | {red} | {exact}/{t} | {fail} | {retries} |".format(
992
                arm=arm, aa=f" ({aa})" if aa else "", subject=subject, n=n, k=k, t=len(group),
993
                p50=fmt(med([r["time_to_landed_ms_p50"] for r in group]), 1000),
994
                mk=fmt(med([r["elapsed_ms"] for r in group]), 1000),
995
                ci=fmt(med([r["ci_runs_total"] for r in group]), 1, 0),
996
                tc=fmt(med([r["tool_call_count"] for r in group]), 1, 0),
997
                tested="n/a" if not tested else f"{100 * statistics.mean(tested):.0f}%",
998
                red=sum(r["red_main_events"] for r in group),
999
                exact=sum(1 for r in group if r["intended_changes_exact"]),
1000
                fail=sum(len(r["failed_agents"]) + len(r["ordinary_lost"]) for r in group),
1001
                retries=fmt(med([r["retries_total"] for r in group]), 1, 0),
1002
            )
1003
        )
1004
    return "\n".join(lines) + "\n"
1005
1006
1007
def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:
1008
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
1009
    parser.add_argument("--arms", default=",".join(ARMS))
1010
    parser.add_argument("--agents", default="1,4", help="comma list of N ordinary agents (the pair is added to each)")
1011
    parser.add_argument("--behind", default="1,50", help="comma list of k commits main moved after the fork")
1012
    parser.add_argument("--runs", type=int, default=3)
1013
    parser.add_argument("--aa-group", default=None, help="label for A/A noise-floor batches")
1014
    parser.add_argument("--ci-added-seconds", type=float, default=10.0)
1015
    parser.add_argument("--oak-bin", action="append", default=[], help="NAME=PATH oak client subject (repeatable)")
1016
    parser.add_argument("--setup-oak", default=None, help="oak binary for untimed seed/mover setup")
1017
    parser.add_argument("--oak-url", default="http://127.0.0.1:3811")
1018
    parser.add_argument("--owner", default="bench")
1019
    parser.add_argument("--token-file", default=None)
1020
    parser.add_argument("--oak-home", default=None, help="isolated HOME for oak clients")
1021
    parser.add_argument("--secret-env", default="OAK_CI_SIGNING_SECRET")
1022
    parser.add_argument("--ci-stub-port", type=int, default=3812, help="port the server's OAK_CI_DISPATCH_URL points at")
1023
    parser.add_argument("--psql", default="psql -h 127.0.0.1 -p 55811 -U oak oak")
1024
    parser.add_argument("--server-binary", default=None, help="oakspace server binary (provenance only)")
1025
    parser.add_argument("--server-source", default=None, help="oakspace source identity (provenance only)")
1026
    parser.add_argument("--scratch", default=None)
1027
    parser.add_argument("--results", type=Path, default=Path("results/stale-land"))
1028
    parser.add_argument("--keep-trials", action="store_true")
1029
    parser.add_argument("--dry-run", action="store_true", help="print the trial schedule and exit")
1030
    return parser.parse_args(argv)
1031
1032
1033
def schedule(args: argparse.Namespace) -> list[tuple[str, tuple[str, str, Optional[Path]], int, int, int]]:
1034
    arms = [a for a in args.arms.split(",") if a]
1035
    for arm in arms:
1036
        if arm not in ARMS:
1037
            raise SystemExit(f"unknown arm {arm!r}; known: {', '.join(ARMS)}")
1038
    oak_subjects: list[tuple[str, str, Optional[Path]]] = []
1039
    for spec in args.oak_bin:
1040
        name, _, path = spec.partition("=")
1041
        oak_subjects.append((name, "oak", Path(path)))
1042
    plan = []
1043
    for n in [int(x) for x in args.agents.split(",")]:
1044
        for k in [int(x) for x in args.behind.split(",")]:
1045
            for run_index in range(1, args.runs + 1):
1046
                for arm in arms:
1047
                    subjects = oak_subjects if arm in OAK_ARMS else [("git", "git", None)]
1048
                    for subject in subjects:
1049
                        plan.append((arm, subject, n, k, run_index))
1050
    return plan
1051
1052
1053
def main(argv: Optional[list[str]] = None) -> int:
1054
    args = parse_args(argv)
1055
    plan = schedule(args)
1056
    if args.dry_run:
1057
        for item in plan:
1058
            print(item[0], item[1][0], f"n={item[2]}", f"k={item[3]}", f"run={item[4]}")
1059
        return 0
1060
    needs_oak = any(item[0] in OAK_ARMS for item in plan)
1061
    stub = None
1062
    if needs_oak:
1063
        if not (args.token_file and args.oak_home and args.setup_oak and os.environ.get(args.secret_env)):
1064
            raise SystemExit("oak arms need --token-file, --oak-home, --setup-oak and $" + args.secret_env)
1065
        stub = CiStub(api_url=args.oak_url, secret=os.environ[args.secret_env], ledger=Path(os.devnull),
1066
                      added_seconds=args.ci_added_seconds, port=args.ci_stub_port).start()
1067
    bench_id = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + f"-{SCENARIO}"
1068
    meta = {
1069
        "bench_id": bench_id,
1070
        "ci_added_seconds": args.ci_added_seconds,
1071
        "aa_group": args.aa_group,
1072
        "host": platform.node(),
1073
        "server_binary_sha256": binary_sha256(Path(args.server_binary)) if args.server_binary else None,
1074
        "server_source": args.server_source,
1075
    }
1076
    store = ResultsStore(args.results, lane="contention", filename_suffix="stale-land", flush_partial=True)
1077
    rows: list[dict[str, Any]] = []
1078
    try:
1079
        for arm, subject, n, k, run_index in plan:
1080
            label = f"{arm} {subject[0]} n={n} k={k} run={run_index}"
1081
            print(f"[stale_land] {label} ...", file=sys.stderr, flush=True)
1082
            try:
1083
                trial_rows = run_trial(args, arm, n, k, run_index, stub, meta, subject)
1084
            except Exception as exc:  # noqa: BLE001 - a setup failure is a recorded row, not a silent gap
1085
                trial_rows = [{
1086
                    **meta, "profile": PROFILE, "scenario": SCENARIO, "scenario_version": SCENARIO_VERSION,
1087
                    "contention_mode": arm, "workers": n, "behind_k": k, "operation": "land.trial", "run": run_index,
1088
                    "subject": subject[0], "subject_kind": subject[1], "elapsed_ms": 0.0, "returncode": 1,
1089
                    "error": f"trial setup/oracle failed: {exc}", "command": arm,
1090
                }]
1091
            trial = trial_rows[-1]
1092
            print(
1093
                f"[stale_land] {label}: rc={trial['returncode']} makespan={trial['elapsed_ms'] / 1000:.1f}s "
1094
                f"p50={trial.get('time_to_landed_ms_p50')} ci={trial.get('ci_runs_total')} red={trial.get('red_main_events')} "
1095
                f"exact={trial.get('intended_changes_exact')} tested={trial.get('combined_tested_fraction')} err={trial.get('error')}",
1096
                file=sys.stderr, flush=True,
1097
            )
1098
            store.append_partial(bench_id, trial_rows)
1099
            rows.extend(trial_rows)
1100
    finally:
1101
        if stub is not None:
1102
            stub.stop()
1103
    complete = [r for r in rows if "tool_call_count" in r or r["operation"] == "land.agent"]
1104
    raw, summary = store.write(bench_id, rows, summary_text([r for r in complete if r["operation"] == "land.trial"] or []))
1105
    print(f"[stale_land] rows: {raw}\n[stale_land] summary: {summary}", file=sys.stderr)
1106
    return 0
1107
1108
1109
if __name__ == "__main__":
1110
    raise SystemExit(main())