| 1 | #!/usr/bin/env python3
|
| 2 | """stale_branch_land_v1: land N agent branches that are k commits behind main.
|
| 3 |
|
| 4 | Baseline instrument for "landing a stale branch" before any server-side land
|
| 5 | primitive exists. Each trial creates a fresh repository whose main has moved k
|
| 6 | commits past the point where N ordinary agent branches (disjoint files) plus
|
| 7 | one seeded semantic-conflict pair (green alone, red combined) were forked, then
|
| 8 | releases every agent at once and measures how each arm lands them.
|
| 9 |
|
| 10 | Arms (``contention_mode`` on every row):
|
| 11 |
|
| 12 | - ``oak_safe`` documented safe flow against a local oakspace server: shallow
|
| 13 | worker -> ``oak pull`` converge (fresh shallow worker + reapply
|
| 14 | on failure) -> local check on the combined tree -> push ->
|
| 15 | exact-commit CI on the branch tip -> main-unchanged recheck ->
|
| 16 | hosted merge -> verify.
|
| 17 | - ``oak_naive`` ``oak merge`` on the stale branch as-is (server 3-way; the CI
|
| 18 | gate only knows the branch tip).
|
| 19 | - ``git_competent`` bare origin + clone per agent over file transport; fetch ->
|
| 20 | rebase -> CI on the combined tree -> push; non-fast-forward
|
| 21 | rejection retries the loop.
|
| 22 | - ``git_fifo`` the same flow serialized through one FIFO lock (a merge queue
|
| 23 | of depth 1); no push races.
|
| 24 |
|
| 25 | CI is a real check script (``python3 ci/check.py``) plus a fixed, labeled
|
| 26 | ``--ci-added-seconds`` delay. Oak arms run it through oakspace's real dispatch
|
| 27 | + merge gate with ``oakbench.ci_stub`` as the loopback execution backend; Git
|
| 28 | arms run the same script as a local subprocess with the same delay.
|
| 29 |
|
| 30 | Independent oracles (never the agent's own report):
|
| 31 | - every main head created during the trial is materialized (Oak: the server's
|
| 32 | token-scoped git export of that exact commit; Git: ``git archive`` from the
|
| 33 | bare origin) and checked -> ``red_main_events``;
|
| 34 | - the final main tree must equal base + k mover files + all N agent patches +
|
| 35 | exactly one member of the pair -> ``intended_changes_exact``;
|
| 36 | - ``combined_tree_tested_before_land``: the landed main commit's tree digest
|
| 37 | was among the tree digests this agent checked (locally or in CI) before its
|
| 38 | landing call.
|
| 39 |
|
| 40 | Rows use the ``contention`` lane contract. Null means unmeasured (ADR-0002).
|
| 41 | Scenario/operation names are immutable identity (ADR-0005).
|
| 42 | """
|
| 43 |
|
| 44 | from __future__ import annotations
|
| 45 |
|
| 46 | import argparse
|
| 47 | import hashlib
|
| 48 | import hmac
|
| 49 | import json
|
| 50 | import os
|
| 51 | import platform
|
| 52 | import random
|
| 53 | import re
|
| 54 | import shutil
|
| 55 | import statistics
|
| 56 | import string
|
| 57 | import subprocess
|
| 58 | import sys
|
| 59 | import tempfile
|
| 60 | import threading
|
| 61 | import time
|
| 62 | from dataclasses import dataclass, field
|
| 63 | from pathlib import Path
|
| 64 | from typing import Any, Callable, Optional
|
| 65 |
|
| 66 | sys.path.insert(0, str(Path(__file__).resolve().parent))
|
| 67 |
|
| 68 | from oakbench.ci_stub import CiStub, read_ledger, tree_digest # noqa: E402
|
| 69 | from oakbench.results import ResultsStore # noqa: E402
|
| 70 | from oakbench.runlock import measurement_lock # noqa: E402
|
| 71 | from oakbench.subjects import binary_sha256 # noqa: E402
|
| 72 |
|
| 73 | SCENARIO = "stale_branch_land_v1"
|
| 74 | SCENARIO_VERSION = 1
|
| 75 | FIXTURE_VERSION = "stale_land_fixture_v1"
|
| 76 | PROFILE = "stale-land"
|
| 77 | ARMS = ("oak_safe", "oak_naive", "git_competent", "git_fifo")
|
| 78 | OAK_ARMS = ("oak_safe", "oak_naive")
|
| 79 | MAX_ATTEMPTS = 8
|
| 80 | CI_WAIT_TIMEOUT_S = 300
|
| 81 | PAIR = ("pair_rename", "pair_consumer")
|
| 82 | CI_TRIGGER_NOTE = (
|
| 83 | "explicit exact-commit `oak ci trigger` after each push: the local server's publication-"
|
| 84 | "triggered path reads workflows with local_chunks=None (oakspace ci.rs:2687), so push/"
|
| 85 | "branch.created runs conclude 'no readable blob' on a local-chunk server; hosted (R2) "
|
| 86 | "auto-triggers. tool_call_count_hosted_equivalent subtracts these trigger calls."
|
| 87 | )
|
| 88 |
|
| 89 | # ---------------------------------------------------------------------------
|
| 90 | # Fixture (pure; unit tested)
|
| 91 | # ---------------------------------------------------------------------------
|
| 92 |
|
| 93 | CHECK_PY = '''import importlib
|
| 94 | import pathlib
|
| 95 | import sys
|
| 96 |
|
| 97 | sys.dont_write_bytecode = True
|
| 98 | ROOT = pathlib.Path(__file__).resolve().parent.parent
|
| 99 | sys.path.insert(0, str(ROOT))
|
| 100 | failures = 0
|
| 101 | for test in sorted((ROOT / "tests").glob("test_*.py")):
|
| 102 | module = importlib.import_module("tests." + test.stem)
|
| 103 | for name in sorted(dir(module)):
|
| 104 | if name.startswith("test_"):
|
| 105 | try:
|
| 106 | getattr(module, name)()
|
| 107 | except Exception as exc: # noqa: BLE001
|
| 108 | failures += 1
|
| 109 | print("FAIL %s::%s: %r" % (test.name, name, exc))
|
| 110 | print("ok" if not failures else "%d failed" % failures)
|
| 111 | sys.exit(1 if failures else 0)
|
| 112 | '''
|
| 113 |
|
| 114 | WORKFLOW_YML = """on: [push, branch.created, merge, manual]
|
| 115 | jobs:
|
| 116 | check:
|
| 117 | steps:
|
| 118 | - name: check
|
| 119 | run: PYTHONDONTWRITEBYTECODE=1 python3 ci/check.py
|
| 120 | """
|
| 121 |
|
| 122 |
|
| 123 | def base_files() -> dict[str, str]:
|
| 124 | return {
|
| 125 | ".oak/workflows/ci.yml": WORKFLOW_YML,
|
| 126 | "ci/check.py": CHECK_PY,
|
| 127 | "pkg/__init__.py": "",
|
| 128 | "pkg/config.py": "LIMIT = 10\n",
|
| 129 | "pkg/core.py": "from pkg import config\n\n\ndef scale(x):\n return x * config.LIMIT\n",
|
| 130 | "tests/__init__.py": "",
|
| 131 | "tests/test_core.py": "from pkg.core import scale\n\n\ndef test_scale():\n assert scale(2) == 20\n",
|
| 132 | "README.md": f"fixture {FIXTURE_VERSION}\n",
|
| 133 | }
|
| 134 |
|
| 135 |
|
| 136 | def agent_names(n: int) -> list[str]:
|
| 137 | return [f"agent{i:02d}" for i in range(n)] + list(PAIR)
|
| 138 |
|
| 139 |
|
| 140 | def agent_patch(name: str) -> dict[str, str]:
|
| 141 | """Files each agent writes. Ordinary agents and the pair touch disjoint paths."""
|
| 142 | if name == "pair_rename":
|
| 143 | return {
|
| 144 | "pkg/config.py": "MAX_LIMIT = 10\n",
|
| 145 | "pkg/core.py": "from pkg import config\n\n\ndef scale(x):\n return x * config.MAX_LIMIT\n",
|
| 146 | }
|
| 147 | if name == "pair_consumer":
|
| 148 | return {
|
| 149 | "pkg/feature_b.py": "from pkg import config\n\n\ndef limit():\n return config.LIMIT\n",
|
| 150 | "tests/test_feature_b.py": "from pkg.feature_b import limit\n\n\ndef test_limit():\n assert limit() == 10\n",
|
| 151 | }
|
| 152 | return {
|
| 153 | f"pkg/{name}.py": f"def value():\n return {name!r}\n",
|
| 154 | f"tests/test_{name}.py": f"from pkg.{name} import value\n\n\ndef test_value():\n assert value() == {name!r}\n",
|
| 155 | }
|
| 156 |
|
| 157 |
|
| 158 | def mover_file(j: int) -> tuple[str, str]:
|
| 159 | return f"hist/h{j:03d}.txt", f"main moved {j}\n"
|
| 160 |
|
| 161 |
|
| 162 | def expected_final_files(n: int, k: int, pair_winner: str) -> dict[str, str]:
|
| 163 | files = dict(base_files())
|
| 164 | for j in range(k):
|
| 165 | path, content = mover_file(j)
|
| 166 | files[path] = content
|
| 167 | for name in agent_names(n)[:n]:
|
| 168 | files.update(agent_patch(name))
|
| 169 | files.update(agent_patch(pair_winner))
|
| 170 | return files
|
| 171 |
|
| 172 |
|
| 173 | def write_files(root: Path, files: dict[str, str]) -> None:
|
| 174 | for rel, content in files.items():
|
| 175 | path = root / rel
|
| 176 | path.parent.mkdir(parents=True, exist_ok=True)
|
| 177 | path.write_text(content)
|
| 178 |
|
| 179 |
|
| 180 | def files_digest(files: dict[str, str]) -> str:
|
| 181 | """tree_digest() of a tree that holds exactly ``files``."""
|
| 182 | with tempfile.TemporaryDirectory(prefix="sl-digest-") as tmp:
|
| 183 | write_files(Path(tmp), files)
|
| 184 | return tree_digest(Path(tmp))
|
| 185 |
|
| 186 |
|
| 187 | def read_tree(root: Path) -> dict[str, str]:
|
| 188 | out: dict[str, str] = {}
|
| 189 | for path in sorted(root.rglob("*")):
|
| 190 | rel = path.relative_to(root).as_posix()
|
| 191 | if not path.is_file() or rel.startswith(".git/") or "__pycache__" in rel:
|
| 192 | continue
|
| 193 | if rel.startswith(".oak/") and not rel.startswith(".oak/workflows/"):
|
| 194 | continue
|
| 195 | out[rel] = path.read_text(errors="replace")
|
| 196 | return out
|
| 197 |
|
| 198 |
|
| 199 | def run_check(root: Path) -> tuple[bool, str]:
|
| 200 | env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
|
| 201 | proc = subprocess.run(
|
| 202 | [sys.executable, "ci/check.py"], cwd=root, env=env, capture_output=True, text=True, timeout=120
|
| 203 | )
|
| 204 | return proc.returncode == 0, (proc.stdout + proc.stderr)[-400:]
|
| 205 |
|
| 206 |
|
| 207 | def classify_final(files: dict[str, str], n: int, k: int) -> dict[str, Any]:
|
| 208 | """Final-state oracle: exactly the intended changes, one pair member."""
|
| 209 | rename_in = files.get("pkg/config.py") == agent_patch("pair_rename")["pkg/config.py"]
|
| 210 | consumer_in = "pkg/feature_b.py" in files
|
| 211 | winners = [name for name, present in (("pair_rename", rename_in), ("pair_consumer", consumer_in)) if present]
|
| 212 | missing_agents = [
|
| 213 | name for name in agent_names(n)[:n] if any(files.get(p) != c for p, c in agent_patch(name).items())
|
| 214 | ]
|
| 215 | exact = False
|
| 216 | if len(winners) == 1:
|
| 217 | exact = files == expected_final_files(n, k, winners[0])
|
| 218 | return {
|
| 219 | "pair_landed": winners,
|
| 220 | "both_pair_landed": len(winners) == 2,
|
| 221 | "missing_agents": missing_agents,
|
| 222 | "intended_changes_exact": exact,
|
| 223 | }
|
| 224 |
|
| 225 |
|
| 226 | # ---------------------------------------------------------------------------
|
| 227 | # Agent tool-call accounting
|
| 228 | # ---------------------------------------------------------------------------
|
| 229 |
|
| 230 |
|
| 231 | @dataclass
|
| 232 | class AgentRun:
|
| 233 | name: str
|
| 234 | arm: str
|
| 235 | tool_calls: int = 0
|
| 236 | ci_trigger_calls: int = 0
|
| 237 | output_bytes: int = 0
|
| 238 | attempts: int = 0
|
| 239 | ci_runs_local: int = 0
|
| 240 | outcome: str = "pending"
|
| 241 | error: Optional[str] = None
|
| 242 | landed_commit: Optional[str] = None
|
| 243 | landed_at: Optional[float] = None
|
| 244 | finished_at: Optional[float] = None
|
| 245 | tested_digests: list[str] = field(default_factory=list)
|
| 246 | local_checks: int = 0
|
| 247 | converge_path: list[str] = field(default_factory=list)
|
| 248 | trace: list[dict[str, Any]] = field(default_factory=list)
|
| 249 | branch: Optional[str] = None
|
| 250 | abandoned_branches: list[str] = field(default_factory=list)
|
| 251 | rate_limited: int = 0
|
| 252 | conflict_retries: int = 0
|
| 253 |
|
| 254 | def call(self, argv: list[str], cwd: Path, env: dict[str, str], timeout: float = 600) -> subprocess.CompletedProcess[str]:
|
| 255 | start = time.perf_counter()
|
| 256 | try:
|
| 257 | proc = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, text=True, timeout=timeout)
|
| 258 | except subprocess.TimeoutExpired as exc:
|
| 259 | proc = subprocess.CompletedProcess(argv, 124, exc.stdout or "", (exc.stderr or "") + "\n[timeout]")
|
| 260 | self.tool_calls += 1
|
| 261 | out = len((proc.stdout or "").encode()) + len((proc.stderr or "").encode())
|
| 262 | self.output_bytes += out
|
| 263 | self.trace.append(
|
| 264 | {
|
| 265 | "argv": [Path(argv[0]).name] + argv[1:],
|
| 266 | "rc": proc.returncode,
|
| 267 | "ms": round((time.perf_counter() - start) * 1000, 1),
|
| 268 | "bytes": out,
|
| 269 | "tail": ((proc.stdout or "") + (proc.stderr or ""))[-300:],
|
| 270 | }
|
| 271 | )
|
| 272 | return proc
|
| 273 |
|
| 274 | def call_retrying(
|
| 275 | self, argv: list[str], cwd: Path, env: dict[str, str], timeout: float = 600, retry_conflicts: int = 0
|
| 276 | ) -> subprocess.CompletedProcess[str]:
|
| 277 | """A competent agent honours HTTP 429 retry-after and, where the caller
|
| 278 | allows it, explicitly retryable 409s. Every retry is a tool call."""
|
| 279 | conflicts = 0
|
| 280 | for _ in range(30):
|
| 281 | proc = self.call(argv, cwd, env, timeout)
|
| 282 | wait = retry_after(proc)
|
| 283 | if wait is not None:
|
| 284 | self.rate_limited += 1
|
| 285 | time.sleep(wait)
|
| 286 | continue
|
| 287 | if proc.returncode != 0 and conflicts < retry_conflicts and retryable_conflict(proc):
|
| 288 | conflicts += 1
|
| 289 | self.conflict_retries += 1
|
| 290 | time.sleep(0.5)
|
| 291 | continue
|
| 292 | return proc
|
| 293 | return proc
|
| 294 |
|
| 295 | def local_check(self, root: Path, added_seconds: float = 0.0) -> bool:
|
| 296 | """One tool call: run the check script on the working tree."""
|
| 297 | start = time.perf_counter()
|
| 298 | self.tested_digests.append(tree_digest(root))
|
| 299 | ok, tail = run_check(root)
|
| 300 | if added_seconds > 0:
|
| 301 | time.sleep(added_seconds)
|
| 302 | self.tool_calls += 1
|
| 303 | self.local_checks += 1
|
| 304 | self.output_bytes += len(tail.encode())
|
| 305 | self.trace.append({"argv": ["check"], "rc": 0 if ok else 1, "ms": round((time.perf_counter() - start) * 1000, 1), "bytes": len(tail.encode()), "tail": tail})
|
| 306 | return ok
|
| 307 |
|
| 308 | def apply_patch(self, root: Path, files: dict[str, str]) -> None:
|
| 309 | """One tool call: the agent re-applies its intended edit (fallback path)."""
|
| 310 | write_files(root, files)
|
| 311 | self.tool_calls += 1
|
| 312 | self.trace.append({"argv": ["apply_patch"], "rc": 0, "ms": 0.0, "bytes": 0, "tail": ""})
|
| 313 |
|
| 314 |
|
| 315 | def last_json(text: str) -> Optional[dict[str, Any]]:
|
| 316 | """Last JSON object in stdout (some clients print human lines first, fb-479)."""
|
| 317 | for line in reversed((text or "").strip().splitlines()):
|
| 318 | line = line.strip()
|
| 319 | if line.startswith("{"):
|
| 320 | try:
|
| 321 | value = json.loads(line)
|
| 322 | except ValueError:
|
| 323 | continue
|
| 324 | if isinstance(value, dict):
|
| 325 | return value
|
| 326 | try:
|
| 327 | value = json.loads(text)
|
| 328 | return value if isinstance(value, dict) else None
|
| 329 | except ValueError:
|
| 330 | return None
|
| 331 |
|
| 332 |
|
| 333 | # ---------------------------------------------------------------------------
|
| 334 | # Oak arms (local oakspace server)
|
| 335 | # ---------------------------------------------------------------------------
|
| 336 |
|
| 337 |
|
| 338 | @dataclass
|
| 339 | class OakServer:
|
| 340 | url: str
|
| 341 | token: str
|
| 342 | secret: str
|
| 343 | owner: str
|
| 344 | psql: list[str]
|
| 345 | home: Path
|
| 346 |
|
| 347 | def env(self) -> dict[str, str]:
|
| 348 | env = dict(os.environ)
|
| 349 | env.update(
|
| 350 | {
|
| 351 | "HOME": str(self.home),
|
| 352 | "OAK_API_KEY": self.token,
|
| 353 | "OAK_URL": self.url,
|
| 354 | "OAK_REMOTE": self.url,
|
| 355 | "OAK_AUTHOR": self.owner,
|
| 356 | "OAK_TRUSTED_REMOTES": self.url,
|
| 357 | "PYTHONDONTWRITEBYTECODE": "1",
|
| 358 | "NO_COLOR": "1",
|
| 359 | }
|
| 360 | )
|
| 361 | return env
|
| 362 |
|
| 363 | def sql(self, query: str) -> list[list[str]]:
|
| 364 | proc = subprocess.run(self.psql + ["-qAt", "-F", "\t", "-c", query], capture_output=True, text=True, timeout=60)
|
| 365 | if proc.returncode != 0:
|
| 366 | raise RuntimeError(f"psql failed: {proc.stderr.strip()[:300]}")
|
| 367 | return [line.split("\t") for line in proc.stdout.splitlines() if line.strip()]
|
| 368 |
|
| 369 | def repo_id(self, repo: str) -> int:
|
| 370 | rows = self.sql(f"select id from repos where name='{repo}'")
|
| 371 | return int(rows[0][0])
|
| 372 |
|
| 373 | def main_heads(self, repo_id: int) -> list[tuple[int, str]]:
|
| 374 | rows = self.sql(
|
| 375 | f"select transition_generation, head_hash from publication_events where repo_id={repo_id} "
|
| 376 | "and branch_name='main' order by transition_generation"
|
| 377 | )
|
| 378 | return [(int(gen), head) for gen, head in rows]
|
| 379 |
|
| 380 | def fetch_token(self, repo_id: int, commit: str) -> str:
|
| 381 | payload = f"{repo_id}.{commit}.{int(time.time()) + 900}"
|
| 382 | return payload + "." + hmac.new(self.secret.encode(), payload.encode(), hashlib.sha256).hexdigest()
|
| 383 |
|
| 384 | def materialize(self, repo_id: int, commit: str, dest: Path) -> Path:
|
| 385 | env = dict(os.environ)
|
| 386 | env.update(
|
| 387 | {
|
| 388 | "GIT_TERMINAL_PROMPT": "0",
|
| 389 | "GIT_CONFIG_COUNT": "1",
|
| 390 | "GIT_CONFIG_KEY_0": "http.extraHeader",
|
| 391 | "GIT_CONFIG_VALUE_0": f"x-oak-ci-fetch-token: {self.fetch_token(repo_id, commit)}",
|
| 392 | }
|
| 393 | )
|
| 394 | shutil.rmtree(dest, ignore_errors=True)
|
| 395 | proc = subprocess.run(
|
| 396 | ["git", "clone", "-q", "--depth", "1", "--no-tags", f"{self.url}/api/ci/fetch/{repo_id}", str(dest)],
|
| 397 | env=env, capture_output=True, text=True, timeout=120,
|
| 398 | )
|
| 399 | if proc.returncode != 0:
|
| 400 | raise RuntimeError(f"materialize {commit[:12]} failed: {proc.stderr.strip()[:300]}")
|
| 401 | return dest
|
| 402 |
|
| 403 |
|
| 404 | RETRY_AFTER_RE = re.compile(r"429 Too Many Requests.*?retry after (\d+) second")
|
| 405 |
|
| 406 |
|
| 407 | def retry_after(proc: subprocess.CompletedProcess[str]) -> Optional[float]:
|
| 408 | """Seconds the server asked us to wait (HTTP 429), else None."""
|
| 409 | match = RETRY_AFTER_RE.search((proc.stderr or "") + (proc.stdout or ""))
|
| 410 | return float(match.group(1)) if match else None
|
| 411 |
|
| 412 |
|
| 413 | def retryable_conflict(proc: subprocess.CompletedProcess[str]) -> bool:
|
| 414 | """A conclusive server refusal that explicitly says it is safe to retry.
|
| 415 |
|
| 416 | Oak prints e.g. ``HTTP 409 Conflict: a verified blob mapping changed before
|
| 417 | branch publication; retry`` (concurrent pushes) or a ``main_moved`` /
|
| 418 | ``branch_moved`` code with ``retryable: true`` (concurrent merges). Nothing
|
| 419 | was written, so retrying is not replaying an ambiguous mutation.
|
| 420 | """
|
| 421 | text = (proc.stderr or "") + (proc.stdout or "")
|
| 422 | return proc.returncode != 0 and "409" in text and ("; retry" in text or '"retryable":true' in text or "retryable: true" in text)
|
| 423 |
|
| 424 |
|
| 425 | def oak_run(oak: Path, args: list[str], cwd: Path, env: dict[str, str], timeout: float = 300) -> subprocess.CompletedProcess[str]:
|
| 426 | """Untimed setup call; honours the server's 429 retry-after hint."""
|
| 427 | for _ in range(30):
|
| 428 | proc = subprocess.run([str(oak)] + args, cwd=cwd, env=env, capture_output=True, text=True, timeout=timeout)
|
| 429 | wait = retry_after(proc)
|
| 430 | if wait is None and retryable_conflict(proc):
|
| 431 | wait = 0.5
|
| 432 | if proc.returncode == 0 or wait is None:
|
| 433 | return proc
|
| 434 | time.sleep(wait)
|
| 435 | return proc
|
| 436 |
|
| 437 |
|
| 438 | def must(proc: subprocess.CompletedProcess[str], what: str) -> subprocess.CompletedProcess[str]:
|
| 439 | if proc.returncode != 0:
|
| 440 | raise RuntimeError(f"setup step failed ({what}): rc={proc.returncode} {(proc.stderr or proc.stdout).strip()[-300:]}")
|
| 441 | return proc
|
| 442 |
|
| 443 |
|
| 444 | def oak_head(oak: Path, cwd: Path, env: dict[str, str]) -> str:
|
| 445 | return must(oak_run(oak, ["hash"], cwd, env), "hash").stdout.strip()
|
| 446 |
|
| 447 |
|
| 448 | def oak_branch(oak: Path, cwd: Path, env: dict[str, str]) -> str:
|
| 449 | return json.loads(must(oak_run(oak, ["status", "--json"], cwd, env), "status").stdout)["branch"]
|
| 450 |
|
| 451 |
|
| 452 | def oak_trigger_and_wait(agent: Optional[AgentRun], oak: Path, cwd: Path, env: dict[str, str], key: str) -> tuple[bool, Optional[str]]:
|
| 453 | """Exact-commit CI: trigger on the current head and wait for its conclusion."""
|
| 454 | call = agent.call_retrying if agent else (lambda argv, cwd, env, timeout=600: oak_run(Path(argv[0]), argv[1:], cwd, env, timeout))
|
| 455 | head = oak_head(oak, cwd, env) if agent is None else None
|
| 456 | if agent is not None:
|
| 457 | proc = call([str(oak), "hash"], cwd, env)
|
| 458 | head = proc.stdout.strip()
|
| 459 | trig = call([str(oak), "ci", "trigger", "--expected-commit", head, "--idempotency-key", key, "--json"], cwd, env)
|
| 460 | if agent is not None:
|
| 461 | agent.ci_trigger_calls += 1
|
| 462 | doc = last_json(trig.stdout) or {}
|
| 463 | run_ids = [str(r) for r in doc.get("run_ids") or []]
|
| 464 | if trig.returncode != 0 or not run_ids:
|
| 465 | return False, f"ci trigger failed: {(trig.stderr or trig.stdout).strip()[-200:]}"
|
| 466 | wait = call([str(oak), "ci", "wait", *run_ids, "--commit", head, "--timeout", str(CI_WAIT_TIMEOUT_S), "--json"], cwd, env, timeout=CI_WAIT_TIMEOUT_S + 30)
|
| 467 | wdoc = last_json(wait.stdout) or {}
|
| 468 | text = json.dumps(wdoc)
|
| 469 | ok = wait.returncode == 0 and '"failure"' not in text and '"cancelled"' not in text and '"timed_out"' not in text
|
| 470 | return ok, None if ok else f"ci not green: rc={wait.returncode} {text[:200]}"
|
| 471 |
|
| 472 |
|
| 473 | @dataclass
|
| 474 | class OakTrial:
|
| 475 | server: OakServer
|
| 476 | oak: Path
|
| 477 | setup_oak: Path
|
| 478 | root: Path
|
| 479 | repo: str
|
| 480 | n: int
|
| 481 | k: int
|
| 482 | arm: str
|
| 483 | stub: CiStub
|
| 484 | repo_id: int = 0
|
| 485 | workers: dict[str, Path] = field(default_factory=dict)
|
| 486 | branches: dict[str, str] = field(default_factory=dict)
|
| 487 | setup_ledger: Path = Path()
|
| 488 | trial_ledger: Path = Path()
|
| 489 | base_generation: int = 0
|
| 490 |
|
| 491 | def setup(self) -> dict[str, Any]:
|
| 492 | env = self.server.env()
|
| 493 | t0 = time.perf_counter()
|
| 494 | seed = self.root / "seed"
|
| 495 | seed.mkdir(parents=True)
|
| 496 | must(oak_run(self.setup_oak, ["init"], seed, env), "init")
|
| 497 | write_files(seed, base_files())
|
| 498 | must(oak_run(self.setup_oak, ["commit"], seed, env), "seed commit")
|
| 499 | must(oak_run(self.setup_oak, ["push", "-r", self.server.url, "--repo", f"{self.server.owner}/{self.repo}"], seed, env), "seed push")
|
| 500 | # Setup-only: bypass the gate (the auto-triggered run cannot read the workflow locally).
|
| 501 | must(oak_run(self.setup_oak, ["merge", "--force", "--json"], seed, env), "seed merge")
|
| 502 | self.repo_id = self.server.repo_id(self.repo)
|
| 503 | self.stub.ledger = self.setup_ledger
|
| 504 |
|
| 505 | errors: list[str] = []
|
| 506 |
|
| 507 | def prep(name: str) -> None:
|
| 508 | try:
|
| 509 | wdir = self.root / f"w-{name}"
|
| 510 | must(oak_run(self.oak, ["clone", "--shallow", f"{self.server.owner}/{self.repo}", str(wdir)], self.root, env), f"clone {name}")
|
| 511 | write_files(wdir, agent_patch(name))
|
| 512 | must(oak_run(self.oak, ["commit"], wdir, env), f"commit {name}")
|
| 513 | must(oak_run(self.oak, ["push"], wdir, env), f"push {name}")
|
| 514 | ok, err = oak_trigger_and_wait(None, self.oak, wdir, env, f"setup-{name}")
|
| 515 | if not ok:
|
| 516 | raise RuntimeError(f"setup CI for {name}: {err}")
|
| 517 | self.workers[name] = wdir
|
| 518 | self.branches[name] = oak_branch(self.oak, wdir, env)
|
| 519 | except Exception as exc: # noqa: BLE001
|
| 520 | errors.append(f"{name}: {exc}")
|
| 521 |
|
| 522 | threads = [threading.Thread(target=prep, args=(name,)) for name in agent_names(self.n)]
|
| 523 | for thread in threads:
|
| 524 | thread.start()
|
| 525 | for thread in threads:
|
| 526 | thread.join()
|
| 527 | if errors:
|
| 528 | raise RuntimeError("; ".join(errors))
|
| 529 |
|
| 530 | mover = self.root / "mover"
|
| 531 | must(oak_run(self.setup_oak, ["clone", "--shallow", f"{self.server.owner}/{self.repo}", str(mover)], self.root, env), "mover clone")
|
| 532 | for j in range(self.k):
|
| 533 | path, content = mover_file(j)
|
| 534 | write_files(mover, {path: content})
|
| 535 | must(oak_run(self.setup_oak, ["commit"], mover, env), "mover commit")
|
| 536 | must(oak_run(self.setup_oak, ["push"], mover, env), "mover push")
|
| 537 | must(oak_run(self.setup_oak, ["merge", "--force", "--json"], mover, env), "mover merge")
|
| 538 | heads = self.server.main_heads(self.repo_id)
|
| 539 | self.base_generation = heads[-1][0]
|
| 540 | self.stub.ledger = self.trial_ledger
|
| 541 | return {"setup_ms": (time.perf_counter() - t0) * 1000, "main_head_before": heads[-1][1], "main_commits_before": len(heads)}
|
| 542 |
|
| 543 | # -- timed flows -----------------------------------------------------------
|
| 544 | def main_head(self, agent: AgentRun, cwd: Path, env: dict[str, str]) -> Optional[str]:
|
| 545 | proc = agent.call_retrying([str(self.oak), "branch", "show", "main", "--remote", "--json"], cwd, env)
|
| 546 | doc = last_json(proc.stdout) or {}
|
| 547 | return doc.get("head")
|
| 548 |
|
| 549 | def flow_naive(self, agent: AgentRun) -> None:
|
| 550 | env = self.server.env()
|
| 551 | wdir = self.workers[agent.name]
|
| 552 | agent.attempts = 1
|
| 553 | proc = agent.call_retrying([str(self.oak), "merge", "--json"], wdir, env, retry_conflicts=3)
|
| 554 | doc = last_json(proc.stdout) or {}
|
| 555 | if proc.returncode == 0 and doc.get("merged"):
|
| 556 | agent.landed_commit = doc.get("commit_hash")
|
| 557 | agent.landed_at = time.perf_counter()
|
| 558 | agent.outcome = "landed"
|
| 559 | else:
|
| 560 | agent.outcome = "failed"
|
| 561 | agent.error = f"merge rc={proc.returncode} {(proc.stderr or proc.stdout).strip()[-200:]}"
|
| 562 |
|
| 563 | def flow_safe(self, agent: AgentRun) -> None:
|
| 564 | env = self.server.env()
|
| 565 | wdir = self.workers[agent.name]
|
| 566 | while agent.attempts < MAX_ATTEMPTS:
|
| 567 | agent.attempts += 1
|
| 568 | seen_main = self.main_head(agent, wdir, env)
|
| 569 | pull = agent.call_retrying([str(self.oak), "pull"], wdir, env)
|
| 570 | if pull.returncode != 0:
|
| 571 | # Documented fallback: fresh shallow worker from current main + reapply.
|
| 572 | agent.converge_path.append("fresh_reapply")
|
| 573 | if agent.branch:
|
| 574 | agent.abandoned_branches.append(agent.branch)
|
| 575 | fresh = self.root / f"w-{agent.name}-fresh{agent.attempts}"
|
| 576 | clone = agent.call_retrying([str(self.oak), "clone", "--shallow", f"{self.server.owner}/{self.repo}", str(fresh)], self.root, env)
|
| 577 | if clone.returncode != 0:
|
| 578 | agent.outcome, agent.error = "failed", f"fresh clone failed: {clone.stderr.strip()[-200:]}"
|
| 579 | return
|
| 580 | seen_main = self.main_head(agent, fresh, env)
|
| 581 | wdir = fresh
|
| 582 | agent.apply_patch(wdir, agent_patch(agent.name))
|
| 583 | commit = agent.call_retrying([str(self.oak), "commit"], wdir, env)
|
| 584 | if commit.returncode != 0:
|
| 585 | agent.outcome, agent.error = "failed", f"commit failed: {commit.stderr.strip()[-200:]}"
|
| 586 | return
|
| 587 | else:
|
| 588 | agent.converge_path.append("pull")
|
| 589 | if not agent.local_check(wdir):
|
| 590 | agent.outcome = "refused_red_combined"
|
| 591 | return
|
| 592 | push = agent.call_retrying([str(self.oak), "push", "--json"], wdir, env, retry_conflicts=3)
|
| 593 | if push.returncode != 0:
|
| 594 | agent.outcome, agent.error = "failed", f"push failed: {(push.stderr or push.stdout).strip()[-200:]}"
|
| 595 | return
|
| 596 | agent.branch = (last_json(push.stdout) or {}).get("branch") or agent.branch
|
| 597 | ok, err = oak_trigger_and_wait(agent, self.oak, wdir, env, f"{agent.name}-a{agent.attempts}")
|
| 598 | if not ok:
|
| 599 | agent.outcome, agent.error = "refused_ci_red", err
|
| 600 | return
|
| 601 | now_main = self.main_head(agent, wdir, env)
|
| 602 | if seen_main is None or now_main != seen_main:
|
| 603 | continue # main moved since converge: the tested tree is stale; re-converge
|
| 604 | merge = agent.call_retrying([str(self.oak), "merge", "--json"], wdir, env)
|
| 605 | doc = last_json(merge.stdout) or {}
|
| 606 | if merge.returncode == 0 and doc.get("merged"):
|
| 607 | agent.landed_commit = doc.get("commit_hash")
|
| 608 | agent.landed_at = time.perf_counter()
|
| 609 | verify = self.main_head(agent, wdir, env)
|
| 610 | agent.outcome = "landed"
|
| 611 | if verify is None:
|
| 612 | agent.error = "post-merge verify could not read main"
|
| 613 | return
|
| 614 | text = (merge.stderr or "") + (merge.stdout or "")
|
| 615 | if "main_moved" in text or "branch_moved" in text or "409" in text:
|
| 616 | continue
|
| 617 | agent.outcome, agent.error = "failed", f"merge rc={merge.returncode} {text.strip()[-200:]}"
|
| 618 | return
|
| 619 | agent.outcome, agent.error = "failed", "attempt budget exhausted"
|
| 620 |
|
| 621 | # -- oracles ---------------------------------------------------------------
|
| 622 | def oracle(self, agents: list[AgentRun]) -> dict[str, Any]:
|
| 623 | heads = [(g, h) for g, h in self.server.main_heads(self.repo_id) if g > self.base_generation]
|
| 624 | red = 0
|
| 625 | digests: dict[str, str] = {}
|
| 626 | final_files: dict[str, str] = {}
|
| 627 | for index, (_, head) in enumerate(heads):
|
| 628 | tree = self.server.materialize(self.repo_id, head, self.root / f"oracle-{index}")
|
| 629 | ok, _ = run_check(tree)
|
| 630 | red += 0 if ok else 1
|
| 631 | digests[head] = tree_digest(tree)
|
| 632 | final_files = read_tree(tree)
|
| 633 | if not heads:
|
| 634 | tree = self.server.materialize(self.repo_id, self.server.main_heads(self.repo_id)[-1][1], self.root / "oracle-final")
|
| 635 | final_files = read_tree(tree)
|
| 636 | ci_trial = read_ledger(self.trial_ledger)
|
| 637 | ci_setup = read_ledger(self.setup_ledger)
|
| 638 | return {"heads": heads, "red": red, "digests": digests, "final_files": final_files, "ci_trial": ci_trial, "ci_setup": ci_setup}
|
| 639 |
|
| 640 |
|
| 641 | # ---------------------------------------------------------------------------
|
| 642 | # Git arms (bare origin, file transport)
|
| 643 | # ---------------------------------------------------------------------------
|
| 644 |
|
| 645 |
|
| 646 | def git(args: list[str], cwd: Path, env: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess[str]:
|
| 647 | return subprocess.run(["git"] + args, cwd=cwd, env=env, capture_output=True, text=True, timeout=300)
|
| 648 |
|
| 649 |
|
| 650 | def git_env() -> dict[str, str]:
|
| 651 | env = dict(os.environ)
|
| 652 | env.update(
|
| 653 | {
|
| 654 | "GIT_AUTHOR_NAME": "bench", "GIT_AUTHOR_EMAIL": "bench@localhost", "GIT_COMMITTER_NAME": "bench",
|
| 655 | "GIT_COMMITTER_EMAIL": "bench@localhost", "GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0",
|
| 656 | "PYTHONDONTWRITEBYTECODE": "1", "HOME": env.get("OAK_BENCH_GIT_HOME", tempfile.gettempdir()),
|
| 657 | }
|
| 658 | )
|
| 659 | return env
|
| 660 |
|
| 661 |
|
| 662 | @dataclass
|
| 663 | class GitTrial:
|
| 664 | root: Path
|
| 665 | n: int
|
| 666 | k: int
|
| 667 | arm: str
|
| 668 | ci_added_seconds: float
|
| 669 | workers: dict[str, Path] = field(default_factory=dict)
|
| 670 | base_main: str = ""
|
| 671 | queue: threading.Lock = field(default_factory=threading.Lock)
|
| 672 |
|
| 673 | @property
|
| 674 | def origin(self) -> Path:
|
| 675 | return self.root / "origin.git"
|
| 676 |
|
| 677 | def setup(self) -> dict[str, Any]:
|
| 678 | env = git_env()
|
| 679 | t0 = time.perf_counter()
|
| 680 | must(git(["init", "-q", "--bare", "-b", "main", str(self.origin)], self.root, env), "bare init")
|
| 681 | seed = self.root / "seed"
|
| 682 | must(git(["clone", "-q", str(self.origin), str(seed)], self.root, env), "seed clone")
|
| 683 | write_files(seed, base_files())
|
| 684 | must(git(["checkout", "-q", "-b", "main"], seed, env), "seed branch")
|
| 685 | must(git(["add", "-A"], seed, env), "seed add")
|
| 686 | must(git(["commit", "-q", "-m", "base"], seed, env), "seed commit")
|
| 687 | must(git(["push", "-q", "origin", "main"], seed, env), "seed push")
|
| 688 | for name in agent_names(self.n):
|
| 689 | wdir = self.root / f"w-{name}"
|
| 690 | must(git(["clone", "-q", str(self.origin), str(wdir)], self.root, env), f"clone {name}")
|
| 691 | must(git(["checkout", "-q", "-b", name], wdir, env), f"branch {name}")
|
| 692 | write_files(wdir, agent_patch(name))
|
| 693 | must(git(["add", "-A"], wdir, env), f"add {name}")
|
| 694 | must(git(["commit", "-q", "-m", name], wdir, env), f"commit {name}")
|
| 695 | self.workers[name] = wdir
|
| 696 | for j in range(self.k):
|
| 697 | path, content = mover_file(j)
|
| 698 | write_files(seed, {path: content})
|
| 699 | must(git(["add", "-A"], seed, env), "mover add")
|
| 700 | must(git(["commit", "-q", "-m", f"move {j}"], seed, env), "mover commit")
|
| 701 | must(git(["push", "-q", "origin", "main"], seed, env), "mover push")
|
| 702 | self.base_main = must(git(["rev-parse", "main"], self.origin, env), "rev-parse").stdout.strip()
|
| 703 | return {"setup_ms": (time.perf_counter() - t0) * 1000, "main_head_before": self.base_main, "main_commits_before": self.k + 1}
|
| 704 |
|
| 705 | def flow(self, agent: AgentRun) -> None:
|
| 706 | env = git_env()
|
| 707 | wdir = self.workers[agent.name]
|
| 708 | g = lambda args: agent.call(["git"] + args, wdir, env) # noqa: E731
|
| 709 | while agent.attempts < MAX_ATTEMPTS * 3:
|
| 710 | ctx = self.queue if self.arm == "git_fifo" else _NullLock()
|
| 711 | with ctx:
|
| 712 | agent.attempts += 1
|
| 713 | if g(["fetch", "-q", "origin"]).returncode != 0:
|
| 714 | continue
|
| 715 | if g(["rebase", "-q", "origin/main"]).returncode != 0:
|
| 716 | g(["rebase", "--abort"])
|
| 717 | agent.outcome, agent.error = "failed", "rebase conflict"
|
| 718 | return
|
| 719 | agent.ci_runs_local += 1
|
| 720 | if not agent.local_check(wdir, added_seconds=self.ci_added_seconds):
|
| 721 | agent.outcome = "refused_red_combined"
|
| 722 | return
|
| 723 | push = g(["push", "-q", "origin", "HEAD:main"])
|
| 724 | if push.returncode == 0:
|
| 725 | agent.landed_commit = must(git(["rev-parse", "HEAD"], wdir, env), "rev-parse").stdout.strip()
|
| 726 | agent.landed_at = time.perf_counter()
|
| 727 | agent.outcome = "landed"
|
| 728 | return
|
| 729 | if "rejected" not in (push.stderr or "") and "fetch first" not in (push.stderr or ""):
|
| 730 | agent.outcome, agent.error = "failed", f"push failed: {push.stderr.strip()[-200:]}"
|
| 731 | return
|
| 732 | agent.outcome, agent.error = "failed", "attempt budget exhausted"
|
| 733 |
|
| 734 | def oracle(self, agents: list[AgentRun]) -> dict[str, Any]:
|
| 735 | env = git_env()
|
| 736 | revs = must(git(["rev-list", "--reverse", f"{self.base_main}..main"], self.origin, env), "rev-list").stdout.split()
|
| 737 | red = 0
|
| 738 | digests: dict[str, str] = {}
|
| 739 | final_files: dict[str, str] = {}
|
| 740 | targets = revs or [self.base_main]
|
| 741 | for index, rev in enumerate(targets):
|
| 742 | dest = self.root / f"oracle-{index}"
|
| 743 | dest.mkdir()
|
| 744 | archive = subprocess.run(["git", "archive", rev], cwd=self.origin, capture_output=True, timeout=120)
|
| 745 | subprocess.run(["tar", "-x", "-C", str(dest)], input=archive.stdout, check=True, timeout=120)
|
| 746 | if revs:
|
| 747 | ok, _ = run_check(dest)
|
| 748 | red += 0 if ok else 1
|
| 749 | digests[rev] = tree_digest(dest)
|
| 750 | final_files = read_tree(dest)
|
| 751 | return {"heads": [(i, r) for i, r in enumerate(revs)], "red": red, "digests": digests, "final_files": final_files, "ci_trial": [], "ci_setup": []}
|
| 752 |
|
| 753 |
|
| 754 | class _NullLock:
|
| 755 | def __enter__(self) -> None:
|
| 756 | return None
|
| 757 |
|
| 758 | def __exit__(self, *exc: Any) -> None:
|
| 759 | return None
|
| 760 |
|
| 761 |
|
| 762 | # ---------------------------------------------------------------------------
|
| 763 | # Trial driver + rows
|
| 764 | # ---------------------------------------------------------------------------
|
| 765 |
|
| 766 |
|
| 767 | def percentile(values: list[float], p: float) -> Optional[float]:
|
| 768 | if not values:
|
| 769 | return None
|
| 770 | ordered = sorted(values)
|
| 771 | index = min(len(ordered) - 1, max(0, int(round(p / 100 * (len(ordered) - 1)))))
|
| 772 | return ordered[index]
|
| 773 |
|
| 774 |
|
| 775 | def run_agents(agents: list[AgentRun], flow: Callable[[AgentRun], None]) -> tuple[float, float]:
|
| 776 | barrier = threading.Barrier(len(agents) + 1)
|
| 777 |
|
| 778 | def body(agent: AgentRun) -> None:
|
| 779 | barrier.wait()
|
| 780 | try:
|
| 781 | flow(agent)
|
| 782 | except Exception as exc: # noqa: BLE001 - recorded as a failed agent, never hidden
|
| 783 | agent.outcome, agent.error = "failed", f"harness exception: {exc}"
|
| 784 | agent.finished_at = time.perf_counter()
|
| 785 |
|
| 786 | threads = [threading.Thread(target=body, args=(agent,)) for agent in agents]
|
| 787 | for thread in threads:
|
| 788 | thread.start()
|
| 789 | barrier.wait()
|
| 790 | start = time.perf_counter()
|
| 791 | for thread in threads:
|
| 792 | thread.join()
|
| 793 | return start, time.perf_counter()
|
| 794 |
|
| 795 |
|
| 796 | def build_rows(
|
| 797 | *,
|
| 798 | meta: dict[str, Any],
|
| 799 | arm: str,
|
| 800 | n: int,
|
| 801 | k: int,
|
| 802 | run_index: int,
|
| 803 | agents: list[AgentRun],
|
| 804 | start: float,
|
| 805 | end: float,
|
| 806 | setup: dict[str, Any],
|
| 807 | oracle: dict[str, Any],
|
| 808 | subject: str,
|
| 809 | subject_kind: str,
|
| 810 | binary_hash: Optional[str],
|
| 811 | ) -> list[dict[str, Any]]:
|
| 812 | is_oak = arm in OAK_ARMS
|
| 813 | ci_by_commit: dict[str, list[dict[str, Any]]] = {}
|
| 814 | for record in oracle["ci_trial"]:
|
| 815 | ci_by_commit.setdefault(record.get("ref") or "", []).append(record)
|
| 816 | setup_by_ref: dict[str, list[dict[str, Any]]] = {}
|
| 817 | for record in oracle["ci_setup"]:
|
| 818 | setup_by_ref.setdefault(record.get("ref") or "", []).append(record)
|
| 819 | common = {
|
| 820 | **meta,
|
| 821 | "profile": PROFILE,
|
| 822 | "scenario": SCENARIO,
|
| 823 | "scenario_version": SCENARIO_VERSION,
|
| 824 | "fixture_version": FIXTURE_VERSION,
|
| 825 | "contention_mode": arm,
|
| 826 | "workers": n,
|
| 827 | "agents_total": n + len(PAIR),
|
| 828 | "behind_k": k,
|
| 829 | "run": run_index,
|
| 830 | "subject": subject,
|
| 831 | "subject_kind": subject_kind,
|
| 832 | "binary_sha256": binary_hash,
|
| 833 | "transport": "loopback" if is_oak else "local-file",
|
| 834 | "hosted_equivalence": "hosted-equivalent-local" if is_oak else "not-applicable-local-git",
|
| 835 | "ci_executor": "oakspace_dispatch+oakbench.ci_stub(loopback)" if is_oak else "local_subprocess",
|
| 836 | "ci_trigger_mode": CI_TRIGGER_NOTE if is_oak else "local_pre_push",
|
| 837 | }
|
| 838 | rows: list[dict[str, Any]] = []
|
| 839 | classification = classify_final(oracle["final_files"], n, k)
|
| 840 | for agent in agents:
|
| 841 | refs = [agent.branch or ""] + list(agent.abandoned_branches)
|
| 842 | ci_runs_trial = [r for ref in refs for r in ci_by_commit.get(ref, [])]
|
| 843 | ci_digests = [r.get("tree_digest") for r in ci_runs_trial if r.get("conclusion") == "success"]
|
| 844 | if arm == "oak_naive":
|
| 845 | ci_digests += [r.get("tree_digest") for r in setup_by_ref.get(agent.branch or "", []) if r.get("conclusion") == "success"]
|
| 846 | landed_digest = oracle["digests"].get(agent.landed_commit or "")
|
| 847 | tested = None
|
| 848 | local_tested = ci_tested = None
|
| 849 | if agent.outcome == "landed":
|
| 850 | local_tested = landed_digest is not None and landed_digest in agent.tested_digests
|
| 851 | ci_tested = landed_digest is not None and landed_digest in ci_digests
|
| 852 | tested = bool(local_tested or ci_tested)
|
| 853 | ci_runs = len(ci_runs_trial) if is_oak else agent.ci_runs_local
|
| 854 | rows.append(
|
| 855 | {
|
| 856 | **common,
|
| 857 | "operation": "land.agent",
|
| 858 | "agent": agent.name,
|
| 859 | "agent_role": "pair" if agent.name in PAIR else "ordinary",
|
| 860 | "elapsed_ms": round(((agent.landed_at or agent.finished_at or end) - start) * 1000, 1),
|
| 861 | "time_to_landed_ms": round((agent.landed_at - start) * 1000, 1) if agent.landed_at else None,
|
| 862 | "returncode": 0 if agent.outcome in ("landed", "refused_red_combined", "refused_ci_red") else 1,
|
| 863 | "outcome": agent.outcome,
|
| 864 | "error": agent.error,
|
| 865 | "tool_call_count": agent.tool_calls,
|
| 866 | "tool_call_count_hosted_equivalent": agent.tool_calls - agent.ci_trigger_calls if is_oak else agent.tool_calls,
|
| 867 | "ci_trigger_calls": agent.ci_trigger_calls if is_oak else None,
|
| 868 | "output_bytes": agent.output_bytes,
|
| 869 | "attempts": agent.attempts,
|
| 870 | "rate_limited_retries": agent.rate_limited if is_oak else None,
|
| 871 | "conflict_retries": agent.conflict_retries if is_oak else None,
|
| 872 | "retries": max(0, agent.attempts - 1),
|
| 873 | "ci_runs": ci_runs,
|
| 874 | "local_checks": agent.local_checks,
|
| 875 | "converge_path": agent.converge_path or None,
|
| 876 | "landed_commit": agent.landed_commit,
|
| 877 | "combined_tree_tested_before_land": tested,
|
| 878 | "combined_tree_ci_tested_before_land": ci_tested,
|
| 879 | "combined_tree_locally_tested_before_land": local_tested,
|
| 880 | "command": " ; ".join(" ".join(t["argv"][:3]) for t in agent.trace)[:2000],
|
| 881 | "trace": agent.trace,
|
| 882 | }
|
| 883 | )
|
| 884 | landed = [a for a in agents if a.outcome == "landed"]
|
| 885 | ordinary = [a for a in agents if a.name not in PAIR]
|
| 886 | times = [(a.landed_at - start) * 1000 for a in ordinary if a.landed_at]
|
| 887 | tested_flags = [r["combined_tree_tested_before_land"] for r in rows if r["outcome"] == "landed"]
|
| 888 | total_ci = len(oracle["ci_trial"]) if is_oak else sum(a.ci_runs_local for a in agents)
|
| 889 | final_green = None
|
| 890 | if oracle["final_files"]:
|
| 891 | with tempfile.TemporaryDirectory(prefix="sl-final-") as tmp:
|
| 892 | write_files(Path(tmp), oracle["final_files"])
|
| 893 | final_green = run_check(Path(tmp))[0]
|
| 894 | lost = [a.name for a in ordinary if a.outcome != "landed"]
|
| 895 | rows.append(
|
| 896 | {
|
| 897 | **common,
|
| 898 | "operation": "land.trial",
|
| 899 | "elapsed_ms": round((end - start) * 1000, 1),
|
| 900 | "returncode": 0 if (classification["intended_changes_exact"] and oracle["red"] == 0 and not lost) else 1,
|
| 901 | "setup_ms": round(setup["setup_ms"], 1),
|
| 902 | "main_commits_before": setup["main_commits_before"],
|
| 903 | "main_heads_created": len(oracle["heads"]),
|
| 904 | "red_main_events": oracle["red"],
|
| 905 | "final_main_green": final_green,
|
| 906 | **classification,
|
| 907 | "ordinary_landed": len(times),
|
| 908 | "ordinary_lost": lost,
|
| 909 | "failed_agents": [a.name for a in agents if a.outcome == "failed"],
|
| 910 | "retries_total": sum(max(0, a.attempts - 1) for a in agents),
|
| 911 | "rate_limited_retries_total": sum(a.rate_limited for a in agents) if is_oak else None,
|
| 912 | "conflict_retries_total": sum(a.conflict_retries for a in agents) if is_oak else None,
|
| 913 | "tool_call_count": sum(a.tool_calls for a in agents),
|
| 914 | "output_bytes": sum(a.output_bytes for a in agents),
|
| 915 | "ci_runs_total": total_ci,
|
| 916 | "time_to_landed_ms_p50": percentile(times, 50),
|
| 917 | "time_to_landed_ms_max": max(times) if times else None,
|
| 918 | "combined_tested_fraction": (sum(1 for f in tested_flags if f) / len(tested_flags)) if tested_flags else None,
|
| 919 | "landed_total": len(landed),
|
| 920 | "human_interventions": 0,
|
| 921 | "human_interventions_source": "scripted_harness_has_no_human_step",
|
| 922 | "command": arm,
|
| 923 | }
|
| 924 | )
|
| 925 | return rows
|
| 926 |
|
| 927 |
|
| 928 | def rand_suffix() -> str:
|
| 929 | return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(6))
|
| 930 |
|
| 931 |
|
| 932 | def run_trial(args: argparse.Namespace, arm: str, n: int, k: int, run_index: int, stub: Optional[CiStub], meta: dict[str, Any], subject: tuple[str, str, Optional[Path]]) -> list[dict[str, Any]]:
|
| 933 | subject_name, subject_kind, oak_bin = subject
|
| 934 | root = Path(tempfile.mkdtemp(prefix=f"sl-{arm}-n{n}-k{k}-r{run_index}-", dir=args.scratch))
|
| 935 | agents = [AgentRun(name=name, arm=arm) for name in agent_names(n)]
|
| 936 | if arm in OAK_ARMS:
|
| 937 | assert stub is not None and oak_bin is not None
|
| 938 | server = OakServer(url=args.oak_url, token=Path(args.token_file).read_text().strip(), secret=os.environ[args.secret_env],
|
| 939 | owner=args.owner, psql=args.psql.split(), home=Path(args.oak_home))
|
| 940 | trial = OakTrial(server=server, oak=oak_bin, setup_oak=Path(args.setup_oak), root=root,
|
| 941 | repo=f"sl-{arm.replace('_', '')}-{subject_name.replace('_', '')}-n{n}-k{k}-r{run_index}-{rand_suffix()}",
|
| 942 | n=n, k=k, arm=arm, stub=stub, setup_ledger=root / "ci-setup.jsonl", trial_ledger=root / "ci-trial.jsonl")
|
| 943 | setup = trial.setup()
|
| 944 | for agent in agents:
|
| 945 | agent.branch = trial.branches.get(agent.name)
|
| 946 | flow = trial.flow_naive if arm == "oak_naive" else trial.flow_safe
|
| 947 | with measurement_lock(f"stale_land:{arm}:n{n}:k{k}:r{run_index}") as lock:
|
| 948 | start, end = run_agents(agents, flow)
|
| 949 | stub.join_runs(timeout=60)
|
| 950 | oracle = trial.oracle(agents)
|
| 951 | binary_hash = binary_sha256(oak_bin)
|
| 952 | else:
|
| 953 | trial_g = GitTrial(root=root, n=n, k=k, arm=arm, ci_added_seconds=args.ci_added_seconds)
|
| 954 | setup = trial_g.setup()
|
| 955 | with measurement_lock(f"stale_land:{arm}:n{n}:k{k}:r{run_index}") as lock:
|
| 956 | start, end = run_agents(agents, trial_g.flow)
|
| 957 | oracle = trial_g.oracle(agents)
|
| 958 | binary_hash = binary_sha256(Path(shutil.which("git") or "git"))
|
| 959 | rows = build_rows(meta={**meta, "lock_wait_ms": lock.wait_ms, "lock_enabled": lock.enabled, "trial_root": str(root)}, arm=arm, n=n, k=k,
|
| 960 | run_index=run_index, agents=agents, start=start, end=end, setup=setup, oracle=oracle,
|
| 961 | subject=subject_name, subject_kind=subject_kind, binary_hash=binary_hash)
|
| 962 | if not args.keep_trials:
|
| 963 | shutil.rmtree(root, ignore_errors=True)
|
| 964 | return rows
|
| 965 |
|
| 966 |
|
| 967 | def summary_text(rows: list[dict[str, Any]]) -> str:
|
| 968 | trials = [r for r in rows if r["operation"] == "land.trial"]
|
| 969 | groups: dict[tuple[Any, ...], list[dict[str, Any]]] = {}
|
| 970 | for row in trials:
|
| 971 | groups.setdefault((row["contention_mode"], row["subject"], row["workers"], row["behind_k"], row.get("aa_group")), []).append(row)
|
| 972 | lines = [
|
| 973 | f"# {SCENARIO} baseline",
|
| 974 | "",
|
| 975 | f"CI added delay: {trials[0].get('ci_added_seconds') if trials else None} s (labeled). Oak rows: loopback, hosted-equivalent-local. Git rows: local-file.",
|
| 976 | "",
|
| 977 | "| arm | subject | N | k | trials | median p50 time-to-landed s | median makespan s | CI runs (median) | tool calls (median) | combined-tested % | red-main events | exact final | failures/lost | retries (median) |",
|
| 978 | "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|",
|
| 979 | ]
|
| 980 |
|
| 981 | def med(values: list[Any]) -> Optional[float]:
|
| 982 | clean = [v for v in values if v is not None]
|
| 983 | return statistics.median(clean) if clean else None
|
| 984 |
|
| 985 | def fmt(value: Optional[float], scale: float = 1.0, digits: int = 1) -> str:
|
| 986 | return "n/a" if value is None else f"{value / scale:.{digits}f}"
|
| 987 |
|
| 988 | for (arm, subject, n, k, aa), group in sorted(groups.items(), key=lambda kv: [str(x) for x in kv[0]]):
|
| 989 | tested = [r["combined_tested_fraction"] for r in group if r["combined_tested_fraction"] is not None]
|
| 990 | lines.append(
|
| 991 | "| {arm}{aa} | {subject} | {n} | {k} | {t} | {p50} | {mk} | {ci} | {tc} | {tested} | {red} | {exact}/{t} | {fail} | {retries} |".format(
|
| 992 | arm=arm, aa=f" ({aa})" if aa else "", subject=subject, n=n, k=k, t=len(group),
|
| 993 | p50=fmt(med([r["time_to_landed_ms_p50"] for r in group]), 1000),
|
| 994 | mk=fmt(med([r["elapsed_ms"] for r in group]), 1000),
|
| 995 | ci=fmt(med([r["ci_runs_total"] for r in group]), 1, 0),
|
| 996 | tc=fmt(med([r["tool_call_count"] for r in group]), 1, 0),
|
| 997 | tested="n/a" if not tested else f"{100 * statistics.mean(tested):.0f}%",
|
| 998 | red=sum(r["red_main_events"] for r in group),
|
| 999 | exact=sum(1 for r in group if r["intended_changes_exact"]),
|
| 1000 | fail=sum(len(r["failed_agents"]) + len(r["ordinary_lost"]) for r in group),
|
| 1001 | retries=fmt(med([r["retries_total"] for r in group]), 1, 0),
|
| 1002 | )
|
| 1003 | )
|
| 1004 | return "\n".join(lines) + "\n"
|
| 1005 |
|
| 1006 |
|
| 1007 | def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:
|
| 1008 | parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 1009 | parser.add_argument("--arms", default=",".join(ARMS))
|
| 1010 | parser.add_argument("--agents", default="1,4", help="comma list of N ordinary agents (the pair is added to each)")
|
| 1011 | parser.add_argument("--behind", default="1,50", help="comma list of k commits main moved after the fork")
|
| 1012 | parser.add_argument("--runs", type=int, default=3)
|
| 1013 | parser.add_argument("--aa-group", default=None, help="label for A/A noise-floor batches")
|
| 1014 | parser.add_argument("--ci-added-seconds", type=float, default=10.0)
|
| 1015 | parser.add_argument("--oak-bin", action="append", default=[], help="NAME=PATH oak client subject (repeatable)")
|
| 1016 | parser.add_argument("--setup-oak", default=None, help="oak binary for untimed seed/mover setup")
|
| 1017 | parser.add_argument("--oak-url", default="http://127.0.0.1:3811")
|
| 1018 | parser.add_argument("--owner", default="bench")
|
| 1019 | parser.add_argument("--token-file", default=None)
|
| 1020 | parser.add_argument("--oak-home", default=None, help="isolated HOME for oak clients")
|
| 1021 | parser.add_argument("--secret-env", default="OAK_CI_SIGNING_SECRET")
|
| 1022 | parser.add_argument("--ci-stub-port", type=int, default=3812, help="port the server's OAK_CI_DISPATCH_URL points at")
|
| 1023 | parser.add_argument("--psql", default="psql -h 127.0.0.1 -p 55811 -U oak oak")
|
| 1024 | parser.add_argument("--server-binary", default=None, help="oakspace server binary (provenance only)")
|
| 1025 | parser.add_argument("--server-source", default=None, help="oakspace source identity (provenance only)")
|
| 1026 | parser.add_argument("--scratch", default=None)
|
| 1027 | parser.add_argument("--results", type=Path, default=Path("results/stale-land"))
|
| 1028 | parser.add_argument("--keep-trials", action="store_true")
|
| 1029 | parser.add_argument("--dry-run", action="store_true", help="print the trial schedule and exit")
|
| 1030 | return parser.parse_args(argv)
|
| 1031 |
|
| 1032 |
|
| 1033 | def schedule(args: argparse.Namespace) -> list[tuple[str, tuple[str, str, Optional[Path]], int, int, int]]:
|
| 1034 | arms = [a for a in args.arms.split(",") if a]
|
| 1035 | for arm in arms:
|
| 1036 | if arm not in ARMS:
|
| 1037 | raise SystemExit(f"unknown arm {arm!r}; known: {', '.join(ARMS)}")
|
| 1038 | oak_subjects: list[tuple[str, str, Optional[Path]]] = []
|
| 1039 | for spec in args.oak_bin:
|
| 1040 | name, _, path = spec.partition("=")
|
| 1041 | oak_subjects.append((name, "oak", Path(path)))
|
| 1042 | plan = []
|
| 1043 | for n in [int(x) for x in args.agents.split(",")]:
|
| 1044 | for k in [int(x) for x in args.behind.split(",")]:
|
| 1045 | for run_index in range(1, args.runs + 1):
|
| 1046 | for arm in arms:
|
| 1047 | subjects = oak_subjects if arm in OAK_ARMS else [("git", "git", None)]
|
| 1048 | for subject in subjects:
|
| 1049 | plan.append((arm, subject, n, k, run_index))
|
| 1050 | return plan
|
| 1051 |
|
| 1052 |
|
| 1053 | def main(argv: Optional[list[str]] = None) -> int:
|
| 1054 | args = parse_args(argv)
|
| 1055 | plan = schedule(args)
|
| 1056 | if args.dry_run:
|
| 1057 | for item in plan:
|
| 1058 | print(item[0], item[1][0], f"n={item[2]}", f"k={item[3]}", f"run={item[4]}")
|
| 1059 | return 0
|
| 1060 | needs_oak = any(item[0] in OAK_ARMS for item in plan)
|
| 1061 | stub = None
|
| 1062 | if needs_oak:
|
| 1063 | if not (args.token_file and args.oak_home and args.setup_oak and os.environ.get(args.secret_env)):
|
| 1064 | raise SystemExit("oak arms need --token-file, --oak-home, --setup-oak and $" + args.secret_env)
|
| 1065 | stub = CiStub(api_url=args.oak_url, secret=os.environ[args.secret_env], ledger=Path(os.devnull),
|
| 1066 | added_seconds=args.ci_added_seconds, port=args.ci_stub_port).start()
|
| 1067 | bench_id = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + f"-{SCENARIO}"
|
| 1068 | meta = {
|
| 1069 | "bench_id": bench_id,
|
| 1070 | "ci_added_seconds": args.ci_added_seconds,
|
| 1071 | "aa_group": args.aa_group,
|
| 1072 | "host": platform.node(),
|
| 1073 | "server_binary_sha256": binary_sha256(Path(args.server_binary)) if args.server_binary else None,
|
| 1074 | "server_source": args.server_source,
|
| 1075 | }
|
| 1076 | store = ResultsStore(args.results, lane="contention", filename_suffix="stale-land", flush_partial=True)
|
| 1077 | rows: list[dict[str, Any]] = []
|
| 1078 | try:
|
| 1079 | for arm, subject, n, k, run_index in plan:
|
| 1080 | label = f"{arm} {subject[0]} n={n} k={k} run={run_index}"
|
| 1081 | print(f"[stale_land] {label} ...", file=sys.stderr, flush=True)
|
| 1082 | try:
|
| 1083 | trial_rows = run_trial(args, arm, n, k, run_index, stub, meta, subject)
|
| 1084 | except Exception as exc: # noqa: BLE001 - a setup failure is a recorded row, not a silent gap
|
| 1085 | trial_rows = [{
|
| 1086 | **meta, "profile": PROFILE, "scenario": SCENARIO, "scenario_version": SCENARIO_VERSION,
|
| 1087 | "contention_mode": arm, "workers": n, "behind_k": k, "operation": "land.trial", "run": run_index,
|
| 1088 | "subject": subject[0], "subject_kind": subject[1], "elapsed_ms": 0.0, "returncode": 1,
|
| 1089 | "error": f"trial setup/oracle failed: {exc}", "command": arm,
|
| 1090 | }]
|
| 1091 | trial = trial_rows[-1]
|
| 1092 | print(
|
| 1093 | f"[stale_land] {label}: rc={trial['returncode']} makespan={trial['elapsed_ms'] / 1000:.1f}s "
|
| 1094 | f"p50={trial.get('time_to_landed_ms_p50')} ci={trial.get('ci_runs_total')} red={trial.get('red_main_events')} "
|
| 1095 | f"exact={trial.get('intended_changes_exact')} tested={trial.get('combined_tested_fraction')} err={trial.get('error')}",
|
| 1096 | file=sys.stderr, flush=True,
|
| 1097 | )
|
| 1098 | store.append_partial(bench_id, trial_rows)
|
| 1099 | rows.extend(trial_rows)
|
| 1100 | finally:
|
| 1101 | if stub is not None:
|
| 1102 | stub.stop()
|
| 1103 | complete = [r for r in rows if "tool_call_count" in r or r["operation"] == "land.agent"]
|
| 1104 | raw, summary = store.write(bench_id, rows, summary_text([r for r in complete if r["operation"] == "land.trial"] or []))
|
| 1105 | print(f"[stale_land] rows: {raw}\n[stale_land] summary: {summary}", file=sys.stderr)
|
| 1106 | return 0
|
| 1107 |
|
| 1108 |
|
| 1109 | if __name__ == "__main__":
|
| 1110 | raise SystemExit(main())
|