Viewing files at commit 39bc21f7f67eCommit detailsView latest

Browse files
scripts/task_loop.py 2639 lines · 106.0 KB · python
1
#!/usr/bin/env python3
2
"""The full task loop, end to end — the product's signature motion.
3
4
oak:  mount -> edit -> commit -> push -> desc -> end, then REMOUNT for a
5
      follow-up task (the warm-cache story) and repeat edit/commit/push/end.
6
git:  worktree add -> branch edit -> add/commit -> push -> worktree remove,
7
      then a second worktree for the follow-up. The task description lives in
8
      the commit message git already paid for; oak pays a separate `desc`
9
      call. That asymmetry is the measurement, not a harness gap
10
      (docs/command-semantics.md): this is agent-default-track, whole-task
11
      evidence, never a step-by-step equivalence claim.
12
13
Transport honesty: oak rows always cross the network to a real oak server.
14
git runs against GitHub (--git-remote github, transport-comparable) or a
15
local bare repo (--git-remote local, the core-equivalent track). Rows carry
16
remote_transport/remote_server; latency comparisons across transports are
17
forbidden — cross-subject evidence at unlike transports is tool calls and
18
tokens only.
19
20
Setup:
21
  OAK_BENCH_SYNC_REPO=oak/bench-sync-tmp \\
22
  [email protected]:oakdotspace/bench-task-loop-tmp.git \\
23
  python3 scripts/task_loop.py --runs 3
24
25
Unconfigured remotes emit skip rows (returncode 77), never silent absence.
26
"""
27
28
from __future__ import annotations
29
30
import argparse
31
import hashlib
32
import json
33
import os
34
import platform
35
import random
36
import re
37
import shutil
38
import socket
39
import sqlite3
40
import stat
41
import subprocess
42
import tempfile
43
import threading
44
import time
45
import urllib.error
46
import urllib.parse
47
import urllib.request
48
from datetime import datetime, timezone
49
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
50
from pathlib import Path
51
from typing import Any
52
53
from oakbench import remotes as oakbench_remotes
54
from oakbench import tokens as oakbench_tokens
55
from oakbench.byteproxy import CountingProxy
56
from oakbench.diskprobe import bounded_tree_usage
57
from oakbench.environment import ENV_ISOLATION_VERSION, base_env as oakbench_base_env
58
from oakbench.environment import command_display
59
from oakbench.execution import run_timed, stop_owned_process
60
from oakbench.results import ResultsStore
61
from oakbench.rows import row_returncode
62
from oakbench.runlock import measurement_lock
63
from oakbench.subjects import (
64
    DEFAULT_OAK_REPO,
65
    Subject,
66
    binary_sha256,
67
    load_subjects,
68
    source_metadata,
69
    subject_details,
70
    subject_version,
71
    subject_versions,
72
)
73
74
ROOT = Path(__file__).resolve().parents[1]
75
DEFAULT_WORKDIR = Path(tempfile.gettempdir()) / "oak-task-loop"
76
SKIP_RETURNCODE = 77
77
VIRTUAL_BRANCH_RE = re.compile(r"virtual branch ([^\s)]+)")
78
PHASES = ("task", "followup")
79
TASK_LOOP_SCENARIO = "task_loop"
80
AGENT_TASK_SCENARIO = "agent_task_baseline_v1"
81
AGENT_TASK_OAK_LOCAL_SCENARIO = "agent_task_oak_local_v1"
82
INTERRUPTED_PUBLICATION_SCENARIO = "interrupted_publication_after_request_v1"
83
LEGACY_SELECTED_MATERIALIZATION_SCENARIO = "legacy_selected_clone_materialization_v1"
84
INTERRUPTED_PUBLICATION_REQUEST_TIMEOUT_S = 2.0
85
INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S = 30.0
86
INTERRUPTED_PUBLICATION_MAX_REQUEST_BYTES = 64 * 1024 * 1024
87
LOCAL_OAK_SERVER_READY_TIMEOUT_S = 10.0
88
89
90
def legacy_selected_receipt_evidence(receipt: dict[str, Any]) -> dict[str, Any]:
91
    result = receipt.get("result") if isinstance(receipt.get("result"), dict) else {}
92
    observed = receipt.get("observed") if isinstance(receipt.get("observed"), dict) else {}
93
    phases = observed.get("pull_phases") if isinstance(observed.get("pull_phases"), list) else []
94
    phase_names = [phase.get("name") for phase in phases if isinstance(phase, dict)]
95
    materialization = (
96
        observed.get("materialization")
97
        if isinstance(observed.get("materialization"), dict)
98
        else {}
99
    )
100
    return {
101
        "legacy_route_observed": phase_names == ["initial", "legacy_selected_branch"],
102
        "pull_phase_names": phase_names,
103
        "pull_phases": phases,
104
        "materialization_passes_observed": materialization.get("passes_observed"),
105
        "written_paths": materialization.get("written_paths"),
106
        "snapshot_bound": result.get("snapshot_bound"),
107
        "head": result.get("head"),
108
        "manifest": result.get("manifest"),
109
    }
110
111
112
def legacy_selected_receipt_contract(evidence: dict[str, Any]) -> dict[str, Any]:
113
    phases = []
114
    for phase in evidence.get("pull_phases", []):
115
        if not isinstance(phase, dict):
116
            continue
117
        phases.append(
118
            {
119
                key: value
120
                for key, value in phase.items()
121
                if key not in {"materialization_passes"}
122
            }
123
        )
124
    return {
125
        "legacy_route_observed": evidence.get("legacy_route_observed"),
126
        "pull_phases": phases,
127
        "snapshot_bound": evidence.get("snapshot_bound"),
128
        "head": evidence.get("head"),
129
        "manifest": evidence.get("manifest"),
130
        "written_paths": evidence.get("written_paths"),
131
    }
132
133
134
def normalized_cache_contract(snapshot: dict[str, Any] | None) -> list[dict[str, Any]] | None:
135
    if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
136
        return None
137
    return [
138
        {key: row.get(key) for key in ("path", "size", "blob_hash")}
139
        for row in snapshot["rows"]
140
        if isinstance(row, dict)
141
    ]
142
143
144
def normalized_status(value: dict[str, Any] | list[Any] | None) -> dict[str, Any] | None:
145
    if not isinstance(value, dict):
146
        return None
147
    return {
148
        key: value.get(key)
149
        for key in ("branch", "parent", "branch_status", "head", "working_changes", "branch_changes")
150
    }
151
152
153
def materialization_timing_controls_met(
154
    metadata: dict[str, Any], runs: int, randomize_order: bool, controls_requested: bool
155
) -> bool:
156
    return bool(
157
        controls_requested
158
        and metadata.get("measurement_lock") == "held"
159
        and randomize_order
160
        and runs >= 10
161
    )
162
163
164
def isolated_oak_fixture_env(task_config_root: Path) -> dict[str, str]:
165
    task_config_root.mkdir(mode=0o700, parents=True, exist_ok=True)
166
    retained = {
167
        "PATH",
168
        "TMPDIR",
169
        "TMP",
170
        "TEMP",
171
        "HOME",
172
        "USER",
173
        "USERNAME",
174
        "SHELL",
175
        "SYSTEMROOT",
176
        "WINDIR",
177
        "COMSPEC",
178
        "PATHEXT",
179
        "SSL_CERT_FILE",
180
        "SSL_CERT_DIR",
181
    }
182
    env = {key: value for key, value in os.environ.items() if key in retained}
183
    env.update(
184
        {
185
            "GIT_AUTHOR_NAME": "Oak Materialization Bench",
186
            "GIT_AUTHOR_EMAIL": "[email protected]",
187
            "GIT_COMMITTER_NAME": "Oak Materialization Bench",
188
            "GIT_COMMITTER_EMAIL": "[email protected]",
189
            "GIT_CONFIG_GLOBAL": os.devnull,
190
            "GIT_CONFIG_NOSYSTEM": "1",
191
            "OAK_AUTHOR": "oak-materialization-bench",
192
            "OAK_NO_UPDATE_CHECK": "1",
193
            "OAK_BENCH_ENV_ISOLATION_VERSION": ENV_ISOLATION_VERSION,
194
            "NO_COLOR": "1",
195
            "CLICOLOR": "0",
196
            "LC_ALL": "C",
197
            "LANG": "C",
198
            "TZ": "UTC",
199
            "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z",
200
            "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z",
201
        }
202
    )
203
    return env
204
205
206
def normalized_worktree_snapshot(root: Path) -> list[dict[str, Any]]:
207
    rows: list[dict[str, Any]] = []
208
    for path in sorted(root.rglob("*")):
209
        relative = path.relative_to(root)
210
        if relative.parts and relative.parts[0] in {".oak", ".git"}:
211
            continue
212
        info = path.lstat()
213
        if stat.S_ISDIR(info.st_mode):
214
            continue
215
        row: dict[str, Any] = {
216
            "path": relative.as_posix(),
217
            "mode": stat.S_IMODE(info.st_mode),
218
        }
219
        if stat.S_ISLNK(info.st_mode):
220
            row.update({"kind": "symlink", "target": os.readlink(path)})
221
        else:
222
            row.update({"kind": "file", "sha256": hashlib.sha256(path.read_bytes()).hexdigest()})
223
        rows.append(row)
224
    return rows
225
226
227
def json_command(
228
    command: list[str], cwd: Path, env: dict[str, str]
229
) -> dict[str, Any] | list[Any] | None:
230
    result = run_bounded_harness_command(command, cwd, env=env)
231
    if result.returncode != 0:
232
        return None
233
    try:
234
        value = json.loads(result.stdout)
235
    except (TypeError, ValueError):
236
        return None
237
    return value if isinstance(value, (dict, list)) else None
238
239
240
def normalized_history(value: dict[str, Any] | list[Any] | None) -> list[dict[str, Any]] | None:
241
    if not isinstance(value, list):
242
        return None
243
    rows: list[dict[str, Any]] = []
244
    for item in value:
245
        if not isinstance(item, dict):
246
            return None
247
        rows.append(
248
            {
249
                key: item.get(key)
250
                for key in (
251
                    "hash",
252
                    "timestamp",
253
                    "branch",
254
                    "description_or_subject",
255
                    "files_changed",
256
                )
257
            }
258
        )
259
    return rows
260
261
262
def normalized_oak_artifacts(root: Path) -> list[str] | None:
263
    oak_dir = root / ".oak"
264
    if not oak_dir.is_dir():
265
        return None
266
    return sorted(
267
        path.relative_to(oak_dir).as_posix() + ("/" if path.is_dir() else "")
268
        for path in oak_dir.rglob("*")
269
    )
270
271
272
def stat_cache_snapshot(root: Path) -> dict[str, Any] | None:
273
    db = root / ".oak/oak.db"
274
    if not db.is_file():
275
        return {"rows": None, "all_metadata_valid": False, "error": "oak_db_missing"}
276
    try:
277
        connection = sqlite3.connect(str(db))
278
        connection.execute("PRAGMA query_only = ON")
279
        raw_rows = connection.execute(
280
            "SELECT path, mtime_ns, ctime_ns, size, blob_hash FROM stat_cache ORDER BY path"
281
        ).fetchall()
282
    except sqlite3.Error as error:
283
        return {"rows": None, "all_metadata_valid": False, "error": str(error)}
284
    finally:
285
        if "connection" in locals():
286
            connection.close()
287
    rows: list[dict[str, Any]] = []
288
    valid = True
289
    for path, mtime_ns, ctime_ns, size, blob_hash in raw_rows:
290
        file_path = root / path
291
        try:
292
            info = file_path.stat()
293
        except OSError:
294
            valid = False
295
            continue
296
        expected_ctime = info.st_ctime_ns if os.name == "posix" else 0
297
        metadata_valid = (
298
            int(size) == info.st_size
299
            and int(mtime_ns) == info.st_mtime_ns
300
            and int(ctime_ns) == expected_ctime
301
        )
302
        valid = valid and metadata_valid
303
        rows.append(
304
            {
305
                "path": str(path),
306
                "size": int(size),
307
                "blob_hash": str(blob_hash),
308
                "metadata_valid": metadata_valid,
309
            }
310
        )
311
    return {"rows": rows, "all_metadata_valid": valid}
312
313
314
def base_env() -> dict[str, str]:
315
    return oakbench_base_env(
316
        author_name="Oak Task Loop",
317
        author_email="[email protected]",
318
        oak_author="oak-task-loop",
319
    )
320
321
322
def parse_args() -> argparse.Namespace:
323
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
324
    parser.add_argument("--subjects", help="Comma-separated subject names from config/subjects.toml")
325
    parser.add_argument("--config", type=Path, default=ROOT / "config" / "subjects.toml")
326
    parser.add_argument("--git-bin", type=Path)
327
    parser.add_argument("--oak-installed-bin", type=Path)
328
    parser.add_argument("--oak-local-bin", type=Path)
329
    parser.add_argument(
330
        "--oak-serve-bin",
331
        type=Path,
332
        help="Immutable Oak binary used only as the fixed loopback server and fixture writer.",
333
    )
334
    parser.add_argument(
335
        "--oak-serve-source",
336
        type=Path,
337
        help="Read-only source checkout that produced --oak-serve-bin.",
338
    )
339
    parser.add_argument(
340
        "--oak-serve-source-head",
341
        help="Exact 64-hex Oak source head that produced --oak-serve-bin.",
342
    )
343
    parser.add_argument(
344
        "--materialization-baseline-subject",
345
        help="Named control subject required by the legacy materialization scenario.",
346
    )
347
    parser.add_argument(
348
        "--materialization-timing-controls-met",
349
        action="store_true",
350
        help=(
351
            "Record that harness-verifiable controls are met only with a held measurement lock, "
352
            "randomized subject order, and at least 10 runs. This is not a speedup claim: "
353
            "separate A/A noise and externally verified quiet-host evidence remain required."
354
        ),
355
    )
356
    parser.add_argument("--runs", type=int, default=3)
357
    parser.add_argument(
358
        "--scenarios",
359
        default=TASK_LOOP_SCENARIO,
360
        help=(
361
            "Comma-separated scenarios: task_loop, agent_task_baseline_v1, "
362
            "agent_task_oak_local_v1, legacy_selected_clone_materialization_v1, "
363
            "and/or interrupted_publication_after_request_v1. "
364
            "The interruption "
365
            "scenario is a loopback-only Oak fixture and never contacts a remote service."
366
        ),
367
    )
368
    parser.add_argument(
369
        "--git-remote",
370
        choices=("github", "local"),
371
        default="github",
372
        help=(
373
            "github: agent-default track, real push to GIT_BENCH_REMOTE "
374
            "(transport-comparable to oak). local: core-equivalent track, "
375
            "local bare remote (no network in git's loop)."
376
        ),
377
    )
378
    parser.add_argument("--workdir", type=Path, default=DEFAULT_WORKDIR)
379
    parser.add_argument("--results", type=Path, default=ROOT / "results" / "task-loop")
380
    parser.add_argument("--oak-repo", type=Path, default=DEFAULT_OAK_REPO)
381
    parser.add_argument("--admitted-output-chars", type=int, default=20_000)
382
    parser.add_argument("--keep-workdirs", action="store_true")
383
    parser.add_argument(
384
        "--randomize-subject-order",
385
        action="store_true",
386
        help="Shuffle subject order independently in each run using --random-seed.",
387
    )
388
    parser.add_argument("--random-seed", type=int, default=20260908)
389
    return parser.parse_args()
390
391
392
def run_untimed(command: list[str], cwd: Path | None, timeout: float = 300.0) -> subprocess.CompletedProcess[str]:
393
    return subprocess.run(
394
        command,
395
        cwd=cwd,
396
        env=base_env(),
397
        text=True,
398
        stdout=subprocess.PIPE,
399
        stderr=subprocess.PIPE,
400
        timeout=timeout,
401
        check=False,
402
    )
403
404
405
def step_row(
406
    subject: Subject,
407
    scenario: str,
408
    operation: str,
409
    command: list[str],
410
    cwd: Path,
411
    run_index: int,
412
    metadata: dict[str, Any],
413
    admitted_output_chars: int,
414
    *,
415
    kind: str = "vcs",
416
    expected_returncodes: tuple[int, ...] = (0,),
417
    preserve_stdout: bool = False,
418
    preserve_stderr: bool = False,
419
    env: dict[str, str] | None = None,
420
    timeout_seconds: float | None = None,
421
) -> dict[str, Any]:
422
    capture = run_timed(
423
        command,
424
        cwd,
425
        env or base_env(),
426
        admitted_output_chars,
427
        timeout_seconds=timeout_seconds,
428
    )
429
    expected = not capture.timed_out and capture.returncode in expected_returncodes
430
    is_vcs = 1 if kind == "vcs" else 0
431
    row = {
432
        **metadata,
433
        "subject": subject.name,
434
        "subject_kind": subject.kind,
435
        "subject_label": subject.label,
436
        "scenario": scenario,
437
        "run": run_index,
438
        "operation": operation,
439
        "step_kind": kind,
440
        "elapsed_ms": round(capture.elapsed_ms, 3),
441
        "returncode": 0 if expected else (capture.returncode or 1),
442
        "process_returncode": capture.returncode,
443
        "timed_out": capture.timed_out,
444
        "expected_returncodes": list(expected_returncodes),
445
        "command": command,
446
        "tool_call_count": 1,
447
        "terminal_tool_call_count": 1,
448
        "vcs_tool_call_count": is_vcs,
449
        **oakbench_tokens.interaction_token_fields(
450
            command_display(command),
451
            capture.stdout_text,
452
            capture.stderr_text,
453
            capture.stdout_bytes,
454
            capture.stderr_bytes,
455
            capture.stdout_truncated,
456
            capture.stderr_truncated,
457
        ),
458
    }
459
    if not expected:
460
        row["stderr"] = capture.stderr_text[-4000:]
461
    if preserve_stdout:
462
        row["stdout"] = capture.stdout_text
463
    if preserve_stderr:
464
        row["stderr"] = capture.stderr_text
465
    return row
466
467
468
def skip_row(
469
    subject: Subject,
470
    scenario: str,
471
    run_index: int,
472
    metadata: dict[str, Any],
473
    reason: str,
474
) -> dict[str, Any]:
475
    return {
476
        **metadata,
477
        "subject": subject.name,
478
        "subject_kind": subject.kind,
479
        "subject_label": subject.label,
480
        "scenario": scenario,
481
        "run": run_index,
482
        "operation": "task_loop.skipped",
483
        "step_kind": "summary",
484
        "elapsed_ms": 0.0,
485
        "returncode": SKIP_RETURNCODE,
486
        "command": [],
487
        "skipped": True,
488
        "skip_reason": reason,
489
        "tool_call_count": 0,
490
        "terminal_tool_call_count": 0,
491
        "vcs_tool_call_count": 0,
492
    }
493
494
495
def unmeasured_step_row(
496
    subject: Subject,
497
    scenario: str,
498
    operation: str,
499
    run_index: int,
500
    metadata: dict[str, Any],
501
    reason: str,
502
) -> dict[str, Any]:
503
    row = skip_row(subject, scenario, run_index, metadata, reason)
504
    row["operation"] = operation
505
    return row
506
507
508
def harness_step_row(
509
    subject: Subject,
510
    scenario: str,
511
    operation: str,
512
    run_index: int,
513
    metadata: dict[str, Any],
514
    elapsed_ms: float,
515
    *,
516
    returncode: int = 0,
517
    harness_command_count: int = 0,
518
) -> dict[str, Any]:
519
    return {
520
        **metadata,
521
        "subject": subject.name,
522
        "subject_kind": subject.kind,
523
        "subject_label": subject.label,
524
        "scenario": scenario,
525
        "run": run_index,
526
        "operation": operation,
527
        "step_kind": "harness",
528
        "elapsed_ms": round(elapsed_ms, 3),
529
        "returncode": returncode,
530
        "command": [],
531
        "tool_call_count": 0,
532
        "terminal_tool_call_count": 0,
533
        "vcs_tool_call_count": 0,
534
        "harness_command_count": harness_command_count,
535
        "stdout_bytes": 0,
536
        "stderr_bytes": 0,
537
        "retry_count": 0,
538
        "human_action_count": 0,
539
    }
540
541
542
def phase_total_row(
543
    rows: list[dict[str, Any]],
544
    subject: Subject,
545
    scenario: str,
546
    run_index: int,
547
    metadata: dict[str, Any],
548
    operation: str,
549
) -> dict[str, Any]:
550
    failures = [row for row in rows if row_returncode(row) not in (0, SKIP_RETURNCODE)]
551
    unmeasured = [
552
        str(row["operation"])
553
        for row in rows
554
        if row_returncode(row) == SKIP_RETURNCODE
555
    ]
556
    phase_elapsed_ms: dict[str, float] = {}
557
    for row in rows:
558
        phase = str(row.get("operation", "unknown")).split(".", 1)[0]
559
        phase_elapsed_ms[phase] = phase_elapsed_ms.get(phase, 0.0) + float(
560
            row.get("elapsed_ms", 0.0)
561
        )
562
    return {
563
        **metadata,
564
        "subject": subject.name,
565
        "subject_kind": subject.kind,
566
        "subject_label": subject.label,
567
        "scenario": scenario,
568
        "run": run_index,
569
        "operation": operation,
570
        "step_kind": "summary",
571
        "elapsed_ms": round(sum(float(row.get("elapsed_ms", 0)) for row in rows), 3),
572
        "returncode": 1 if failures else 0,
573
        "coverage_complete": not unmeasured,
574
        "unmeasured_operations": unmeasured,
575
        "command": [row["command"] for row in rows],
576
        "tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("tool_call_count")) for row in rows),
577
        "terminal_tool_call_count": sum(
578
            oakbench_tokens.int_or_zero(row.get("terminal_tool_call_count")) for row in rows
579
        ),
580
        "vcs_tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("vcs_tool_call_count")) for row in rows),
581
        "agent_tool_call_count": sum(
582
            oakbench_tokens.int_or_zero(row.get("tool_call_count")) for row in rows
583
        ),
584
        "harness_command_count": sum(
585
            oakbench_tokens.int_or_zero(row.get("harness_command_count")) for row in rows
586
        ),
587
        "retry_count": sum(oakbench_tokens.int_or_zero(row.get("retry_count")) for row in rows),
588
        "human_action_count": sum(
589
            oakbench_tokens.int_or_zero(row.get("human_action_count")) for row in rows
590
        ),
591
        "phase_elapsed_ms": {
592
            phase: round(elapsed, 3) for phase, elapsed in sorted(phase_elapsed_ms.items())
593
        },
594
        **oakbench_tokens.summed_token_fields(
595
            rows, "sum_of_task_loop_steps_command_plus_admitted_output_chars_div_4"
596
        ),
597
        "summarized_operations": [row["operation"] for row in rows],
598
    }
599
600
601
def has_lane_failures(rows: list[dict[str, Any]]) -> bool:
602
    return any(row_returncode(row) not in (0, SKIP_RETURNCODE) for row in rows)
603
604
605
def enforce_oracle_result(total: dict[str, Any], oracle_ok: bool | None) -> dict[str, Any]:
606
    if oracle_ok is False:
607
        total["returncode"] = 1
608
        total["stderr"] = "oracle_payload_missing_from_remote_branch"
609
    return total
610
611
612
def interrupted_publication_fields(
613
    *,
614
    request_count: int,
615
    request_bytes: int | None,
616
    declared_request_bytes: int | None,
617
    request_framing_valid: bool,
618
    request_body_timed_out: bool,
619
    local_head_before: str | None,
620
    local_head_after: str | None,
621
    status_after: dict[str, Any] | None,
622
    worktree_payload_before: str | None,
623
    worktree_payload_after: str | None,
624
    cli_observation: dict[str, str],
625
) -> dict[str, Any]:
626
    """Evidence fields for a single interrupted mutation attempt.
627
628
    The fixture's receipt of a full request is ground truth for the harness,
629
    not a server durability receipt available to the CLI.  A disconnect after
630
    that point is therefore unknown to the client and must never imply replay.
631
    """
632
    request_complete = bool(
633
        request_count == 1
634
        and request_framing_valid
635
        and declared_request_bytes is not None
636
        and request_bytes == declared_request_bytes
637
    )
638
    if request_count == 0:
639
        delivery_state = "no_request_observed"
640
        fixture_outcome = "not_sent_observed"
641
    elif not request_framing_valid:
642
        delivery_state = "invalid_request_framing"
643
        fixture_outcome = "indeterminate_after_invalid_request_disconnect"
644
    elif request_complete:
645
        delivery_state = "complete_request_observed"
646
        fixture_outcome = "unknown_after_complete_request_disconnect"
647
    else:
648
        delivery_state = "partial_request_observed"
649
        fixture_outcome = "indeterminate_after_partial_request_disconnect"
650
    payload_preserved = bool(
651
        worktree_payload_before
652
        and worktree_payload_before == worktree_payload_after
653
    )
654
    unpublished_state_preserved = bool(status_after and status_after.get("needs_push"))
655
    return {
656
        "publication_outcome": cli_observation["reported_outcome"],
657
        "cli_observation": cli_observation["observation"],
658
        "fixture_delivery_state": delivery_state,
659
        "fixture_outcome_knowledge": fixture_outcome,
660
        "fixture_received_complete_request_before_disconnect": request_complete,
661
        "fixture_request_framing_valid": request_framing_valid,
662
        "fixture_request_body_timed_out": request_body_timed_out,
663
        "fixture_declared_request_bytes": declared_request_bytes,
664
        "mutation_requests_observed": request_count,
665
        "blind_replay_requests": max(0, request_count - 1),
666
        "network_request_bytes_observed": request_bytes,
667
        "local_head_preserved": bool(local_head_before) and local_head_before == local_head_after,
668
        "worktree_payload_preserved": payload_preserved,
669
        "unpublished_state_preserved": unpublished_state_preserved,
670
        "local_work_preserved": payload_preserved and unpublished_state_preserved,
671
        "working_tree_dirty_after": None if status_after is None else status_after.get("dirty"),
672
    }
673
674
675
def cli_publication_observation(row: dict[str, Any], *, json_requested: bool) -> dict[str, str]:
676
    """Classify only what the CLI reported, never what the fault fixture knows."""
677
    if row.get("timed_out"):
678
        return {"reported_outcome": "unreported", "observation": "process_timeout"}
679
    stdout = str(row.get("stdout") or "")
680
    stderr = str(row.get("stderr") or "")
681
    value = _parse_json_object(stdout) if json_requested else None
682
    if value is not None and row.get("process_returncode") == 0 and value.get("published") is True:
683
        return {"reported_outcome": "published", "observation": "json_success_receipt"}
684
    if value is not None and isinstance(value.get("error"), dict):
685
        message = str(value["error"].get("message") or "").lower()
686
        if any(marker in message for marker in ("unconfirmed", "unknown", "may have")):
687
            return {"reported_outcome": "unknown", "observation": "json_error_unknown"}
688
        return {
689
            "reported_outcome": "unreported",
690
            "observation": "json_error_without_publication_outcome",
691
        }
692
    message = f"{stdout}\n{stderr}".lower()
693
    if any(marker in message for marker in ("unconfirmed", "unknown", "may have")):
694
        return {"reported_outcome": "unknown", "observation": "text_error_unknown"}
695
    return {"reported_outcome": "unreported", "observation": "text_error_without_publication_outcome"}
696
697
698
def file_sha256(path: Path) -> str | None:
699
    try:
700
        return hashlib.sha256(path.read_bytes()).hexdigest()
701
    except OSError:
702
        return None
703
704
705
class _DisconnectAfterPushHandler(BaseHTTPRequestHandler):
706
    """Minimal loopback fixture for a complete request with a lost reply."""
707
708
    server_version = "oakbench-interruption/1"
709
710
    def log_message(self, _format: str, *_args: object) -> None:
711
        return
712
713
    def _json(self, status: int, payload: dict[str, Any]) -> None:
714
        encoded = json.dumps(payload).encode()
715
        self.send_response(status)
716
        self.send_header("Content-Type", "application/json")
717
        self.send_header("Content-Length", str(len(encoded)))
718
        self.end_headers()
719
        self.wfile.write(encoded)
720
721
    def do_GET(self) -> None:  # noqa: N802 - BaseHTTPRequestHandler API
722
        self._json(404, {"error": "absent synthetic repo"})
723
724
    def do_POST(self) -> None:  # noqa: N802 - BaseHTTPRequestHandler API
725
        declared_size: int | None = None
726
        framing_valid = False
727
        body = b""
728
        body_timed_out = False
729
        raw_sizes = self.headers.get_all("Content-Length", failobj=[])
730
        try:
731
            raw_size = raw_sizes[0].strip(" \t") if len(raw_sizes) == 1 else ""
732
            parsed_size = int(raw_size) if raw_size.isascii() and raw_size.isdecimal() else -1
733
            framing_valid = (
734
                0 <= parsed_size <= INTERRUPTED_PUBLICATION_MAX_REQUEST_BYTES
735
                and self.headers.get("Transfer-Encoding") is None
736
            )
737
            if framing_valid:
738
                declared_size = parsed_size
739
                self.connection.settimeout(INTERRUPTED_PUBLICATION_REQUEST_TIMEOUT_S)
740
                while len(body) < parsed_size:
741
                    try:
742
                        chunk = self.rfile.read1(parsed_size - len(body))
743
                    except (socket.timeout, TimeoutError):
744
                        body_timed_out = True
745
                        break
746
                    except OSError:
747
                        break
748
                    if not chunk:
749
                        break
750
                    body += chunk
751
        except ValueError:
752
            framing_valid = False
753
        server = self.server
754
        if self.path == "/api/repos":
755
            self._json(200, {"name": "repo"})
756
            return
757
        if self.path == "/api/oak/repo/push":
758
            with server.receipt_lock:  # type: ignore[attr-defined]
759
                server.push_request_count += 1  # type: ignore[attr-defined]
760
                server.push_request_bytes += len(body)  # type: ignore[attr-defined]
761
                server.push_declared_request_bytes = declared_size  # type: ignore[attr-defined]
762
                server.push_request_framing_valid = framing_valid  # type: ignore[attr-defined]
763
                server.push_request_body_timed_out = body_timed_out  # type: ignore[attr-defined]
764
                server.request_complete_before_disconnect = (  # type: ignore[attr-defined]
765
                    framing_valid and len(body) == declared_size
766
                )
767
            self.close_connection = True
768
            try:
769
                self.connection.shutdown(socket.SHUT_RDWR)
770
            except OSError:
771
                pass
772
            self.connection.close()
773
            return
774
        self._json(404, {"error": "unexpected synthetic path"})
775
776
777
def start_interrupted_publication_server() -> tuple[ThreadingHTTPServer, threading.Thread]:
778
    server = ThreadingHTTPServer(("127.0.0.1", 0), _DisconnectAfterPushHandler)
779
    server.receipt_lock = threading.Lock()  # type: ignore[attr-defined]
780
    server.push_request_count = 0  # type: ignore[attr-defined]
781
    server.push_request_bytes = 0  # type: ignore[attr-defined]
782
    server.push_declared_request_bytes = None  # type: ignore[attr-defined]
783
    server.push_request_framing_valid = False  # type: ignore[attr-defined]
784
    server.push_request_body_timed_out = False  # type: ignore[attr-defined]
785
    server.request_complete_before_disconnect = False  # type: ignore[attr-defined]
786
    thread = threading.Thread(target=server.serve_forever, name="oakbench-interrupted-push", daemon=True)
787
    thread.start()
788
    return server, thread
789
790
791
def _parse_json_object(text: str | None) -> dict[str, Any] | None:
792
    try:
793
        value = json.loads(text or "")
794
    except (TypeError, json.JSONDecodeError):
795
        return None
796
    return value if isinstance(value, dict) else None
797
798
799
def run_interrupted_publication(
800
    subject: Subject,
801
    run_root: Path,
802
    run_index: int,
803
    metadata: dict[str, Any],
804
    admitted: int,
805
) -> list[dict[str, Any]]:
806
    """Run one real Oak push against a loopback server that drops its reply.
807
808
    The fixture observes request delivery but deliberately has no durable
809
    remote state. That makes the client's outcome unknown and lets the lane
810
    prove work preservation and absence of blind replay without production
811
    mutations or a fake success claim.
812
    """
813
    scenario = INTERRUPTED_PUBLICATION_SCENARIO
814
    metadata = {
815
        **metadata,
816
        "benchmark_track": "failure-evidence",
817
        "remote_transport": "loopback_fault_fixture",
818
        "remote_server": "127.0.0.1",
819
    }
820
    if subject.kind != "oak":
821
        return [
822
            skip_row(
823
                subject,
824
                scenario,
825
                run_index,
826
                metadata,
827
                "interrupted publication fixture currently exercises the actual Oak HTTP client only",
828
            )
829
        ]
830
831
    workspace = run_root / f"interrupted-{subject.name}-r{run_index}"
832
    workspace.mkdir(parents=True)
833
    rows: list[dict[str, Any]] = []
834
    vcs = str(subject.bin)
835
    rows.append(
836
        step_row(subject, scenario, "setup.init", [vcs, "init", "."], workspace, run_index, metadata, admitted)
837
    )
838
    if row_returncode(rows[-1]) != 0:
839
        return rows
840
    rows.append(
841
        edit_step(
842
            subject,
843
            scenario,
844
            "setup.edit",
845
            workspace,
846
            f"interrupted publication run={run_index}",
847
            run_index,
848
            metadata,
849
            admitted,
850
        )
851
    )
852
    rows.append(
853
        step_row(subject, scenario, "setup.checkpoint", [vcs, "commit"], workspace, run_index, metadata, admitted)
854
    )
855
    if row_returncode(rows[-1]) != 0:
856
        return rows
857
858
    before = run_untimed([vcs, "hash"], workspace)
859
    head_before = before.stdout.strip() if before.returncode == 0 else None
860
    payload_path = workspace / "task-notes.md"
861
    payload_before = file_sha256(payload_path)
862
    before_usage = bounded_tree_usage(workspace)
863
    push_help = run_untimed([vcs, "push", "--help"], workspace)
864
    push_supports_json = "--json" in (push_help.stdout + push_help.stderr)
865
    server, thread = start_interrupted_publication_server()
866
    host, port = server.server_address[:2]
867
    remote = f"http://{host}:{port}"
868
    fixture_env = base_env()
869
    fixture_env.pop("OAK_API_KEY", None)
870
    fixture_env.pop("OAK_REMOTE", None)
871
    fixture_env["OAK_NO_UPDATE_CHECK"] = "1"
872
    try:
873
        push_command = [vcs, "push", "--repo", "oak/repo", "--remote", remote]
874
        if push_supports_json:
875
            push_command.append("--json")
876
        publication_attempt = step_row(
877
            subject,
878
            scenario,
879
            "publication.attempt",
880
            push_command,
881
            workspace,
882
            run_index,
883
            metadata,
884
            admitted,
885
            expected_returncodes=tuple(range(1, 128)),
886
            preserve_stdout=True,
887
            preserve_stderr=True,
888
            env=fixture_env,
889
            timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
890
        )
891
        rows.append(publication_attempt)
892
    finally:
893
        server.shutdown()
894
        server.server_close()
895
        thread.join(timeout=2.0)
896
897
    state_row = step_row(
898
        subject,
899
        scenario,
900
        "reconcile.local_state",
901
        [vcs, "agent", "state", "--json", "--compact"],
902
        workspace,
903
        run_index,
904
        metadata,
905
        admitted,
906
        preserve_stdout=True,
907
        env=fixture_env,
908
    )
909
    rows.append(state_row)
910
    after = run_untimed([vcs, "hash"], workspace)
911
    head_after = after.stdout.strip() if after.returncode == 0 else None
912
    payload_after = file_sha256(payload_path)
913
    status_after = _parse_json_object(state_row.get("stdout"))
914
    with server.receipt_lock:  # type: ignore[attr-defined]
915
        request_count = int(server.push_request_count)  # type: ignore[attr-defined]
916
        request_bytes = int(server.push_request_bytes)  # type: ignore[attr-defined]
917
        declared_request_bytes = server.push_declared_request_bytes  # type: ignore[attr-defined]
918
        request_framing_valid = bool(server.push_request_framing_valid)  # type: ignore[attr-defined]
919
        request_body_timed_out = bool(server.push_request_body_timed_out)  # type: ignore[attr-defined]
920
    cli_observation = cli_publication_observation(
921
        publication_attempt,
922
        json_requested=push_supports_json,
923
    )
924
    evidence = interrupted_publication_fields(
925
        request_count=request_count,
926
        request_bytes=request_bytes if request_count else None,
927
        declared_request_bytes=declared_request_bytes,
928
        request_framing_valid=request_framing_valid,
929
        request_body_timed_out=request_body_timed_out,
930
        local_head_before=head_before,
931
        local_head_after=head_after,
932
        status_after=status_after,
933
        worktree_payload_before=payload_before,
934
        worktree_payload_after=payload_after,
935
        cli_observation=cli_observation,
936
    )
937
    evidence["workspace_usage_before"] = before_usage
938
    evidence["workspace_usage_after"] = bounded_tree_usage(workspace)
939
    evidence["remote_transport"] = "loopback_fault_fixture"
940
    evidence["remote_server"] = str(host)
941
    evidence["human_action_count"] = 0
942
    evidence["compatibility_probe_calls"] = 1
943
    evidence["harness_probe_calls"] = 3
944
    evidence["push_json_supported"] = push_supports_json
945
    total = phase_total_row(rows, subject, scenario, run_index, metadata, "interrupted_publication.total")
946
    evidence["harness_command_count"] = total["harness_command_count"] + 3
947
    total.update(evidence)
948
    if not (
949
        evidence["fixture_received_complete_request_before_disconnect"]
950
        and request_count == 1
951
        and evidence["local_head_preserved"]
952
        and evidence["local_work_preserved"]
953
        and not publication_attempt["timed_out"]
954
        and row_returncode(rows[-1]) == 0
955
    ):
956
        total["returncode"] = 1
957
        total["stderr"] = "interrupted_publication_evidence_incomplete"
958
    rows.append(total)
959
    return rows
960
961
962
def edit_step(
963
    subject: Subject,
964
    scenario: str,
965
    operation: str,
966
    workspace: Path,
967
    payload: str,
968
    run_index: int,
969
    metadata: dict[str, Any],
970
    admitted: int,
971
) -> dict[str, Any]:
972
    command = ["/bin/zsh", "-c", f"echo {payload!r} >> task-notes.md"]
973
    return step_row(
974
        subject, scenario, operation, command, workspace, run_index, metadata, admitted, kind="edit"
975
    )
976
977
978
def oak_virtual_branch(subject: Subject, mount_dir: Path) -> str | None:
979
    proc = run_untimed([str(subject.bin), "status"], mount_dir)
980
    match = VIRTUAL_BRANCH_RE.search(proc.stdout + proc.stderr)
981
    return match.group(1) if match else None
982
983
984
def run_oak_loop(
985
    subject: Subject,
986
    remote: oakbench_remotes.RemoteResolution,
987
    run_root: Path,
988
    run_index: int,
989
    metadata: dict[str, Any],
990
    admitted: int,
991
) -> list[dict[str, Any]]:
992
    scenario = "task_loop"
993
    meta = {**metadata, **remote.row_fields()}
994
    rows: list[dict[str, Any]] = []
995
    branches: dict[str, str | None] = {}
996
    payloads: dict[str, str] = {}
997
    mounts: list[Path] = []
998
    vcs = str(subject.bin)
999
    try:
1000
        for phase in PHASES:
1001
            workspace = run_root / f"tl-{subject.name}-r{run_index}-{phase}"
1002
            payload = f"payload {phase} run={run_index} bench={metadata['bench_id']}"
1003
            payloads[phase] = payload
1004
            acquire = step_row(
1005
                subject, scenario, f"{phase}.acquire", [vcs, "mount", remote.repo or "", str(workspace)],
1006
                run_root, run_index, meta, admitted,
1007
            )
1008
            rows.append(acquire)
1009
            if int(acquire["returncode"]) != 0:
1010
                return rows
1011
            mounts.append(workspace)
1012
            branches[phase] = oak_virtual_branch(subject, workspace)
1013
            rows.append(edit_step(subject, scenario, f"{phase}.edit", workspace, payload, run_index, meta, admitted))
1014
            rows.append(step_row(subject, scenario, f"{phase}.snapshot", [vcs, "commit", "--no-verify"], workspace, run_index, meta, admitted))
1015
            rows.append(step_row(subject, scenario, f"{phase}.publish", [vcs, "push"], workspace, run_index, meta, admitted))
1016
            if phase == "task":
1017
                rows.append(
1018
                    step_row(
1019
                        subject, scenario, "task.describe",
1020
                        [vcs, "desc", f"bench task loop {metadata['bench_id']} r{run_index}"],
1021
                        workspace, run_index, meta, admitted,
1022
                    )
1023
                )
1024
            end = step_row(
1025
                subject, scenario, f"{phase}.end", [vcs, "mount", "end", str(workspace)],
1026
                run_root, run_index, meta, admitted,
1027
            )
1028
            rows.append(end)
1029
            if int(end["returncode"]) == 0 and workspace in mounts:
1030
                mounts.remove(workspace)
1031
1032
        # Untimed oracle: each phase's payload must be on its pushed branch.
1033
        verifier = run_root / f"tl-verify-{subject.name}-r{run_index}"
1034
        clone = run_untimed([vcs, "clone", remote.repo or "", str(verifier)], run_root)
1035
        oracle_ok: bool | None = None
1036
        if clone.returncode == 0:
1037
            oracle_ok = True
1038
            for phase in PHASES:
1039
                branch = branches.get(phase)
1040
                if not branch:
1041
                    oracle_ok = None
1042
                    break
1043
                switch = run_untimed([vcs, "switch", branch], verifier)
1044
                content = (verifier / "task-notes.md")
1045
                seen = content.read_text() if content.exists() else ""
1046
                if switch.returncode != 0 or payloads[phase] not in seen:
1047
                    oracle_ok = False
1048
        total = {
1049
            **phase_total_row(rows, subject, scenario, run_index, meta, "task_loop.total"),
1050
            "oracle_payload_on_remote_branch": oracle_ok,
1051
            "workspace_branches": branches,
1052
            "measurement_source": "verifier_clone_switch_grep; null oracle means branch name unparsed",
1053
        }
1054
        rows.append(enforce_oracle_result(total, oracle_ok))
1055
        return rows
1056
    finally:
1057
        for mount_dir in mounts:
1058
            run_untimed([vcs, "mount", "end", str(mount_dir), "-f"], run_root)
1059
1060
1061
def ensure_git_remote_seeded(subject: Subject, remote: oakbench_remotes.RemoteResolution, scratch: Path) -> str | None:
1062
    """Untimed: make sure the git remote has a main with a seed commit."""
1063
    vcs = str(subject.bin)
1064
    probe = run_untimed([vcs, "ls-remote", str(remote.repo), "refs/heads/main"], None)
1065
    if probe.returncode == 0 and probe.stdout.strip():
1066
        if remote.transport == oakbench_remotes.TRANSPORT_LOCAL_FILE:
1067
            head = run_untimed([vcs, "symbolic-ref", "HEAD", "refs/heads/main"], Path(str(remote.repo)))
1068
            if head.returncode != 0:
1069
                return f"git remote HEAD setup failed: {head.stderr.strip()[:200]}"
1070
        return None
1071
    seed = scratch / "git-seed"
1072
    if seed.exists():
1073
        shutil.rmtree(seed)
1074
    seed.mkdir(parents=True)
1075
    (seed / "README.md").write_text("Task-loop benchmark repo (disposable).\n")
1076
    (seed / "task-notes.md").write_text("# task notes\n")
1077
    for command in (
1078
        [vcs, "init", "-q", "-b", "main"],
1079
        [vcs, "add", "."],
1080
        [vcs, "commit", "-q", "-m", "seed"],
1081
        [vcs, "push", "-q", str(remote.repo), "main:main"],
1082
    ):
1083
        proc = run_untimed(command, seed)
1084
        if proc.returncode != 0:
1085
            return f"git remote seed `{' '.join(command[1:3])}` failed: {proc.stderr.strip()[:200]}"
1086
    if remote.transport == oakbench_remotes.TRANSPORT_LOCAL_FILE:
1087
        head = run_untimed([vcs, "symbolic-ref", "HEAD", "refs/heads/main"], Path(str(remote.repo)))
1088
        if head.returncode != 0:
1089
            return f"git remote HEAD setup failed: {head.stderr.strip()[:200]}"
1090
    return None
1091
1092
1093
def run_git_agent_task_baseline(
1094
    subject: Subject,
1095
    remote: oakbench_remotes.RemoteResolution,
1096
    run_root: Path,
1097
    run_index: int,
1098
    metadata: dict[str, Any],
1099
    admitted: int,
1100
    remote_create_ms: float = 0.0,
1101
) -> list[dict[str, Any]]:
1102
    """Measure a cold clone and a warm isolated worktree on one local remote.
1103
1104
    This is the core-equivalent, local-file cell. Hosted CI is deliberately a
1105
    skip row: a local commit oracle is not relabelled as provider CI.
1106
    """
1107
    scenario = AGENT_TASK_SCENARIO
1108
    meta = {
1109
        **metadata,
1110
        **remote.row_fields(),
1111
        "benchmark_track": (
1112
            "core-equivalent"
1113
            if remote.transport == oakbench_remotes.TRANSPORT_LOCAL_FILE
1114
            else "agent-default"
1115
        ),
1116
    }
1117
    vcs = str(subject.bin)
1118
    rows: list[dict[str, Any]] = [
1119
        harness_step_row(
1120
            subject,
1121
            scenario,
1122
            "setup.remote_container",
1123
            run_index,
1124
            meta,
1125
            remote_create_ms,
1126
            harness_command_count=1,
1127
        )
1128
    ]
1129
    setup_start = time.perf_counter()
1130
    seed_error = ensure_git_remote_seeded(subject, remote, run_root)
1131
    rows.append(
1132
        harness_step_row(
1133
            subject,
1134
            scenario,
1135
            "setup.remote_seed",
1136
            run_index,
1137
            meta,
1138
            (time.perf_counter() - setup_start) * 1000,
1139
            returncode=1 if seed_error else 0,
1140
            harness_command_count=6,
1141
        )
1142
    )
1143
    if seed_error:
1144
        rows[-1]["stderr"] = seed_error
1145
        return rows
1146
1147
    base = run_root / f"agent-task-{subject.name}-r{run_index}-cold"
1148
    cold_branch = oakbench_remotes.disposable_branch(
1149
        str(metadata["bench_id"]), "agent-task-cold", subject.name, run_index
1150
    )
1151
    warm_branch = oakbench_remotes.disposable_branch(
1152
        str(metadata["bench_id"]), "agent-task-warm", subject.name, run_index
1153
    )
1154
    cold_payload = f"cold task run={run_index} bench={metadata['bench_id']}"
1155
    warm_payload = f"warm task run={run_index} bench={metadata['bench_id']}"
1156
1157
    rows.append(
1158
        step_row(
1159
            subject,
1160
            scenario,
1161
            "cold.acquire",
1162
            [vcs, "clone", "-q", str(remote.repo), str(base)],
1163
            run_root,
1164
            run_index,
1165
            meta,
1166
            admitted,
1167
        )
1168
    )
1169
    if row_returncode(rows[-1]) != 0:
1170
        return rows
1171
    rows.append(
1172
        step_row(
1173
            subject, scenario, "cold.branch", [vcs, "switch", "-q", "-c", cold_branch],
1174
            base, run_index, meta, admitted,
1175
        )
1176
    )
1177
    rows.append(
1178
        step_row(
1179
            subject, scenario, "cold.inspect", [vcs, "status", "--porcelain=v1"],
1180
            base, run_index, meta, admitted,
1181
        )
1182
    )
1183
    rows.append(edit_step(subject, scenario, "cold.edit", base, cold_payload, run_index, meta, admitted))
1184
    rows.append(step_row(subject, scenario, "cold.checkpoint.add", [vcs, "add", "."], base, run_index, meta, admitted))
1185
    rows.append(
1186
        step_row(
1187
            subject, scenario, "cold.checkpoint.commit",
1188
            [vcs, "commit", "-q", "-m", f"agent task cold {metadata['bench_id']} r{run_index}"],
1189
            base, run_index, meta, admitted,
1190
        )
1191
    )
1192
    rows.append(
1193
        step_row(
1194
            subject, scenario, "cold.publish", [vcs, "push", "-q", "-u", "origin", cold_branch],
1195
            base, run_index, meta, admitted,
1196
        )
1197
    )
1198
    rows.append(
1199
        unmeasured_step_row(
1200
            subject, scenario, "cold.ci.observe", run_index, meta,
1201
            "local-file transport has no hosted CI provider; exact CI observation is unmeasured",
1202
        )
1203
    )
1204
    rows.append(
1205
        step_row(
1206
            subject, scenario, "cold.review_handoff",
1207
            [vcs, "show", "--format=%H%n%s", "--stat", "--oneline", "HEAD"],
1208
            base, run_index, meta, admitted,
1209
        )
1210
    )
1211
1212
    warm = run_root / f"agent-task-{subject.name}-r{run_index}-warm"
1213
    rows.append(
1214
        step_row(
1215
            subject, scenario, "warm.acquire",
1216
            [vcs, "worktree", "add", "-q", str(warm), "-b", warm_branch, "origin/main"],
1217
            base, run_index, meta, admitted,
1218
        )
1219
    )
1220
    if row_returncode(rows[-1]) == 0:
1221
        rows.append(
1222
            step_row(
1223
                subject, scenario, "warm.inspect", [vcs, "status", "--porcelain=v1"],
1224
                warm, run_index, meta, admitted,
1225
            )
1226
        )
1227
        rows.append(edit_step(subject, scenario, "warm.edit", warm, warm_payload, run_index, meta, admitted))
1228
        rows.append(step_row(subject, scenario, "warm.checkpoint.add", [vcs, "add", "."], warm, run_index, meta, admitted))
1229
        rows.append(
1230
            step_row(
1231
                subject, scenario, "warm.checkpoint.commit",
1232
                [vcs, "commit", "-q", "-m", f"agent task warm {metadata['bench_id']} r{run_index}"],
1233
                warm, run_index, meta, admitted,
1234
            )
1235
        )
1236
        rows.append(
1237
            step_row(
1238
                subject, scenario, "warm.publish", [vcs, "push", "-q", "-u", "origin", warm_branch],
1239
                warm, run_index, meta, admitted,
1240
            )
1241
        )
1242
        rows.append(
1243
            unmeasured_step_row(
1244
                subject, scenario, "warm.ci.observe", run_index, meta,
1245
                "local-file transport has no hosted CI provider; exact CI observation is unmeasured",
1246
            )
1247
        )
1248
        rows.append(
1249
            step_row(
1250
                subject, scenario, "warm.review_handoff",
1251
                [vcs, "show", "--format=%H%n%s", "--stat", "--oneline", "HEAD"],
1252
                warm, run_index, meta, admitted,
1253
            )
1254
        )
1255
1256
    usage_before_cleanup = {
1257
        "cold": bounded_tree_usage(base),
1258
        "warm": bounded_tree_usage(warm),
1259
    }
1260
    oracle_ok = True
1261
    for branch, payload in ((cold_branch, cold_payload), (warm_branch, warm_payload)):
1262
        show = run_untimed([vcs, "--git-dir", str(remote.repo), "show", f"{branch}:task-notes.md"], run_root)
1263
        if show.returncode != 0 or payload not in show.stdout:
1264
            oracle_ok = False
1265
1266
    if warm.exists():
1267
        rows.append(
1268
            step_row(
1269
                subject, scenario, "cleanup.warm",
1270
                [vcs, "worktree", "remove", str(warm)], base, run_index, meta, admitted,
1271
            )
1272
        )
1273
    cleanup_start = time.perf_counter()
1274
    shutil.rmtree(base, ignore_errors=True)
1275
    rows.append(
1276
        harness_step_row(
1277
            subject, scenario, "cleanup.cold", run_index, meta,
1278
            (time.perf_counter() - cleanup_start) * 1000,
1279
        )
1280
    )
1281
1282
    total = phase_total_row(rows, subject, scenario, run_index, meta, "agent_task.total")
1283
    total.update(
1284
        {
1285
            "oracle_payload_on_remote_branch": oracle_ok,
1286
            "acquisition_models": {
1287
                "cold": "fresh_full_clone",
1288
                "warm": "shared_object_worktree",
1289
            },
1290
            "repository_cache_state": {
1291
                "cold": "fresh_destination_no_shared_object_store",
1292
                "warm": "explicit_shared_object_store",
1293
            },
1294
            "os_cache_state": "unknown",
1295
            "os_cache_state_reason": "page_cache_not_purged_or_instrumented",
1296
            "workspace_usage_before_cleanup": usage_before_cleanup,
1297
            "network_bytes": None,
1298
            "network_bytes_reason": "local_file_transport_not_observed_by_byte_proxy",
1299
            "partial_clone_comparator": None,
1300
            "partial_clone_reason": "local file transport does not establish a network partial-clone comparison",
1301
            "human_action_count": 0,
1302
            "parallel_evidence_lane": "parallel_contention.py/workspace_per_task",
1303
            "oracle_probe_calls": 2,
1304
            "harness_command_count": total["harness_command_count"] + 2,
1305
        }
1306
    )
1307
    if not oracle_ok:
1308
        total["returncode"] = 1
1309
        total["stderr"] = "agent_task_remote_payload_oracle_failed"
1310
    rows.append(total)
1311
    return rows
1312
1313
1314
def reserve_loopback_port() -> int:
1315
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
1316
        listener.bind(("127.0.0.1", 0))
1317
        return int(listener.getsockname()[1])
1318
1319
1320
def wait_for_loopback_port(
1321
    process: subprocess.Popen[Any], port: int, timeout_s: float = LOCAL_OAK_SERVER_READY_TIMEOUT_S
1322
) -> bool:
1323
    deadline = time.monotonic() + timeout_s
1324
    while time.monotonic() < deadline:
1325
        if process.poll() is not None:
1326
            return False
1327
        try:
1328
            with socket.create_connection(("127.0.0.1", port), timeout=0.2):
1329
                return True
1330
        except OSError:
1331
            time.sleep(0.05)
1332
    return False
1333
1334
1335
def harness_command_row(
1336
    subject: Subject,
1337
    scenario: str,
1338
    operation: str,
1339
    command: list[str],
1340
    cwd: Path,
1341
    run_index: int,
1342
    metadata: dict[str, Any],
1343
) -> tuple[dict[str, Any], subprocess.CompletedProcess[str]]:
1344
    started = time.perf_counter()
1345
    result = run_bounded_harness_command(command, cwd)
1346
    row = harness_step_row(
1347
        subject,
1348
        scenario,
1349
        operation,
1350
        run_index,
1351
        metadata,
1352
        (time.perf_counter() - started) * 1000,
1353
        returncode=0 if result.returncode == 0 else result.returncode,
1354
        harness_command_count=1,
1355
    )
1356
    if result.returncode != 0:
1357
        row["stderr"] = str(result.stderr)[-4000:]
1358
    return row, result
1359
1360
1361
def run_bounded_harness_command(
1362
    command: list[str], cwd: Path, *, env: dict[str, str] | None = None
1363
) -> subprocess.CompletedProcess[str]:
1364
    capture = run_timed(
1365
        command,
1366
        cwd,
1367
        env or base_env(),
1368
        admitted_output_chars=4_000,
1369
        timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
1370
    )
1371
    return subprocess.CompletedProcess(
1372
        command,
1373
        capture.returncode,
1374
        capture.stdout_text,
1375
        capture.stderr_text,
1376
    )
1377
1378
1379
def remote_branch_head(remote: str, repo_spec: str, branch: str) -> str | None:
1380
    owner, repo = repo_spec.split("/", 1)
1381
    branch_segment = urllib.parse.quote(branch, safe="")
1382
    url = f"{remote}/api/{owner}/{repo}/branches/{branch_segment}"
1383
    try:
1384
        with urllib.request.urlopen(url, timeout=2.0) as response:
1385
            value = json.loads(response.read())
1386
    except (OSError, ValueError, urllib.error.URLError):
1387
        return None
1388
    head = value.get("head") if isinstance(value, dict) else None
1389
    return head if isinstance(head, str) else None
1390
1391
1392
def run_oak_local_agent_task_baseline(
1393
    subject: Subject,
1394
    run_root: Path,
1395
    run_index: int,
1396
    metadata: dict[str, Any],
1397
    admitted: int,
1398
) -> list[dict[str, Any]]:
1399
    """Exercise two fresh Oak checkouts against an owned loopback Oak server."""
1400
    run_root = run_root.resolve()
1401
    scenario = AGENT_TASK_OAK_LOCAL_SCENARIO
1402
    meta = {
1403
        **metadata,
1404
        "benchmark_track": "oak-local-loopback",
1405
        "remote_transport": "loopback_http",
1406
        "remote_server": "127.0.0.1",
1407
    }
1408
    rows: list[dict[str, Any]] = []
1409
    vcs = str(subject.bin)
1410
    server_root = run_root / f"oak-local-server-{subject.name}-r{run_index}"
1411
    seed = run_root / f"oak-local-seed-{subject.name}-r{run_index}"
1412
    cold = run_root / f"oak-local-cold-{subject.name}-r{run_index}"
1413
    followup = run_root / f"oak-local-followup-{subject.name}-r{run_index}"
1414
    verifier = run_root / f"oak-local-verifier-{subject.name}-r{run_index}"
1415
    repo_spec = f"qa/task-loop-{subject.name}-r{run_index}"
1416
    port = reserve_loopback_port()
1417
    remote = f"http://127.0.0.1:{port}"
1418
    server_root.mkdir(parents=True)
1419
    server_started = time.perf_counter()
1420
    server: subprocess.Popen[Any] | None = None
1421
    server_start_error: str | None = None
1422
    try:
1423
        server = subprocess.Popen(
1424
            [vcs, "serve", "--dir", str(server_root), "--host", "127.0.0.1", "--port", str(port)],
1425
            cwd=run_root,
1426
            env=base_env(),
1427
            stdout=subprocess.DEVNULL,
1428
            stderr=subprocess.DEVNULL,
1429
            start_new_session=os.name == "posix",
1430
        )
1431
    except OSError as error:
1432
        server_start_error = type(error).__name__
1433
    ready = server is not None and wait_for_loopback_port(server, port)
1434
    rows.append(
1435
        harness_step_row(
1436
            subject,
1437
            scenario,
1438
            "setup.server",
1439
            run_index,
1440
            meta,
1441
            (time.perf_counter() - server_started) * 1000,
1442
            returncode=0 if ready else 1,
1443
            harness_command_count=1,
1444
        )
1445
    )
1446
    branch: str | None = None
1447
    expected_head: str | None = None
1448
    expected_payload = "# task notes\n"
1449
    failure: str | None = None if ready else (
1450
        f"local_oak_server_start_failed:{server_start_error}"
1451
        if server_start_error else "local_oak_server_not_ready"
1452
    )
1453
    try:
1454
        if failure is None:
1455
            seed.mkdir(parents=True)
1456
            (seed / "task-notes.md").write_text(expected_payload)
1457
            for operation, command in (
1458
                ("setup.seed_init", [vcs, "init", "."]),
1459
                ("setup.seed_commit", [vcs, "commit"]),
1460
                ("setup.seed_push", [vcs, "push", "--repo", repo_spec, "--remote", remote]),
1461
            ):
1462
                row, result = harness_command_row(
1463
                    subject, scenario, operation, command, seed, run_index, meta
1464
                )
1465
                rows.append(row)
1466
                if result.returncode != 0:
1467
                    failure = f"{operation}_failed"
1468
                    break
1469
1470
        def agent_step(operation: str, command: list[str], cwd: Path, *, preserve: bool = False) -> dict[str, Any]:
1471
            row = step_row(
1472
                subject,
1473
                scenario,
1474
                operation,
1475
                command,
1476
                cwd,
1477
                run_index,
1478
                meta,
1479
                admitted,
1480
                preserve_stdout=preserve,
1481
                timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
1482
            )
1483
            rows.append(row)
1484
            return row
1485
1486
        if failure is None:
1487
            row = agent_step(
1488
                "cold.acquire",
1489
                [vcs, "clone", repo_spec, str(cold), "--remote", remote, "--allow-legacy-scope"],
1490
                run_root,
1491
            )
1492
            if row_returncode(row) != 0:
1493
                failure = "cold.acquire_failed"
1494
        if failure is None:
1495
            status = agent_step("cold.inspect", [vcs, "status", "--json"], cold, preserve=True)
1496
            status_value = _parse_json_object(status.get("stdout"))
1497
            branch_value = status_value.get("branch") if status_value else None
1498
            branch = branch_value if isinstance(branch_value, str) else None
1499
            if not branch:
1500
                failure = "cold.branch_identity_missing"
1501
        if failure is None:
1502
            cold_payload = f"cold task run={run_index} bench={metadata['bench_id']}\n"
1503
            expected_payload += cold_payload
1504
            rows.append(edit_step(subject, scenario, "cold.edit", cold, cold_payload.strip(), run_index, meta, admitted))
1505
            if row_returncode(agent_step("cold.checkpoint", [vcs, "commit"], cold)) != 0:
1506
                failure = "cold.checkpoint_failed"
1507
            elif row_returncode(
1508
                agent_step("cold.publish", [vcs, "push", "--repo", repo_spec, "--remote", remote], cold)
1509
            ) != 0:
1510
                failure = "cold.publish_failed"
1511
            rows.append(
1512
                unmeasured_step_row(
1513
                    subject, scenario, "cold.ci.observe", run_index, meta,
1514
                    "owned local oak serve has no configured CI provider",
1515
                )
1516
            )
1517
            rows.append(
1518
                unmeasured_step_row(
1519
                    subject, scenario, "cold.review_handoff", run_index, meta,
1520
                    "owned local oak serve does not provide hosted review semantics",
1521
                )
1522
            )
1523
        if failure is None and branch is not None:
1524
            row = agent_step(
1525
                "followup.acquire",
1526
                [
1527
                    vcs, "clone", repo_spec, str(followup), "--remote", remote,
1528
                    "--branch", branch, "--allow-legacy-scope",
1529
                ],
1530
                run_root,
1531
            )
1532
            if row_returncode(row) != 0:
1533
                failure = "followup.acquire_failed"
1534
        if failure is None:
1535
            agent_step("followup.inspect", [vcs, "status", "--json"], followup)
1536
            followup_payload = f"followup task run={run_index} bench={metadata['bench_id']}\n"
1537
            expected_payload += followup_payload
1538
            rows.append(
1539
                edit_step(
1540
                    subject, scenario, "followup.edit", followup,
1541
                    followup_payload.strip(), run_index, meta, admitted,
1542
                )
1543
            )
1544
            if row_returncode(agent_step("followup.checkpoint", [vcs, "commit"], followup)) != 0:
1545
                failure = "followup.checkpoint_failed"
1546
            else:
1547
                head_result = run_bounded_harness_command([vcs, "hash"], followup)
1548
                expected_head = head_result.stdout.strip() if head_result.returncode == 0 else None
1549
                if row_returncode(
1550
                    agent_step(
1551
                        "followup.publish",
1552
                        [vcs, "push", "--repo", repo_spec, "--remote", remote],
1553
                        followup,
1554
                    )
1555
                ) != 0:
1556
                    failure = "followup.publish_failed"
1557
            rows.append(
1558
                unmeasured_step_row(
1559
                    subject, scenario, "followup.ci.observe", run_index, meta,
1560
                    "owned local oak serve has no configured CI provider",
1561
                )
1562
            )
1563
            rows.append(
1564
                unmeasured_step_row(
1565
                    subject, scenario, "followup.review_handoff", run_index, meta,
1566
                    "owned local oak serve does not provide hosted review semantics",
1567
                )
1568
            )
1569
1570
        payload_ok = False
1571
        verifier_head: str | None = None
1572
        advertised_head: str | None = None
1573
        if failure is None and branch is not None:
1574
            clone = run_bounded_harness_command(
1575
                [
1576
                    vcs, "clone", repo_spec, str(verifier), "--remote", remote,
1577
                    "--branch", branch, "--allow-legacy-scope",
1578
                ],
1579
                run_root,
1580
            )
1581
            if clone.returncode == 0:
1582
                payload_path = verifier / "task-notes.md"
1583
                payload_ok = payload_path.exists() and payload_path.read_text() == expected_payload
1584
                verifier_result = run_bounded_harness_command([vcs, "hash"], verifier)
1585
                verifier_head = verifier_result.stdout.strip() if verifier_result.returncode == 0 else None
1586
                advertised_head = remote_branch_head(remote, repo_spec, branch)
1587
            else:
1588
                failure = "verifier_clone_failed"
1589
        head_ok = bool(expected_head and expected_head == verifier_head == advertised_head)
1590
        if not payload_ok:
1591
            failure = failure or "local_oak_payload_oracle_failed"
1592
        if not head_ok:
1593
            failure = failure or "local_oak_head_oracle_failed"
1594
    finally:
1595
        if server is not None:
1596
            stop_owned_process(server)
1597
        server_stopped = server is None or server.poll() is not None
1598
        cleanup_started = time.perf_counter()
1599
        for path in (seed, cold, followup, verifier, server_root):
1600
            shutil.rmtree(path, ignore_errors=True)
1601
        rows.append(
1602
            harness_step_row(
1603
                subject,
1604
                scenario,
1605
                "cleanup.local_server_and_checkouts",
1606
                run_index,
1607
                meta,
1608
                (time.perf_counter() - cleanup_started) * 1000,
1609
            )
1610
        )
1611
1612
    total = phase_total_row(rows, subject, scenario, run_index, meta, "agent_task_oak_local.total")
1613
    total.update(
1614
        {
1615
            "oracle_payload_on_remote_branch": payload_ok and head_ok,
1616
            "oracle_payload_matches": payload_ok,
1617
            "oracle_branch_head_matches": head_ok,
1618
            "server_process_stopped": server_stopped,
1619
            "acquisition_models": {
1620
                "cold": "fresh_full_clone",
1621
                "followup": "second_fresh_full_clone",
1622
            },
1623
            "repository_cache_state": {
1624
                "cold": "fresh_destination_no_shared_object_store",
1625
                "followup": "no_shared_object_store",
1626
            },
1627
            "os_cache_state": "unknown",
1628
            "os_cache_state_reason": "page_cache_not_purged_or_instrumented",
1629
            "network_bytes": None,
1630
            "network_bytes_reason": "loopback transport not observed by byte proxy",
1631
            "hosted_ci_configured": False,
1632
            "oracle_probe_calls": 4,
1633
            # The expected-head hash plus verifier clone/hash/HTTP head probe
1634
            # are oracle/setup commands, not agent-facing tool calls.
1635
            "harness_command_count": total["harness_command_count"] + 4,
1636
        }
1637
    )
1638
    if failure or not server_stopped:
1639
        total["returncode"] = 1
1640
        total["stderr"] = failure or "local_oak_server_cleanup_failed"
1641
    rows.append(total)
1642
    return rows
1643
1644
1645
def run_legacy_selected_materialization_campaign(
1646
    subjects: list[Subject],
1647
    server_bin: Path | None,
1648
    server_source: Path | None,
1649
    server_source_head: str | None,
1650
    baseline_subject_name: str | None,
1651
    run_root: Path,
1652
    runs: int,
1653
    metadata: dict[str, Any],
1654
    admitted: int,
1655
    randomize_order: bool,
1656
    random_seed: int,
1657
    timing_controls_requested: bool,
1658
) -> list[dict[str, Any]]:
1659
    """Measure one legacy selected-branch clone against one fixed Serve fixture."""
1660
    scenario = LEGACY_SELECTED_MATERIALIZATION_SCENARIO
1661
    oak_subjects = [subject for subject in subjects if subject.kind == "oak"]
1662
    if not oak_subjects:
1663
        return []
1664
    if (
1665
        server_bin is None
1666
        or not server_bin.is_absolute()
1667
        or not server_bin.is_file()
1668
        or not os.access(server_bin, os.X_OK)
1669
    ):
1670
        return [
1671
            unmeasured_step_row(
1672
                subject,
1673
                scenario,
1674
                "scenario.skip",
1675
                0,
1676
                metadata,
1677
                "--oak-serve-bin must name an absolute executable used only as the fixed Serve fixture",
1678
            )
1679
            for subject in oak_subjects
1680
        ]
1681
    if server_source_head is None or re.fullmatch(r"[0-9a-f]{64}", server_source_head) is None:
1682
        return [
1683
            unmeasured_step_row(
1684
                subject,
1685
                scenario,
1686
                "scenario.skip",
1687
                0,
1688
                metadata,
1689
                "the fixed Serve binary's exact 64-hex --oak-serve-source-head is required",
1690
            )
1691
            for subject in oak_subjects
1692
        ]
1693
    baseline_subject = next(
1694
        (subject for subject in oak_subjects if subject.name == baseline_subject_name), None
1695
    )
1696
    if baseline_subject is None:
1697
        return [
1698
            unmeasured_step_row(
1699
                subject,
1700
                scenario,
1701
                "scenario.skip",
1702
                0,
1703
                metadata,
1704
                "--materialization-baseline-subject must name one enabled Oak subject",
1705
            )
1706
            for subject in oak_subjects
1707
        ]
1708
    from oakbench.subjects import which_measured_git
1709
    git_bin = which_measured_git()
1710
    if git_bin is None:
1711
        return [
1712
            unmeasured_step_row(
1713
                subject,
1714
                scenario,
1715
                "scenario.skip",
1716
                0,
1717
                metadata,
1718
                "git is required only to create the deterministic main-history fixture",
1719
            )
1720
            for subject in oak_subjects
1721
        ]
1722
1723
    server_bin = server_bin.resolve()
1724
    resolved_server_source: Path | None = None
1725
    if server_source is not None:
1726
        if not server_source.is_absolute() or not server_source.is_dir():
1727
            return [
1728
                unmeasured_step_row(
1729
                    subject,
1730
                    scenario,
1731
                    "scenario.skip",
1732
                    0,
1733
                    metadata,
1734
                    "--oak-serve-source must name an absolute read-only source checkout",
1735
                )
1736
                for subject in oak_subjects
1737
            ]
1738
        resolved_server_source = server_source.resolve()
1739
        source_head_result = run_bounded_harness_command(
1740
            [str(server_bin), "hash"],
1741
            resolved_server_source,
1742
            env=isolated_oak_fixture_env(run_root / "source-check"),
1743
        )
1744
        if source_head_result.returncode != 0 or source_head_result.stdout.strip() != server_source_head:
1745
            return [
1746
                unmeasured_step_row(
1747
                    subject,
1748
                    scenario,
1749
                    "scenario.skip",
1750
                    0,
1751
                    metadata,
1752
                    "fixed Serve source checkout does not match --oak-serve-source-head",
1753
                )
1754
                for subject in oak_subjects
1755
            ]
1756
    baseline_binary_sha256 = binary_sha256(baseline_subject.bin)
1757
    timing_controls_met = materialization_timing_controls_met(
1758
        metadata, runs, randomize_order, timing_controls_requested
1759
    )
1760
    fixture_root = run_root / "legacy-selected-materialization-fixture"
1761
    server_root = fixture_root / "server"
1762
    git_seed = fixture_root / "git-seed.git"
1763
    seed = fixture_root / "seed"
1764
    task_config_root = fixture_root / "config"
1765
    for path in (server_root, git_seed, task_config_root):
1766
        path.mkdir(parents=True, exist_ok=True)
1767
    fixture_env = isolated_oak_fixture_env(task_config_root)
1768
    fixture_meta = {
1769
        **metadata,
1770
        "benchmark_track": "oak-local-loopback-materialization",
1771
        "remote_transport": "loopback_http",
1772
        "remote_server": "127.0.0.1",
1773
        "serve_binary": str(server_bin),
1774
        "serve_binary_sha256": binary_sha256(server_bin),
1775
        "serve_source": str(resolved_server_source) if resolved_server_source else None,
1776
        "serve_source_checkout_verification": (
1777
            "exact_head_verified" if resolved_server_source else "unavailable_binary_hash_and_head_supplied"
1778
        ),
1779
        "serve_source_head": server_source_head,
1780
        "serve_version": subject_version(Subject("serve_fixture", "oak", "Serve fixture", server_bin)),
1781
        "baseline_subject": baseline_subject.name,
1782
        "baseline_binary_sha256": baseline_binary_sha256,
1783
        "fixture_id": "legacy-selected-materialization-192x16k-v1",
1784
        "fixture_regular_files": 192,
1785
        "fixture_regular_file_bytes": 16 * 1024,
1786
        "fixture_empty_files": 1,
1787
        "fixture_executable_files": 8,
1788
        "fixture_selected_only_files": 1,
1789
        "os_cache_state": "warm_or_unknown_after_required_fixture_probe",
1790
        "materialization_timing_controls_met": timing_controls_met,
1791
        "materialization_timing_controls_reason": (
1792
            "harness_controls_met_aa_and_quiet_host_evidence_still_required"
1793
            if timing_controls_met
1794
            else "correctness_only_or_harness_controls_not_met"
1795
        ),
1796
    }
1797
    rows: list[dict[str, Any]] = []
1798
    setup_subject = baseline_subject
1799
1800
    def setup_command(operation: str, command: list[str], cwd: Path) -> bool:
1801
        started = time.perf_counter()
1802
        result = run_bounded_harness_command(command, cwd, env=fixture_env)
1803
        rows.append(
1804
            harness_step_row(
1805
                setup_subject,
1806
                scenario,
1807
                operation,
1808
                0,
1809
                fixture_meta,
1810
                (time.perf_counter() - started) * 1000,
1811
                returncode=0 if result.returncode == 0 else result.returncode,
1812
                harness_command_count=1,
1813
            )
1814
        )
1815
        if result.returncode != 0:
1816
            rows[-1]["stderr"] = result.stderr[-4000:]
1817
        return result.returncode == 0
1818
1819
    server: subprocess.Popen[Any] | None = None
1820
    proxy: CountingProxy | None = None
1821
    failure: str | None = None
1822
    expected_artifacts: list[str] | None = None
1823
    baseline_cache: dict[str, Any] | None = None
1824
    baseline_receipt_contract: dict[str, Any] | None = None
1825
    baseline_network_contract: dict[str, Any] | None = None
1826
    expected_status_contract: dict[str, Any] | None = None
1827
    selected_branch = "selected-materialization"
1828
    repo_spec = "qa/materialization-bench"
1829
    try:
1830
        for index in range(192):
1831
            directory = git_seed / f"group-{index % 12:02d}"
1832
            directory.mkdir(parents=True, exist_ok=True)
1833
            digest = hashlib.sha256(f"oak-materialization-{index}".encode()).digest()
1834
            content = (digest * ((16 * 1024 + len(digest) - 1) // len(digest)))[: 16 * 1024]
1835
            path = directory / f"file-{index:03d}.bin"
1836
            path.write_bytes(content)
1837
            if index < 8:
1838
                path.chmod(0o755)
1839
        (git_seed / "empty.txt").write_bytes(b"")
1840
        for operation, command in (
1841
            ("setup.git_init", [git_bin, "init", "-q", "-b", "main"]),
1842
            ("setup.git_add", [git_bin, "add", "."]),
1843
            ("setup.git_commit", [git_bin, "commit", "-q", "-m", "fixture main"]),
1844
        ):
1845
            if not setup_command(operation, command, git_seed):
1846
                failure = f"{operation}_failed"
1847
                break
1848
1849
        port = reserve_loopback_port()
1850
        direct_remote = f"http://127.0.0.1:{port}"
1851
        if failure is None:
1852
            server = subprocess.Popen(
1853
                [
1854
                    str(server_bin),
1855
                    "serve",
1856
                    "--dir",
1857
                    str(server_root),
1858
                    "--host",
1859
                    "127.0.0.1",
1860
                    "--port",
1861
                    str(port),
1862
                ],
1863
                cwd=fixture_root,
1864
                env=fixture_env,
1865
                stdout=subprocess.DEVNULL,
1866
                stderr=subprocess.DEVNULL,
1867
                start_new_session=os.name == "posix",
1868
            )
1869
            if not wait_for_loopback_port(server, port):
1870
                failure = "setup.server_not_ready"
1871
        if failure is None:
1872
            for operation, command, cwd in (
1873
                ("setup.oak_import", [str(server_bin), "clone", str(git_seed), str(seed)], fixture_root),
1874
                ("setup.branch", [str(server_bin), "switch", "-c", selected_branch], seed),
1875
            ):
1876
                if not setup_command(operation, command, cwd):
1877
                    failure = f"{operation}_failed"
1878
                    break
1879
        if failure is None:
1880
            (seed / "selected.txt").write_text("selected branch\n")
1881
            for operation, command in (
1882
                ("setup.oak_commit", [str(server_bin), "commit"]),
1883
                (
1884
                    "setup.oak_push",
1885
                    [str(server_bin), "push", "--repo", repo_spec, "--remote", direct_remote],
1886
                ),
1887
            ):
1888
                if not setup_command(operation, command, seed):
1889
                    failure = f"{operation}_failed"
1890
                    break
1891
1892
        expected_head_result = (
1893
            run_bounded_harness_command([str(server_bin), "hash"], seed, env=fixture_env)
1894
            if failure is None
1895
            else None
1896
        )
1897
        expected_head = (
1898
            expected_head_result.stdout.strip()
1899
            if expected_head_result is not None and expected_head_result.returncode == 0
1900
            else None
1901
        )
1902
        expected_history = (
1903
            normalized_history(json_command([str(server_bin), "log", "--json"], seed, fixture_env))
1904
            if failure is None
1905
            else None
1906
        )
1907
        expected_worktree = normalized_worktree_snapshot(seed) if failure is None else None
1908
        if not expected_head or expected_history is None or expected_worktree is None:
1909
            failure = failure or "setup.fixture_oracle_failed"
1910
1911
        if failure is None:
1912
            proxy = CountingProxy("127.0.0.1", port).start()
1913
            measured_remote = f"http://127.0.0.1:{proxy.address[1]}"
1914
            probe = fixture_root / "baseline-probe"
1915
            probe_env = isolated_oak_fixture_env(fixture_root / "baseline-probe-config")
1916
            proxy.reset_counts()
1917
            result = run_bounded_harness_command(
1918
                [
1919
                    str(baseline_subject.bin),
1920
                    "clone",
1921
                    "--json",
1922
                    "--allow-legacy-scope",
1923
                    "--remote",
1924
                    measured_remote,
1925
                    "--branch",
1926
                    selected_branch,
1927
                    repo_spec,
1928
                    str(probe),
1929
                ],
1930
                fixture_root,
1931
                env=probe_env,
1932
            )
1933
            receipt = _parse_json_object(result.stdout)
1934
            evidence = legacy_selected_receipt_evidence(receipt or {})
1935
            baseline_proxy_stats = proxy.snapshot()
1936
            baseline_cache = stat_cache_snapshot(probe)
1937
            expected_artifacts = normalized_oak_artifacts(probe)
1938
            baseline_receipt_contract = legacy_selected_receipt_contract(evidence)
1939
            baseline_network_contract = {
1940
                "connections": baseline_proxy_stats.connections,
1941
                "receipt": baseline_receipt_contract,
1942
            }
1943
            expected_status_contract = normalized_status(
1944
                json_command([str(baseline_subject.bin), "status", "--json"], probe, probe_env)
1945
            )
1946
            probe_checks = {
1947
                "returncode": result.returncode == 0,
1948
                "legacy_route": evidence["legacy_route_observed"],
1949
                "passes_two": evidence["materialization_passes_observed"] == 2,
1950
                "head": evidence["head"] == expected_head,
1951
                "manifest": bool(evidence["manifest"]),
1952
                "artifacts": expected_artifacts is not None,
1953
                "cache": normalized_cache_contract(baseline_cache) is not None,
1954
                "status": expected_status_contract is not None,
1955
                "network": baseline_proxy_stats.connection_errors == 0,
1956
            }
1957
            if not all(probe_checks.values()):
1958
                failed_checks = ",".join(key for key, passed in probe_checks.items() if not passed)
1959
                failure = f"baseline_probe_failed:{failed_checks}"
1960
                if isinstance(baseline_cache, dict) and baseline_cache.get("error"):
1961
                    failure += f":{baseline_cache['error']}"
1962
            shutil.rmtree(probe, ignore_errors=True)
1963
1964
        if failure is None and proxy is not None:
1965
            measured_remote = f"http://127.0.0.1:{proxy.address[1]}"
1966
            order_rng = random.Random(random_seed)
1967
            for run_index in range(runs):
1968
                run_subjects = list(oak_subjects)
1969
                if randomize_order:
1970
                    order_rng.shuffle(run_subjects)
1971
                for subject in run_subjects:
1972
                    destination = run_root / f"materialization-{subject.name}-r{run_index}"
1973
                    subject_env = isolated_oak_fixture_env(
1974
                        run_root / f"materialization-config-{subject.name}-r{run_index}"
1975
                    )
1976
                    proxy.reset_counts()
1977
                    row = step_row(
1978
                        subject,
1979
                        scenario,
1980
                        "legacy_selected.acquire",
1981
                        [
1982
                            str(subject.bin),
1983
                            "clone",
1984
                            "--json",
1985
                            "--allow-legacy-scope",
1986
                            "--remote",
1987
                            measured_remote,
1988
                            "--branch",
1989
                            selected_branch,
1990
                            repo_spec,
1991
                            str(destination),
1992
                        ],
1993
                        fixture_root,
1994
                        run_index,
1995
                        {
1996
                            **fixture_meta,
1997
                            "run_subject_order": [item.name for item in run_subjects],
1998
                        },
1999
                        admitted,
2000
                        preserve_stdout=True,
2001
                        env=subject_env,
2002
                        timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
2003
                    )
2004
                    receipt = _parse_json_object(row.get("stdout"))
2005
                    evidence = legacy_selected_receipt_evidence(receipt or {})
2006
                    proxy_stats = proxy.snapshot()
2007
                    cache_before_status = stat_cache_snapshot(destination)
2008
                    artifacts = normalized_oak_artifacts(destination)
2009
                    trial_meta = {
2010
                        **fixture_meta,
2011
                        "run_subject_order": [item.name for item in run_subjects],
2012
                    }
2013
                    status_row = step_row(
2014
                        subject,
2015
                        scenario,
2016
                        "legacy_selected.first_status",
2017
                        [str(subject.bin), "status", "--json"],
2018
                        destination,
2019
                        run_index,
2020
                        trial_meta,
2021
                        admitted,
2022
                        preserve_stdout=True,
2023
                        env=subject_env,
2024
                        timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
2025
                    )
2026
                    status = normalized_status(_parse_json_object(status_row.get("stdout")) or {})
2027
                    commit_row = step_row(
2028
                        subject,
2029
                        scenario,
2030
                        "legacy_selected.first_noop_commit",
2031
                        [str(subject.bin), "commit"],
2032
                        destination,
2033
                        run_index,
2034
                        trial_meta,
2035
                        admitted,
2036
                        env=subject_env,
2037
                        timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
2038
                    )
2039
                    history = normalized_history(
2040
                        json_command([str(subject.bin), "log", "--json"], destination, subject_env)
2041
                    )
2042
                    head_result = run_bounded_harness_command(
2043
                        [str(subject.bin), "hash"], destination, env=subject_env
2044
                    )
2045
                    local_head = head_result.stdout.strip() if head_result.returncode == 0 else None
2046
                    worktree = normalized_worktree_snapshot(destination) if destination.exists() else None
2047
                    advertised_head = remote_branch_head(direct_remote, repo_spec, selected_branch)
2048
                    subject_sha256 = binary_sha256(subject.bin)
2049
                    expected_passes = 2 if subject_sha256 == baseline_binary_sha256 else 1
2050
                    receipt_contract = legacy_selected_receipt_contract(evidence)
2051
                    network_contract = {
2052
                        "connections": proxy_stats.connections,
2053
                        "receipt": receipt_contract,
2054
                    }
2055
                    cache_contract = normalized_cache_contract(cache_before_status)
2056
                    baseline_cache_contract = normalized_cache_contract(baseline_cache)
2057
                    expected_cache_valid = subject_sha256 != baseline_binary_sha256
2058
                    cache_valid = bool(
2059
                        isinstance(cache_before_status, dict)
2060
                        and cache_before_status.get("all_metadata_valid") is expected_cache_valid
2061
                    )
2062
                    status_ok = status == expected_status_contract
2063
                    artifacts_ok = artifacts == expected_artifacts
2064
                    network_ok = network_contract == baseline_network_contract
2065
                    oracle_ok = bool(
2066
                        row_returncode(row) == 0
2067
                        and row_returncode(status_row) == 0
2068
                        and row_returncode(commit_row) == 0
2069
                        and evidence["legacy_route_observed"]
2070
                        and evidence["snapshot_bound"] is False
2071
                        and evidence["materialization_passes_observed"] == expected_passes
2072
                        and evidence["head"] == expected_head == local_head == advertised_head
2073
                        and evidence["manifest"] == baseline_receipt_contract["manifest"]
2074
                        and evidence["written_paths"] == len(expected_worktree)
2075
                        and receipt_contract == baseline_receipt_contract
2076
                        and history == expected_history
2077
                        and worktree == expected_worktree
2078
                        and status_ok
2079
                        and artifacts_ok
2080
                        and cache_contract == baseline_cache_contract
2081
                        and cache_valid
2082
                        and network_ok
2083
                        and proxy_stats.connection_errors == 0
2084
                    )
2085
                    row.update(
2086
                        {
2087
                            **evidence,
2088
                            "network_connections_observed": proxy_stats.connections,
2089
                            "network_client_to_server_bytes_observed": proxy_stats.client_to_upstream_bytes,
2090
                            "network_server_to_client_bytes_observed": proxy_stats.upstream_to_client_bytes,
2091
                            "network_connection_errors": proxy_stats.connection_errors,
2092
                            "normalized_network_work": network_contract,
2093
                            "oracle_network_work_matches_baseline": network_ok,
2094
                            "oracle_head_matches": expected_head == local_head == advertised_head,
2095
                            "oracle_manifest_matches": evidence["manifest"]
2096
                            == baseline_receipt_contract["manifest"],
2097
                            "oracle_receipt_contract_matches": receipt_contract
2098
                            == baseline_receipt_contract,
2099
                            "oracle_history_matches": history == expected_history,
2100
                            "oracle_worktree_bytes_and_modes_match": worktree == expected_worktree,
2101
                            "oracle_clean_status": status_ok,
2102
                            "oracle_local_artifact_names_match": artifacts_ok,
2103
                            "oracle_stat_cache_contract_matches": cache_contract
2104
                            == baseline_cache_contract,
2105
                            "stat_cache_metadata_valid_before_status": (
2106
                                cache_before_status.get("all_metadata_valid")
2107
                                if isinstance(cache_before_status, dict)
2108
                                else None
2109
                            ),
2110
                            "stat_cache_rows_before_status": (
2111
                                len(cache_before_status.get("rows", []))
2112
                                if isinstance(cache_before_status, dict)
2113
                                and isinstance(cache_before_status.get("rows"), list)
2114
                                else None
2115
                            ),
2116
                            "stat_cache_expected_valid_before_status": expected_cache_valid,
2117
                            "oracle_clone_residue_absent": artifacts_ok,
2118
                            "oracle_passed": oracle_ok,
2119
                            "repository_cache_state": "fresh_destination_no_shared_object_store",
2120
                        }
2121
                    )
2122
                    for followup_row in (status_row, commit_row):
2123
                        followup_row.update(
2124
                            {
2125
                                "materialization_timing_controls_met": timing_controls_met,
2126
                                "stat_cache_expected_valid_before_first_status": expected_cache_valid,
2127
                                "oracle_passed": oracle_ok,
2128
                            }
2129
                        )
2130
                    if not oracle_ok:
2131
                        row["returncode"] = 1
2132
                        row["stderr"] = "legacy_selected_materialization_oracle_failed"
2133
                        status_row["returncode"] = 1
2134
                        commit_row["returncode"] = 1
2135
                    total = phase_total_row(
2136
                        [row, status_row, commit_row],
2137
                        subject,
2138
                        scenario,
2139
                        run_index,
2140
                        trial_meta,
2141
                        "legacy_selected_clone_materialization.total",
2142
                    )
2143
                    total.update(
2144
                        {
2145
                            key: row[key]
2146
                            for key in (
2147
                                "materialization_passes_observed",
2148
                                "network_connections_observed",
2149
                                "network_client_to_server_bytes_observed",
2150
                                "network_server_to_client_bytes_observed",
2151
                                "oracle_passed",
2152
                                "oracle_head_matches",
2153
                                "oracle_manifest_matches",
2154
                                "oracle_receipt_contract_matches",
2155
                                "oracle_history_matches",
2156
                                "oracle_worktree_bytes_and_modes_match",
2157
                                "oracle_clean_status",
2158
                                "oracle_local_artifact_names_match",
2159
                                "oracle_stat_cache_contract_matches",
2160
                                "stat_cache_metadata_valid_before_status",
2161
                                "stat_cache_rows_before_status",
2162
                                "oracle_clone_residue_absent",
2163
                                "oracle_network_work_matches_baseline",
2164
                                "serve_binary_sha256",
2165
                                "serve_source_head",
2166
                                "fixture_id",
2167
                                "os_cache_state",
2168
                                "materialization_timing_controls_met",
2169
                            )
2170
                        }
2171
                    )
2172
                    total.update(
2173
                        {
2174
                            "clone_elapsed_ms": row.get("elapsed_ms"),
2175
                            "first_status_elapsed_ms": status_row.get("elapsed_ms"),
2176
                            "first_noop_commit_elapsed_ms": commit_row.get("elapsed_ms"),
2177
                        }
2178
                    )
2179
                    rows.extend([row, status_row, commit_row, total])
2180
                    shutil.rmtree(destination, ignore_errors=True)
2181
    finally:
2182
        if proxy is not None:
2183
            proxy.stop()
2184
        if server is not None:
2185
            stop_owned_process(server)
2186
        shutil.rmtree(fixture_root, ignore_errors=True)
2187
2188
    if failure is not None:
2189
        rows.append(
2190
            harness_step_row(
2191
                setup_subject,
2192
                scenario,
2193
                "scenario.fixture_failure",
2194
                0,
2195
                fixture_meta,
2196
                0.0,
2197
                returncode=1,
2198
            )
2199
        )
2200
        rows[-1]["stderr"] = failure
2201
    return rows
2202
2203
2204
def run_git_loop(
2205
    subject: Subject,
2206
    remote: oakbench_remotes.RemoteResolution,
2207
    run_root: Path,
2208
    run_index: int,
2209
    metadata: dict[str, Any],
2210
    admitted: int,
2211
) -> list[dict[str, Any]]:
2212
    scenario = "task_loop"
2213
    meta = {
2214
        **metadata,
2215
        **remote.row_fields(),
2216
        "benchmark_track": (
2217
            "core-equivalent"
2218
            if remote.transport == oakbench_remotes.TRANSPORT_LOCAL_FILE
2219
            else "agent-default"
2220
        ),
2221
    }
2222
    vcs = str(subject.bin)
2223
    rows: list[dict[str, Any]] = []
2224
2225
    seed_error = ensure_git_remote_seeded(subject, remote, run_root)
2226
    if seed_error:
2227
        return [skip_row(subject, scenario, run_index, meta, seed_error)]
2228
2229
    # Untimed: the base clone is the long-lived checkout worktrees hang off;
2230
    # an agent fleet pays it once, not per task.
2231
    base = run_root / f"tl-git-base-r{run_index}"
2232
    clone = run_untimed([vcs, "clone", "-q", str(remote.repo), str(base)], run_root)
2233
    if clone.returncode != 0:
2234
        return [skip_row(subject, scenario, run_index, meta, f"base clone failed: {clone.stderr.strip()[:200]}")]
2235
2236
    branches: dict[str, str] = {}
2237
    payloads: dict[str, str] = {}
2238
    for phase in PHASES:
2239
        workspace = run_root / f"tl-{subject.name}-r{run_index}-{phase}"
2240
        branch = oakbench_remotes.disposable_branch(
2241
            str(metadata["bench_id"]), f"taskloop-{phase}", subject.name, run_index
2242
        )
2243
        branches[phase] = branch
2244
        payload = f"payload {phase} run={run_index} bench={metadata['bench_id']}"
2245
        payloads[phase] = payload
2246
        acquire = step_row(
2247
            subject, scenario, f"{phase}.acquire",
2248
            [vcs, "worktree", "add", str(workspace), "-b", branch], base, run_index, meta, admitted,
2249
        )
2250
        rows.append(acquire)
2251
        if int(acquire["returncode"]) != 0:
2252
            return rows
2253
        rows.append(edit_step(subject, scenario, f"{phase}.edit", workspace, payload, run_index, meta, admitted))
2254
        # Agent-default snapshot: git pays stage+commit as two calls.
2255
        rows.append(step_row(subject, scenario, f"{phase}.snapshot.add", [vcs, "add", "."], workspace, run_index, meta, admitted))
2256
        rows.append(
2257
            step_row(
2258
                subject, scenario, f"{phase}.snapshot.commit",
2259
                [vcs, "commit", "-q", "-m", f"bench task loop {metadata['bench_id']} r{run_index} {phase}"],
2260
                workspace, run_index, meta, admitted,
2261
            )
2262
        )
2263
        rows.append(
2264
            step_row(
2265
                subject, scenario, f"{phase}.publish",
2266
                [vcs, "push", "-q", "-u", "origin", branch], workspace, run_index, meta, admitted,
2267
            )
2268
        )
2269
        # No task.describe: git's description is the commit message already
2270
        # paid in {phase}.snapshot.commit (asymmetry documented in the lane doc).
2271
        rows.append(
2272
            step_row(
2273
                subject, scenario, f"{phase}.end",
2274
                [vcs, "worktree", "remove", str(workspace)], base, run_index, meta, admitted,
2275
            )
2276
        )
2277
2278
    # Untimed oracle: payloads visible on the remote branches.
2279
    oracle_ok: bool | None = True
2280
    for phase in PHASES:
2281
        fetch = run_untimed([vcs, "fetch", "-q", "origin", branches[phase]], base)
2282
        show = run_untimed([vcs, "show", f"FETCH_HEAD:task-notes.md"], base)
2283
        if fetch.returncode != 0 or payloads[phase] not in show.stdout:
2284
            oracle_ok = False
2285
    rows.append(
2286
        {
2287
            **phase_total_row(rows, subject, scenario, run_index, meta, "task_loop.total"),
2288
            "oracle_payload_on_remote_branch": oracle_ok,
2289
            "workspace_branches": branches,
2290
            "measurement_source": "fetch_plus_show_grep",
2291
        }
2292
    )
2293
    return rows
2294
2295
2296
def loop_summary_fields(rows: list[dict[str, Any]]) -> dict[str, Any]:
2297
    """followup-vs-task warm deltas computed from this run's step rows."""
2298
    def op_ms(operation: str) -> float | None:
2299
        for row in rows:
2300
            if row.get("operation") == operation:
2301
                return float(row.get("elapsed_ms", 0.0))
2302
        return None
2303
2304
    def phase_ms(prefix: str) -> float:
2305
        return sum(
2306
            float(row.get("elapsed_ms", 0.0))
2307
            for row in rows
2308
            if str(row.get("operation", "")).startswith(prefix + ".")
2309
        )
2310
2311
    task_acquire = op_ms("task.acquire")
2312
    followup_acquire = op_ms("followup.acquire")
2313
    fields: dict[str, Any] = {
2314
        "task_phase_ms": round(phase_ms("task"), 3),
2315
        "followup_phase_ms": round(phase_ms("followup"), 3),
2316
    }
2317
    if task_acquire is not None and followup_acquire is not None:
2318
        fields["followup_warm_delta_ms"] = round(followup_acquire - task_acquire, 3)
2319
        fields["acquire_ms_task"] = round(task_acquire, 3)
2320
        fields["acquire_ms_followup"] = round(followup_acquire, 3)
2321
    else:
2322
        fields["followup_warm_delta_ms"] = None
2323
    return fields
2324
2325
2326
def summary_text(rows: list[dict[str, Any]]) -> str:
2327
    totals = [row for row in rows if row.get("operation") == "task_loop.total"]
2328
    lines = [
2329
        "# Task Loop Summary",
2330
        "",
2331
        "Whole-task cost of the signature motion: acquire workspace -> edit -> snapshot ->",
2332
        "publish -> describe -> end, then a follow-up task in a fresh workspace.",
2333
        "Command sequences are NOT step-equivalent across subjects",
2334
        "(oak pays desc; git's description is its commit message). Latency across unlike",
2335
        "transports is never comparable; scripted call equivalents and estimated token proxies are.",
2336
        "",
2337
        "| Subject | Run | Transport | Wall ms | Scripted call equivalents | Cost-wt tokens (env) | Warm acquire delta ms | Oracle |",
2338
        "| --- | ---: | --- | ---: | ---: | ---: | ---: | --- |",
2339
    ]
2340
    for row in totals:
2341
        oracle = row.get("oracle_payload_on_remote_branch")
2342
        lines.append(
2343
            "| `{}` | {} | {} | {} | {} | {} | {} | {} |".format(
2344
                row["subject"],
2345
                row["run"],
2346
                row.get("remote_transport", "unmeasured"),
2347
                fmt_ms(row.get("elapsed_ms")),
2348
                row.get("tool_call_count", "unmeasured"),
2349
                row.get("estimated_cost_weighted_tokens_with_envelope", "unmeasured"),
2350
                row.get("followup_warm_delta_ms", "unmeasured"),
2351
                "pass" if oracle else ("FAIL" if oracle is False else "unmeasured"),
2352
            )
2353
        )
2354
    skips = [
2355
        row for row in rows
2356
        if row.get("skipped") and row.get("scenario") == TASK_LOOP_SCENARIO
2357
    ]
2358
    for row in skips:
2359
        lines.append(f"| `{row['subject']}` | {row['run']} | skipped | | | | | {row['skip_reason'][:70]} |")
2360
2361
    agent_totals = [
2362
        row
2363
        for row in rows
2364
        if row.get("operation") in ("agent_task.total", "agent_task_oak_local.total")
2365
    ]
2366
    if agent_totals:
2367
        lines.extend(
2368
            [
2369
                "",
2370
                "## Cold clone and warm isolated task",
2371
                "",
2372
                "Local-file Git rows compare a fresh full clone with a worktree sharing",
2373
                "the first clone's object store. Oak local rows use two independent fresh",
2374
                "clones against an owned loopback server; they do not claim a shared cache.",
2375
                "OS cache and network bytes remain unknown. Hosted CI/review are skips.",
2376
                "",
2377
                "| Subject | Run | Wall ms | Scripted call equivalents | Output bytes | Coverage | Oracle |",
2378
                "| --- | ---: | ---: | ---: | ---: | --- | --- |",
2379
            ]
2380
        )
2381
        for row in agent_totals:
2382
            lines.append(
2383
                "| `{}` | {} | {} | {} | {} | {} | {} |".format(
2384
                    row["subject"],
2385
                    row["run"],
2386
                    fmt_ms(row.get("elapsed_ms")),
2387
                    row.get("tool_call_count", "unmeasured"),
2388
                    row.get("raw_output_bytes", "unmeasured"),
2389
                    "complete" if row.get("coverage_complete") else "partial",
2390
                    "pass" if row.get("oracle_payload_on_remote_branch") else "FAIL",
2391
                )
2392
            )
2393
2394
    materialization_totals = [
2395
        row
2396
        for row in rows
2397
        if row.get("operation") == "legacy_selected_clone_materialization.total"
2398
    ]
2399
    if materialization_totals:
2400
        lines.extend(
2401
            [
2402
                "",
2403
                "## Legacy selected-branch materialization",
2404
                "",
2405
                "One fixed loopback Serve and one immutable fixture are shared across subjects.",
2406
                "Fresh destinations do not imply a cold OS page cache. Timing claims require",
2407
                "separate interleaved A/A noise and A/B campaigns.",
2408
                "",
2409
                "| Subject | Run | Clone ms | First status ms | First no-op commit ms | Passes | Harness controls met | Oracle |",
2410
                "| --- | ---: | ---: | ---: | ---: | ---: | --- | --- |",
2411
            ]
2412
        )
2413
        for row in materialization_totals:
2414
            lines.append(
2415
                "| `{}` | {} | {} | {} | {} | {} | {} | {} |".format(
2416
                    row["subject"],
2417
                    row["run"],
2418
                    fmt_ms(row.get("clone_elapsed_ms")),
2419
                    fmt_ms(row.get("first_status_elapsed_ms")),
2420
                    fmt_ms(row.get("first_noop_commit_elapsed_ms")),
2421
                    row.get("materialization_passes_observed", "unmeasured"),
2422
                    "yes" if row.get("materialization_timing_controls_met") else "no",
2423
                    "pass" if row.get("oracle_passed") else "FAIL",
2424
                )
2425
            )
2426
2427
    interrupted_totals = [
2428
        row for row in rows if row.get("operation") == "interrupted_publication.total"
2429
    ]
2430
    if interrupted_totals:
2431
        lines.extend(
2432
            [
2433
                "",
2434
                "## Interrupted publication",
2435
                "",
2436
                "A loopback fixture validates the request framing and drops the response.",
2437
                "Fixture delivery knowledge is reported separately from the CLI's own outcome",
2438
                "claim. Request receipt is not durability proof; the harness never retries.",
2439
                "",
2440
                "| Subject | Run | Wall ms | Scripted call equivalents | Output bytes | CLI outcome | Fixture delivery | POSTs | Request bytes | Head/work preserved |",
2441
                "| --- | ---: | ---: | ---: | ---: | --- | --- | ---: | ---: | --- |",
2442
            ]
2443
        )
2444
        for row in interrupted_totals:
2445
            lines.append(
2446
                "| `{}` | {} | {} | {} | {} | `{}` | `{}` | {} | {} | {}/{} |".format(
2447
                    row["subject"],
2448
                    row["run"],
2449
                    fmt_ms(row.get("elapsed_ms")),
2450
                    row.get("tool_call_count", "unmeasured"),
2451
                    row.get("raw_output_bytes", "unmeasured"),
2452
                    row.get("publication_outcome", "unmeasured"),
2453
                    row.get("fixture_delivery_state", "unmeasured"),
2454
                    row.get("mutation_requests_observed", "unmeasured"),
2455
                    row.get("network_request_bytes_observed", "unmeasured"),
2456
                    "yes" if row.get("local_head_preserved") else "no",
2457
                    "yes" if row.get("local_work_preserved") else "no",
2458
                )
2459
            )
2460
    return "\n".join(lines) + "\n"
2461
2462
2463
def fmt_ms(value: Any) -> str:
2464
    try:
2465
        return f"{float(value):.0f}"
2466
    except (TypeError, ValueError):
2467
        return "unmeasured"
2468
2469
2470
def main() -> int:
2471
    args = parse_args()
2472
    scenarios = [item.strip() for item in args.scenarios.split(",") if item.strip()]
2473
    known_scenarios = {
2474
        TASK_LOOP_SCENARIO,
2475
        AGENT_TASK_SCENARIO,
2476
        AGENT_TASK_OAK_LOCAL_SCENARIO,
2477
        LEGACY_SELECTED_MATERIALIZATION_SCENARIO,
2478
        INTERRUPTED_PUBLICATION_SCENARIO,
2479
    }
2480
    unknown_scenarios = sorted(set(scenarios) - known_scenarios)
2481
    if unknown_scenarios:
2482
        raise SystemExit(
2483
            "Unknown scenarios: " + ", ".join(unknown_scenarios)
2484
            + "; valid: " + ", ".join(sorted(known_scenarios))
2485
        )
2486
    subjects = load_subjects(args)
2487
    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
2488
    args.results.mkdir(parents=True, exist_ok=True)
2489
    args.workdir.mkdir(parents=True, exist_ok=True)
2490
    run_root = args.workdir / "runs" / timestamp
2491
    run_root.mkdir(parents=True, exist_ok=True)
2492
2493
    metadata = {
2494
        "bench_id": timestamp,
2495
        "profile": "task-loop",
2496
        "benchmark_track": "agent-default",
2497
        "timestamp_utc": timestamp,
2498
        "host": platform.node(),
2499
        "platform": platform.platform(),
2500
        "machine": platform.machine(),
2501
        "python": platform.python_version(),
2502
        "env_isolation_version": ENV_ISOLATION_VERSION,
2503
        "subject_versions": subject_versions(subjects),
2504
        "subject_details": subject_details(subjects),
2505
        "source": source_metadata(args.oak_repo),
2506
        "git_remote_mode": args.git_remote,
2507
        "trial_order_randomized": args.randomize_subject_order,
2508
        "trial_order_seed": args.random_seed if args.randomize_subject_order else None,
2509
    }
2510
2511
    rows: list[dict[str, Any]] = []
2512
    with measurement_lock("task_loop") as lock_info:
2513
        metadata["measurement_lock_wait_ms"] = lock_info.wait_ms
2514
        metadata["measurement_lock"] = "held" if lock_info.enabled else "disabled"
2515
        order_rng = random.Random(args.random_seed)
2516
        if LEGACY_SELECTED_MATERIALIZATION_SCENARIO in scenarios:
2517
            rows.extend(
2518
                run_legacy_selected_materialization_campaign(
2519
                    subjects,
2520
                    args.oak_serve_bin,
2521
                    args.oak_serve_source,
2522
                    args.oak_serve_source_head,
2523
                    args.materialization_baseline_subject,
2524
                    run_root,
2525
                    args.runs,
2526
                    metadata,
2527
                    args.admitted_output_chars,
2528
                    args.randomize_subject_order,
2529
                    args.random_seed,
2530
                    args.materialization_timing_controls_met,
2531
                )
2532
            )
2533
        ordinary_scenarios = [
2534
            scenario for scenario in scenarios if scenario != LEGACY_SELECTED_MATERIALIZATION_SCENARIO
2535
        ]
2536
        for run_index in range(args.runs):
2537
            run_subjects = list(subjects)
2538
            if args.randomize_subject_order:
2539
                order_rng.shuffle(run_subjects)
2540
            run_metadata = {
2541
                **metadata,
2542
                "run_subject_order": [subject.name for subject in run_subjects],
2543
            }
2544
            for subject in run_subjects:
2545
                for scenario in ordinary_scenarios:
2546
                    print(f"[run] {scenario} run={run_index} subject={subject.name}", flush=True)
2547
                    if scenario == INTERRUPTED_PUBLICATION_SCENARIO:
2548
                        rows.extend(
2549
                            run_interrupted_publication(
2550
                                subject, run_root, run_index, run_metadata, args.admitted_output_chars
2551
                            )
2552
                        )
2553
                        continue
2554
                    if scenario == AGENT_TASK_SCENARIO:
2555
                        if subject.kind != "git":
2556
                            rows.append(
2557
                                unmeasured_step_row(
2558
                                    subject,
2559
                                    scenario,
2560
                                    "agent_task.skipped",
2561
                                    run_index,
2562
                                    run_metadata,
2563
                                    "Oak end-to-end publication and exact CI require an explicitly configured disposable staging lane; production mutation is forbidden",
2564
                                )
2565
                            )
2566
                            continue
2567
                        remote_create_start = time.perf_counter()
2568
                        remote = oakbench_remotes.make_git_bare_remote(
2569
                            subject.bin, run_root, f"agent-task-remote-r{run_index}.git"
2570
                        )
2571
                        remote_create_ms = (time.perf_counter() - remote_create_start) * 1000
2572
                        rows.extend(
2573
                            run_git_agent_task_baseline(
2574
                                subject,
2575
                                remote,
2576
                                run_root,
2577
                                run_index,
2578
                                run_metadata,
2579
                                args.admitted_output_chars,
2580
                                remote_create_ms,
2581
                            )
2582
                        )
2583
                        continue
2584
                    if scenario == AGENT_TASK_OAK_LOCAL_SCENARIO:
2585
                        if subject.kind != "oak":
2586
                            rows.append(
2587
                                unmeasured_step_row(
2588
                                    subject,
2589
                                    scenario,
2590
                                    "agent_task_oak_local.skipped",
2591
                                    run_index,
2592
                                    run_metadata,
2593
                                    "owned local oak serve scenario applies only to Oak subjects",
2594
                                )
2595
                            )
2596
                            continue
2597
                        rows.extend(
2598
                            run_oak_local_agent_task_baseline(
2599
                                subject,
2600
                                run_root,
2601
                                run_index,
2602
                                run_metadata,
2603
                                args.admitted_output_chars,
2604
                            )
2605
                        )
2606
                        continue
2607
                    if subject.kind == "git":
2608
                        if args.git_remote == "github":
2609
                            remote = oakbench_remotes.resolve_git_github_remote()
2610
                        else:
2611
                            remote = oakbench_remotes.make_git_bare_remote(
2612
                                subject.bin, run_root, f"task-loop-remote-r{run_index}.git"
2613
                            )
2614
                        if not remote.resolved:
2615
                            rows.append(skip_row(subject, TASK_LOOP_SCENARIO, run_index, run_metadata, remote.skip_reason or ""))
2616
                            continue
2617
                        subject_rows = run_git_loop(subject, remote, run_root, run_index, run_metadata, args.admitted_output_chars)
2618
                    else:
2619
                        remote = oakbench_remotes.resolve_oak_remote("sync")
2620
                        if not remote.resolved:
2621
                            rows.append(skip_row(subject, TASK_LOOP_SCENARIO, run_index, run_metadata, remote.skip_reason or ""))
2622
                            continue
2623
                        subject_rows = run_oak_loop(subject, remote, run_root, run_index, run_metadata, args.admitted_output_chars)
2624
                    total = next((row for row in subject_rows if row.get("operation") == "task_loop.total"), None)
2625
                    if total is not None:
2626
                        total.update(loop_summary_fields(subject_rows))
2627
                    rows.extend(subject_rows)
2628
2629
    store = ResultsStore(args.results, lane="task-loop")
2630
    raw_path, summary_path = store.write(timestamp, rows, summary_text(rows))
2631
    if not args.keep_workdirs:
2632
        shutil.rmtree(run_root, ignore_errors=True)
2633
    print(f"[result] {raw_path}")
2634
    print(f"[summary] {summary_path}")
2635
    return 1 if has_lane_failures(rows) else 0
2636
2637
2638
if __name__ == "__main__":
2639
    raise SystemExit(main())