Log in
scripts/task_loop.py 2638 lines · 105.9 KB · python Blame
1
#!/usr/bin/env python3
2
"""The full task loop, end to end β€” the product's signature motion.
3
4
oak:  mount -> edit -> commit -> push -> desc -> end, then REMOUNT for a
5
      follow-up task (the warm-cache story) and repeat edit/commit/push/end.
6
git:  worktree add -> branch edit -> add/commit -> push -> worktree remove,
7
      then a second worktree for the follow-up. The task description lives in
8
      the commit message git already paid for; oak pays a separate `desc`
9
      call. That asymmetry is the measurement, not a harness gap
10
      (docs/command-semantics.md): this is agent-default-track, whole-task
11
      evidence, never a step-by-step equivalence claim.
12
13
Transport honesty: oak rows always cross the network to a real oak server.
14
git runs against GitHub (--git-remote github, transport-comparable) or a
15
local bare repo (--git-remote local, the core-equivalent track). Rows carry
16
remote_transport/remote_server; latency comparisons across transports are
17
forbidden β€” cross-subject evidence at unlike transports is tool calls and
18
tokens only.
19
20
Setup:
21
  OAK_BENCH_SYNC_REPO=oak/bench-sync-tmp \\
22
  [email protected]:oakdotspace/bench-task-loop-tmp.git \\
23
  python3 scripts/task_loop.py --runs 3
24
25
Unconfigured remotes emit skip rows (returncode 77), never silent absence.
26
"""
27
28
from __future__ import annotations
29
30
import argparse
31
import hashlib
32
import json
33
import os
34
import platform
35
import random
36
import re
37
import shutil
38
import socket
39
import sqlite3
40
import stat
41
import subprocess
42
import tempfile
43
import threading
44
import time
45
import urllib.error
46
import urllib.parse
47
import urllib.request
48
from datetime import datetime, timezone
49
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
50
from pathlib import Path
51
from typing import Any
52
53
from oakbench import remotes as oakbench_remotes
54
from oakbench import tokens as oakbench_tokens
55
from oakbench.byteproxy import CountingProxy
56
from oakbench.diskprobe import bounded_tree_usage
57
from oakbench.environment import ENV_ISOLATION_VERSION, base_env as oakbench_base_env
58
from oakbench.environment import command_display
59
from oakbench.execution import run_timed, stop_owned_process
60
from oakbench.results import ResultsStore
61
from oakbench.rows import row_returncode
62
from oakbench.runlock import measurement_lock
63
from oakbench.subjects import (
64
    DEFAULT_OAK_REPO,
65
    Subject,
66
    binary_sha256,
67
    load_subjects,
68
    source_metadata,
69
    subject_details,
70
    subject_version,
71
    subject_versions,
72
)
73
74
ROOT = Path(__file__).resolve().parents[1]
75
DEFAULT_WORKDIR = Path(tempfile.gettempdir()) / "oak-task-loop"
76
SKIP_RETURNCODE = 77
77
VIRTUAL_BRANCH_RE = re.compile(r"virtual branch ([^\s)]+)")
78
PHASES = ("task", "followup")
79
TASK_LOOP_SCENARIO = "task_loop"
80
AGENT_TASK_SCENARIO = "agent_task_baseline_v1"
81
AGENT_TASK_OAK_LOCAL_SCENARIO = "agent_task_oak_local_v1"
82
INTERRUPTED_PUBLICATION_SCENARIO = "interrupted_publication_after_request_v1"
83
LEGACY_SELECTED_MATERIALIZATION_SCENARIO = "legacy_selected_clone_materialization_v1"
84
INTERRUPTED_PUBLICATION_REQUEST_TIMEOUT_S = 2.0
85
INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S = 30.0
86
INTERRUPTED_PUBLICATION_MAX_REQUEST_BYTES = 64 * 1024 * 1024
87
LOCAL_OAK_SERVER_READY_TIMEOUT_S = 10.0
88
89
90
def legacy_selected_receipt_evidence(receipt: dict[str, Any]) -> dict[str, Any]:
91
    result = receipt.get("result") if isinstance(receipt.get("result"), dict) else {}
92
    observed = receipt.get("observed") if isinstance(receipt.get("observed"), dict) else {}
93
    phases = observed.get("pull_phases") if isinstance(observed.get("pull_phases"), list) else []
94
    phase_names = [phase.get("name") for phase in phases if isinstance(phase, dict)]
95
    materialization = (
96
        observed.get("materialization")
97
        if isinstance(observed.get("materialization"), dict)
98
        else {}
99
    )
100
    return {
101
        "legacy_route_observed": phase_names == ["initial", "legacy_selected_branch"],
102
        "pull_phase_names": phase_names,
103
        "pull_phases": phases,
104
        "materialization_passes_observed": materialization.get("passes_observed"),
105
        "written_paths": materialization.get("written_paths"),
106
        "snapshot_bound": result.get("snapshot_bound"),
107
        "head": result.get("head"),
108
        "manifest": result.get("manifest"),
109
    }
110
111
112
def legacy_selected_receipt_contract(evidence: dict[str, Any]) -> dict[str, Any]:
113
    phases = []
114
    for phase in evidence.get("pull_phases", []):
115
        if not isinstance(phase, dict):
116
            continue
117
        phases.append(
118
            {
119
                key: value
120
                for key, value in phase.items()
121
                if key not in {"materialization_passes"}
122
            }
123
        )
124
    return {
125
        "legacy_route_observed": evidence.get("legacy_route_observed"),
126
        "pull_phases": phases,
127
        "snapshot_bound": evidence.get("snapshot_bound"),
128
        "head": evidence.get("head"),
129
        "manifest": evidence.get("manifest"),
130
        "written_paths": evidence.get("written_paths"),
131
    }
132
133
134
def normalized_cache_contract(snapshot: dict[str, Any] | None) -> list[dict[str, Any]] | None:
135
    if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
136
        return None
137
    return [
138
        {key: row.get(key) for key in ("path", "size", "blob_hash")}
139
        for row in snapshot["rows"]
140
        if isinstance(row, dict)
141
    ]
142
143
144
def normalized_status(value: dict[str, Any] | list[Any] | None) -> dict[str, Any] | None:
145
    if not isinstance(value, dict):
146
        return None
147
    return {
148
        key: value.get(key)
149
        for key in ("branch", "parent", "branch_status", "head", "working_changes", "branch_changes")
150
    }
151
152
153
def materialization_timing_controls_met(
154
    metadata: dict[str, Any], runs: int, randomize_order: bool, controls_requested: bool
155
) -> bool:
156
    return bool(
157
        controls_requested
158
        and metadata.get("measurement_lock") == "held"
159
        and randomize_order
160
        and runs >= 10
161
    )
162
163
164
def isolated_oak_fixture_env(task_config_root: Path) -> dict[str, str]:
165
    task_config_root.mkdir(mode=0o700, parents=True, exist_ok=True)
166
    retained = {
167
        "PATH",
168
        "TMPDIR",
169
        "TMP",
170
        "TEMP",
171
        "HOME",
172
        "USER",
173
        "USERNAME",
174
        "SHELL",
175
        "SYSTEMROOT",
176
        "WINDIR",
177
        "COMSPEC",
178
        "PATHEXT",
179
        "SSL_CERT_FILE",
180
        "SSL_CERT_DIR",
181
    }
182
    env = {key: value for key, value in os.environ.items() if key in retained}
183
    env.update(
184
        {
185
            "GIT_AUTHOR_NAME": "Oak Materialization Bench",
186
            "GIT_AUTHOR_EMAIL": "[email protected]",
187
            "GIT_COMMITTER_NAME": "Oak Materialization Bench",
188
            "GIT_COMMITTER_EMAIL": "[email protected]",
189
            "GIT_CONFIG_GLOBAL": os.devnull,
190
            "GIT_CONFIG_NOSYSTEM": "1",
191
            "OAK_AUTHOR": "oak-materialization-bench",
192
            "OAK_NO_UPDATE_CHECK": "1",
193
            "OAK_BENCH_ENV_ISOLATION_VERSION": ENV_ISOLATION_VERSION,
194
            "NO_COLOR": "1",
195
            "CLICOLOR": "0",
196
            "LC_ALL": "C",
197
            "LANG": "C",
198
            "TZ": "UTC",
199
            "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z",
200
            "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z",
201
        }
202
    )
203
    return env
204
205
206
def normalized_worktree_snapshot(root: Path) -> list[dict[str, Any]]:
207
    rows: list[dict[str, Any]] = []
208
    for path in sorted(root.rglob("*")):
209
        relative = path.relative_to(root)
210
        if relative.parts and relative.parts[0] in {".oak", ".git"}:
211
            continue
212
        info = path.lstat()
213
        if stat.S_ISDIR(info.st_mode):
214
            continue
215
        row: dict[str, Any] = {
216
            "path": relative.as_posix(),
217
            "mode": stat.S_IMODE(info.st_mode),
218
        }
219
        if stat.S_ISLNK(info.st_mode):
220
            row.update({"kind": "symlink", "target": os.readlink(path)})
221
        else:
222
            row.update({"kind": "file", "sha256": hashlib.sha256(path.read_bytes()).hexdigest()})
223
        rows.append(row)
224
    return rows
225
226
227
def json_command(
228
    command: list[str], cwd: Path, env: dict[str, str]
229
) -> dict[str, Any] | list[Any] | None:
230
    result = run_bounded_harness_command(command, cwd, env=env)
231
    if result.returncode != 0:
232
        return None
233
    try:
234
        value = json.loads(result.stdout)
235
    except (TypeError, ValueError):
236
        return None
237
    return value if isinstance(value, (dict, list)) else None
238
239
240
def normalized_history(value: dict[str, Any] | list[Any] | None) -> list[dict[str, Any]] | None:
241
    if not isinstance(value, list):
242
        return None
243
    rows: list[dict[str, Any]] = []
244
    for item in value:
245
        if not isinstance(item, dict):
246
            return None
247
        rows.append(
248
            {
249
                key: item.get(key)
250
                for key in (
251
                    "hash",
252
                    "timestamp",
253
                    "branch",
254
                    "description_or_subject",
255
                    "files_changed",
256
                )
257
            }
258
        )
259
    return rows
260
261
262
def normalized_oak_artifacts(root: Path) -> list[str] | None:
263
    oak_dir = root / ".oak"
264
    if not oak_dir.is_dir():
265
        return None
266
    return sorted(
267
        path.relative_to(oak_dir).as_posix() + ("/" if path.is_dir() else "")
268
        for path in oak_dir.rglob("*")
269
    )
270
271
272
def stat_cache_snapshot(root: Path) -> dict[str, Any] | None:
273
    db = root / ".oak/oak.db"
274
    if not db.is_file():
275
        return {"rows": None, "all_metadata_valid": False, "error": "oak_db_missing"}
276
    try:
277
        connection = sqlite3.connect(str(db))
278
        connection.execute("PRAGMA query_only = ON")
279
        raw_rows = connection.execute(
280
            "SELECT path, mtime_ns, ctime_ns, size, blob_hash FROM stat_cache ORDER BY path"
281
        ).fetchall()
282
    except sqlite3.Error as error:
283
        return {"rows": None, "all_metadata_valid": False, "error": str(error)}
284
    finally:
285
        if "connection" in locals():
286
            connection.close()
287
    rows: list[dict[str, Any]] = []
288
    valid = True
289
    for path, mtime_ns, ctime_ns, size, blob_hash in raw_rows:
290
        file_path = root / path
291
        try:
292
            info = file_path.stat()
293
        except OSError:
294
            valid = False
295
            continue
296
        expected_ctime = info.st_ctime_ns if os.name == "posix" else 0
297
        metadata_valid = (
298
            int(size) == info.st_size
299
            and int(mtime_ns) == info.st_mtime_ns
300
            and int(ctime_ns) == expected_ctime
301
        )
302
        valid = valid and metadata_valid
303
        rows.append(
304
            {
305
                "path": str(path),
306
                "size": int(size),
307
                "blob_hash": str(blob_hash),
308
                "metadata_valid": metadata_valid,
309
            }
310
        )
311
    return {"rows": rows, "all_metadata_valid": valid}
312
313
314
def base_env() -> dict[str, str]:
315
    return oakbench_base_env(
316
        author_name="Oak Task Loop",
317
        author_email="[email protected]",
318
        oak_author="oak-task-loop",
319
    )
320
321
322
def parse_args() -> argparse.Namespace:
323
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
324
    parser.add_argument("--subjects", help="Comma-separated subject names from config/subjects.toml")
325
    parser.add_argument("--config", type=Path, default=ROOT / "config" / "subjects.toml")
326
    parser.add_argument("--git-bin", type=Path)
327
    parser.add_argument("--oak-installed-bin", type=Path)
328
    parser.add_argument("--oak-local-bin", type=Path)
329
    parser.add_argument(
330
        "--oak-serve-bin",
331
        type=Path,
332
        help="Immutable Oak binary used only as the fixed loopback server and fixture writer.",
333
    )
334
    parser.add_argument(
335
        "--oak-serve-source",
336
        type=Path,
337
        help="Read-only source checkout that produced --oak-serve-bin.",
338
    )
339
    parser.add_argument(
340
        "--oak-serve-source-head",
341
        help="Exact 64-hex Oak source head that produced --oak-serve-bin.",
342
    )
343
    parser.add_argument(
344
        "--materialization-baseline-subject",
345
        help="Named control subject required by the legacy materialization scenario.",
346
    )
347
    parser.add_argument(
348
        "--materialization-timing-controls-met",
349
        action="store_true",
350
        help=(
351
            "Record that harness-verifiable controls are met only with a held measurement lock, "
352
            "randomized subject order, and at least 10 runs. This is not a speedup claim: "
353
            "separate A/A noise and externally verified quiet-host evidence remain required."
354
        ),
355
    )
356
    parser.add_argument("--runs", type=int, default=3)
357
    parser.add_argument(
358
        "--scenarios",
359
        default=TASK_LOOP_SCENARIO,
360
        help=(
361
            "Comma-separated scenarios: task_loop, agent_task_baseline_v1, "
362
            "agent_task_oak_local_v1, legacy_selected_clone_materialization_v1, "
363
            "and/or interrupted_publication_after_request_v1. "
364
            "The interruption "
365
            "scenario is a loopback-only Oak fixture and never contacts a remote service."
366
        ),
367
    )
368
    parser.add_argument(
369
        "--git-remote",
370
        choices=("github", "local"),
371
        default="github",
372
        help=(
373
            "github: agent-default track, real push to GIT_BENCH_REMOTE "
374
            "(transport-comparable to oak). local: core-equivalent track, "
375
            "local bare remote (no network in git's loop)."
376
        ),
377
    )
378
    parser.add_argument("--workdir", type=Path, default=DEFAULT_WORKDIR)
379
    parser.add_argument("--results", type=Path, default=ROOT / "results" / "task-loop")
380
    parser.add_argument("--oak-repo", type=Path, default=DEFAULT_OAK_REPO)
381
    parser.add_argument("--admitted-output-chars", type=int, default=20_000)
382
    parser.add_argument("--keep-workdirs", action="store_true")
383
    parser.add_argument(
384
        "--randomize-subject-order",
385
        action="store_true",
386
        help="Shuffle subject order independently in each run using --random-seed.",
387
    )
388
    parser.add_argument("--random-seed", type=int, default=20260908)
389
    return parser.parse_args()
390
391
392
def run_untimed(command: list[str], cwd: Path | None, timeout: float = 300.0) -> subprocess.CompletedProcess[str]:
393
    return subprocess.run(
394
        command,
395
        cwd=cwd,
396
        env=base_env(),
397
        text=True,
398
        stdout=subprocess.PIPE,
399
        stderr=subprocess.PIPE,
400
        timeout=timeout,
401
        check=False,
402
    )
403
404
405
def step_row(
406
    subject: Subject,
407
    scenario: str,
408
    operation: str,
409
    command: list[str],
410
    cwd: Path,
411
    run_index: int,
412
    metadata: dict[str, Any],
413
    admitted_output_chars: int,
414
    *,
415
    kind: str = "vcs",
416
    expected_returncodes: tuple[int, ...] = (0,),
417
    preserve_stdout: bool = False,
418
    preserve_stderr: bool = False,
419
    env: dict[str, str] | None = None,
420
    timeout_seconds: float | None = None,
421
) -> dict[str, Any]:
422
    capture = run_timed(
423
        command,
424
        cwd,
425
        env or base_env(),
426
        admitted_output_chars,
427
        timeout_seconds=timeout_seconds,
428
    )
429
    expected = not capture.timed_out and capture.returncode in expected_returncodes
430
    is_vcs = 1 if kind == "vcs" else 0
431
    row = {
432
        **metadata,
433
        "subject": subject.name,
434
        "subject_kind": subject.kind,
435
        "subject_label": subject.label,
436
        "scenario": scenario,
437
        "run": run_index,
438
        "operation": operation,
439
        "step_kind": kind,
440
        "elapsed_ms": round(capture.elapsed_ms, 3),
441
        "returncode": 0 if expected else (capture.returncode or 1),
442
        "process_returncode": capture.returncode,
443
        "timed_out": capture.timed_out,
444
        "expected_returncodes": list(expected_returncodes),
445
        "command": command,
446
        "tool_call_count": 1,
447
        "terminal_tool_call_count": 1,
448
        "vcs_tool_call_count": is_vcs,
449
        **oakbench_tokens.interaction_token_fields(
450
            command_display(command),
451
            capture.stdout_text,
452
            capture.stderr_text,
453
            capture.stdout_bytes,
454
            capture.stderr_bytes,
455
            capture.stdout_truncated,
456
            capture.stderr_truncated,
457
        ),
458
    }
459
    if not expected:
460
        row["stderr"] = capture.stderr_text[-4000:]
461
    if preserve_stdout:
462
        row["stdout"] = capture.stdout_text
463
    if preserve_stderr:
464
        row["stderr"] = capture.stderr_text
465
    return row
466
467
468
def skip_row(
469
    subject: Subject,
470
    scenario: str,
471
    run_index: int,
472
    metadata: dict[str, Any],
473
    reason: str,
474
) -> dict[str, Any]:
475
    return {
476
        **metadata,
477
        "subject": subject.name,
478
        "subject_kind": subject.kind,
479
        "subject_label": subject.label,
480
        "scenario": scenario,
481
        "run": run_index,
482
        "operation": "task_loop.skipped",
483
        "step_kind": "summary",
484
        "elapsed_ms": 0.0,
485
        "returncode": SKIP_RETURNCODE,
486
        "command": [],
487
        "skipped": True,
488
        "skip_reason": reason,
489
        "tool_call_count": 0,
490
        "terminal_tool_call_count": 0,
491
        "vcs_tool_call_count": 0,
492
    }
493
494
495
def unmeasured_step_row(
496
    subject: Subject,
497
    scenario: str,
498
    operation: str,
499
    run_index: int,
500
    metadata: dict[str, Any],
501
    reason: str,
502
) -> dict[str, Any]:
503
    row = skip_row(subject, scenario, run_index, metadata, reason)
504
    row["operation"] = operation
505
    return row
506
507
508
def harness_step_row(
509
    subject: Subject,
510
    scenario: str,
511
    operation: str,
512
    run_index: int,
513
    metadata: dict[str, Any],
514
    elapsed_ms: float,
515
    *,
516
    returncode: int = 0,
517
    harness_command_count: int = 0,
518
) -> dict[str, Any]:
519
    return {
520
        **metadata,
521
        "subject": subject.name,
522
        "subject_kind": subject.kind,
523
        "subject_label": subject.label,
524
        "scenario": scenario,
525
        "run": run_index,
526
        "operation": operation,
527
        "step_kind": "harness",
528
        "elapsed_ms": round(elapsed_ms, 3),
529
        "returncode": returncode,
530
        "command": [],
531
        "tool_call_count": 0,
532
        "terminal_tool_call_count": 0,
533
        "vcs_tool_call_count": 0,
534
        "harness_command_count": harness_command_count,
535
        "stdout_bytes": 0,
536
        "stderr_bytes": 0,
537
        "retry_count": 0,
538
        "human_action_count": 0,
539
    }
540
541
542
def phase_total_row(
543
    rows: list[dict[str, Any]],
544
    subject: Subject,
545
    scenario: str,
546
    run_index: int,
547
    metadata: dict[str, Any],
548
    operation: str,
549
) -> dict[str, Any]:
550
    failures = [row for row in rows if row_returncode(row) not in (0, SKIP_RETURNCODE)]
551
    unmeasured = [
552
        str(row["operation"])
553
        for row in rows
554
        if row_returncode(row) == SKIP_RETURNCODE
555
    ]
556
    phase_elapsed_ms: dict[str, float] = {}
557
    for row in rows:
558
        phase = str(row.get("operation", "unknown")).split(".", 1)[0]
559
        phase_elapsed_ms[phase] = phase_elapsed_ms.get(phase, 0.0) + float(
560
            row.get("elapsed_ms", 0.0)
561
        )
562
    return {
563
        **metadata,
564
        "subject": subject.name,
565
        "subject_kind": subject.kind,
566
        "subject_label": subject.label,
567
        "scenario": scenario,
568
        "run": run_index,
569
        "operation": operation,
570
        "step_kind": "summary",
571
        "elapsed_ms": round(sum(float(row.get("elapsed_ms", 0)) for row in rows), 3),
572
        "returncode": 1 if failures else 0,
573
        "coverage_complete": not unmeasured,
574
        "unmeasured_operations": unmeasured,
575
        "command": [row["command"] for row in rows],
576
        "tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("tool_call_count")) for row in rows),
577
        "terminal_tool_call_count": sum(
578
            oakbench_tokens.int_or_zero(row.get("terminal_tool_call_count")) for row in rows
579
        ),
580
        "vcs_tool_call_count": sum(oakbench_tokens.int_or_zero(row.get("vcs_tool_call_count")) for row in rows),
581
        "agent_tool_call_count": sum(
582
            oakbench_tokens.int_or_zero(row.get("tool_call_count")) for row in rows
583
        ),
584
        "harness_command_count": sum(
585
            oakbench_tokens.int_or_zero(row.get("harness_command_count")) for row in rows
586
        ),
587
        "retry_count": sum(oakbench_tokens.int_or_zero(row.get("retry_count")) for row in rows),
588
        "human_action_count": sum(
589
            oakbench_tokens.int_or_zero(row.get("human_action_count")) for row in rows
590
        ),
591
        "phase_elapsed_ms": {
592
            phase: round(elapsed, 3) for phase, elapsed in sorted(phase_elapsed_ms.items())
593
        },
594
        **oakbench_tokens.summed_token_fields(
595
            rows, "sum_of_task_loop_steps_command_plus_admitted_output_chars_div_4"
596
        ),
597
        "summarized_operations": [row["operation"] for row in rows],
598
    }
599
600
601
def has_lane_failures(rows: list[dict[str, Any]]) -> bool:
602
    return any(row_returncode(row) not in (0, SKIP_RETURNCODE) for row in rows)
603
604
605
def enforce_oracle_result(total: dict[str, Any], oracle_ok: bool | None) -> dict[str, Any]:
606
    if oracle_ok is False:
607
        total["returncode"] = 1
608
        total["stderr"] = "oracle_payload_missing_from_remote_branch"
609
    return total
610
611
612
def interrupted_publication_fields(
613
    *,
614
    request_count: int,
615
    request_bytes: int | None,
616
    declared_request_bytes: int | None,
617
    request_framing_valid: bool,
618
    request_body_timed_out: bool,
619
    local_head_before: str | None,
620
    local_head_after: str | None,
621
    status_after: dict[str, Any] | None,
622
    worktree_payload_before: str | None,
623
    worktree_payload_after: str | None,
624
    cli_observation: dict[str, str],
625
) -> dict[str, Any]:
626
    """Evidence fields for a single interrupted mutation attempt.
627
628
    The fixture's receipt of a full request is ground truth for the harness,
629
    not a server durability receipt available to the CLI.  A disconnect after
630
    that point is therefore unknown to the client and must never imply replay.
631
    """
632
    request_complete = bool(
633
        request_count == 1
634
        and request_framing_valid
635
        and declared_request_bytes is not None
636
        and request_bytes == declared_request_bytes
637
    )
638
    if request_count == 0:
639
        delivery_state = "no_request_observed"
640
        fixture_outcome = "not_sent_observed"
641
    elif not request_framing_valid:
642
        delivery_state = "invalid_request_framing"
643
        fixture_outcome = "indeterminate_after_invalid_request_disconnect"
644
    elif request_complete:
645
        delivery_state = "complete_request_observed"
646
        fixture_outcome = "unknown_after_complete_request_disconnect"
647
    else:
648
        delivery_state = "partial_request_observed"
649
        fixture_outcome = "indeterminate_after_partial_request_disconnect"
650
    payload_preserved = bool(
651
        worktree_payload_before
652
        and worktree_payload_before == worktree_payload_after
653
    )
654
    unpublished_state_preserved = bool(status_after and status_after.get("needs_push"))
655
    return {
656
        "publication_outcome": cli_observation["reported_outcome"],
657
        "cli_observation": cli_observation["observation"],
658
        "fixture_delivery_state": delivery_state,
659
        "fixture_outcome_knowledge": fixture_outcome,
660
        "fixture_received_complete_request_before_disconnect": request_complete,
661
        "fixture_request_framing_valid": request_framing_valid,
662
        "fixture_request_body_timed_out": request_body_timed_out,
663
        "fixture_declared_request_bytes": declared_request_bytes,
664
        "mutation_requests_observed": request_count,
665
        "blind_replay_requests": max(0, request_count - 1),
666
        "network_request_bytes_observed": request_bytes,
667
        "local_head_preserved": bool(local_head_before) and local_head_before == local_head_after,
668
        "worktree_payload_preserved": payload_preserved,
669
        "unpublished_state_preserved": unpublished_state_preserved,
670
        "local_work_preserved": payload_preserved and unpublished_state_preserved,
671
        "working_tree_dirty_after": None if status_after is None else status_after.get("dirty"),
672
    }
673
674
675
def cli_publication_observation(row: dict[str, Any], *, json_requested: bool) -> dict[str, str]:
676
    """Classify only what the CLI reported, never what the fault fixture knows."""
677
    if row.get("timed_out"):
678
        return {"reported_outcome": "unreported", "observation": "process_timeout"}
679
    stdout = str(row.get("stdout") or "")
680
    stderr = str(row.get("stderr") or "")
681
    value = _parse_json_object(stdout) if json_requested else None
682
    if value is not None and row.get("process_returncode") == 0 and value.get("published") is True:
683
        return {"reported_outcome": "published", "observation": "json_success_receipt"}
684
    if value is not None and isinstance(value.get("error"), dict):
685
        message = str(value["error"].get("message") or "").lower()
686
        if any(marker in message for marker in ("unconfirmed", "unknown", "may have")):
687
            return {"reported_outcome": "unknown", "observation": "json_error_unknown"}
688
        return {
689
            "reported_outcome": "unreported",
690
            "observation": "json_error_without_publication_outcome",
691
        }
692
    message = f"{stdout}\n{stderr}".lower()
693
    if any(marker in message for marker in ("unconfirmed", "unknown", "may have")):
694
        return {"reported_outcome": "unknown", "observation": "text_error_unknown"}
695
    return {"reported_outcome": "unreported", "observation": "text_error_without_publication_outcome"}
696
697
698
def file_sha256(path: Path) -> str | None:
699
    try:
700
        return hashlib.sha256(path.read_bytes()).hexdigest()
701
    except OSError:
702
        return None
703
704
705
class _DisconnectAfterPushHandler(BaseHTTPRequestHandler):
706
    """Minimal loopback fixture for a complete request with a lost reply."""
707
708
    server_version = "oakbench-interruption/1"
709
710
    def log_message(self, _format: str, *_args: object) -> None:
711
        return
712
713
    def _json(self, status: int, payload: dict[str, Any]) -> None:
714
        encoded = json.dumps(payload).encode()
715
        self.send_response(status)
716
        self.send_header("Content-Type", "application/json")
717
        self.send_header("Content-Length", str(len(encoded)))
718
        self.end_headers()
719
        self.wfile.write(encoded)
720
721
    def do_GET(self) -> None:  # noqa: N802 - BaseHTTPRequestHandler API
722
        self._json(404, {"error": "absent synthetic repo"})
723
724
    def do_POST(self) -> None:  # noqa: N802 - BaseHTTPRequestHandler API
725
        declared_size: int | None = None
726
        framing_valid = False
727
        body = b""
728
        body_timed_out = False
729
        raw_sizes = self.headers.get_all("Content-Length", failobj=[])
730
        try:
731
            raw_size = raw_sizes[0].strip(" \t") if len(raw_sizes) == 1 else ""
732
            parsed_size = int(raw_size) if raw_size.isascii() and raw_size.isdecimal() else -1
733
            framing_valid = (
734
                0 <= parsed_size <= INTERRUPTED_PUBLICATION_MAX_REQUEST_BYTES
735
                and self.headers.get("Transfer-Encoding") is None
736
            )
737
            if framing_valid:
738
                declared_size = parsed_size
739
                self.connection.settimeout(INTERRUPTED_PUBLICATION_REQUEST_TIMEOUT_S)
740
                while len(body) < parsed_size:
741
                    try:
742
                        chunk = self.rfile.read1(parsed_size - len(body))
743
                    except (socket.timeout, TimeoutError):
744
                        body_timed_out = True
745
                        break
746
                    except OSError:
747
                        break
748
                    if not chunk:
749
                        break
750
                    body += chunk
751
        except ValueError:
752
            framing_valid = False
753
        server = self.server
754
        if self.path == "/api/repos":
755
            self._json(200, {"name": "repo"})
756
            return
757
        if self.path == "/api/oak/repo/push":
758
            with server.receipt_lock:  # type: ignore[attr-defined]
759
                server.push_request_count += 1  # type: ignore[attr-defined]
760
                server.push_request_bytes += len(body)  # type: ignore[attr-defined]
761
                server.push_declared_request_bytes = declared_size  # type: ignore[attr-defined]
762
                server.push_request_framing_valid = framing_valid  # type: ignore[attr-defined]
763
                server.push_request_body_timed_out = body_timed_out  # type: ignore[attr-defined]
764
                server.request_complete_before_disconnect = (  # type: ignore[attr-defined]
765
                    framing_valid and len(body) == declared_size
766
                )
767
            self.close_connection = True
768
            try:
769
                self.connection.shutdown(socket.SHUT_RDWR)
770
            except OSError:
771
                pass
772
            self.connection.close()
773
            return
774
        self._json(404, {"error": "unexpected synthetic path"})
775
776
777
def start_interrupted_publication_server() -> tuple[ThreadingHTTPServer, threading.Thread]:
778
    server = ThreadingHTTPServer(("127.0.0.1", 0), _DisconnectAfterPushHandler)
779
    server.receipt_lock = threading.Lock()  # type: ignore[attr-defined]
780
    server.push_request_count = 0  # type: ignore[attr-defined]
781
    server.push_request_bytes = 0  # type: ignore[attr-defined]
782
    server.push_declared_request_bytes = None  # type: ignore[attr-defined]
783
    server.push_request_framing_valid = False  # type: ignore[attr-defined]
784
    server.push_request_body_timed_out = False  # type: ignore[attr-defined]
785
    server.request_complete_before_disconnect = False  # type: ignore[attr-defined]
786
    thread = threading.Thread(target=server.serve_forever, name="oakbench-interrupted-push", daemon=True)
787
    thread.start()
788
    return server, thread
789
790
791
def _parse_json_object(text: str | None) -> dict[str, Any] | None:
792
    try:
793
        value = json.loads(text or "")
794
    except (TypeError, json.JSONDecodeError):
795
        return None
796
    return value if isinstance(value, dict) else None
797
798
799
def run_interrupted_publication(
800
    subject: Subject,
801
    run_root: Path,
802
    run_index: int,
803
    metadata: dict[str, Any],
804
    admitted: int,
805
) -> list[dict[str, Any]]:
806
    """Run one real Oak push against a loopback server that drops its reply.
807
808
    The fixture observes request delivery but deliberately has no durable
809
    remote state. That makes the client's outcome unknown and lets the lane
810
    prove work preservation and absence of blind replay without production
811
    mutations or a fake success claim.
812
    """
813
    scenario = INTERRUPTED_PUBLICATION_SCENARIO
814
    metadata = {
815
        **metadata,
816
        "benchmark_track": "failure-evidence",
817
        "remote_transport": "loopback_fault_fixture",
818
        "remote_server": "127.0.0.1",
819
    }
820
    if subject.kind != "oak":
821
        return [
822
            skip_row(
823
                subject,
824
                scenario,
825
                run_index,
826
                metadata,
827
                "interrupted publication fixture currently exercises the actual Oak HTTP client only",
828
            )
829
        ]
830
831
    workspace = run_root / f"interrupted-{subject.name}-r{run_index}"
832
    workspace.mkdir(parents=True)
833
    rows: list[dict[str, Any]] = []
834
    vcs = str(subject.bin)
835
    rows.append(
836
        step_row(subject, scenario, "setup.init", [vcs, "init", "."], workspace, run_index, metadata, admitted)
837
    )
838
    if row_returncode(rows[-1]) != 0:
839
        return rows
840
    rows.append(
841
        edit_step(
842
            subject,
843
            scenario,
844
            "setup.edit",
845
            workspace,
846
            f"interrupted publication run={run_index}",
847
            run_index,
848
            metadata,
849
            admitted,
850
        )
851
    )
852
    rows.append(
853
        step_row(subject, scenario, "setup.checkpoint", [vcs, "commit"], workspace, run_index, metadata, admitted)
854
    )
855
    if row_returncode(rows[-1]) != 0:
856
        return rows
857
858
    before = run_untimed([vcs, "hash"], workspace)
859
    head_before = before.stdout.strip() if before.returncode == 0 else None
860
    payload_path = workspace / "task-notes.md"
861
    payload_before = file_sha256(payload_path)
862
    before_usage = bounded_tree_usage(workspace)
863
    push_help = run_untimed([vcs, "push", "--help"], workspace)
864
    push_supports_json = "--json" in (push_help.stdout + push_help.stderr)
865
    server, thread = start_interrupted_publication_server()
866
    host, port = server.server_address[:2]
867
    remote = f"http://{host}:{port}"
868
    fixture_env = base_env()
869
    fixture_env.pop("OAK_API_KEY", None)
870
    fixture_env.pop("OAK_REMOTE", None)
871
    fixture_env["OAK_NO_UPDATE_CHECK"] = "1"
872
    try:
873
        push_command = [vcs, "push", "--repo", "oak/repo", "--remote", remote]
874
        if push_supports_json:
875
            push_command.append("--json")
876
        publication_attempt = step_row(
877
            subject,
878
            scenario,
879
            "publication.attempt",
880
            push_command,
881
            workspace,
882
            run_index,
883
            metadata,
884
            admitted,
885
            expected_returncodes=tuple(range(1, 128)),
886
            preserve_stdout=True,
887
            preserve_stderr=True,
888
            env=fixture_env,
889
            timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
890
        )
891
        rows.append(publication_attempt)
892
    finally:
893
        server.shutdown()
894
        server.server_close()
895
        thread.join(timeout=2.0)
896
897
    state_row = step_row(
898
        subject,
899
        scenario,
900
        "reconcile.local_state",
901
        [vcs, "agent", "state", "--json", "--compact"],
902
        workspace,
903
        run_index,
904
        metadata,
905
        admitted,
906
        preserve_stdout=True,
907
        env=fixture_env,
908
    )
909
    rows.append(state_row)
910
    after = run_untimed([vcs, "hash"], workspace)
911
    head_after = after.stdout.strip() if after.returncode == 0 else None
912
    payload_after = file_sha256(payload_path)
913
    status_after = _parse_json_object(state_row.get("stdout"))
914
    with server.receipt_lock:  # type: ignore[attr-defined]
915
        request_count = int(server.push_request_count)  # type: ignore[attr-defined]
916
        request_bytes = int(server.push_request_bytes)  # type: ignore[attr-defined]
917
        declared_request_bytes = server.push_declared_request_bytes  # type: ignore[attr-defined]
918
        request_framing_valid = bool(server.push_request_framing_valid)  # type: ignore[attr-defined]
919
        request_body_timed_out = bool(server.push_request_body_timed_out)  # type: ignore[attr-defined]
920
    cli_observation = cli_publication_observation(
921
        publication_attempt,
922
        json_requested=push_supports_json,
923
    )
924
    evidence = interrupted_publication_fields(
925
        request_count=request_count,
926
        request_bytes=request_bytes if request_count else None,
927
        declared_request_bytes=declared_request_bytes,
928
        request_framing_valid=request_framing_valid,
929
        request_body_timed_out=request_body_timed_out,
930
        local_head_before=head_before,
931
        local_head_after=head_after,
932
        status_after=status_after,
933
        worktree_payload_before=payload_before,
934
        worktree_payload_after=payload_after,
935
        cli_observation=cli_observation,
936
    )
937
    evidence["workspace_usage_before"] = before_usage
938
    evidence["workspace_usage_after"] = bounded_tree_usage(workspace)
939
    evidence["remote_transport"] = "loopback_fault_fixture"
940
    evidence["remote_server"] = str(host)
941
    evidence["human_action_count"] = 0
942
    evidence["compatibility_probe_calls"] = 1
943
    evidence["harness_probe_calls"] = 3
944
    evidence["push_json_supported"] = push_supports_json
945
    total = phase_total_row(rows, subject, scenario, run_index, metadata, "interrupted_publication.total")
946
    evidence["harness_command_count"] = total["harness_command_count"] + 3
947
    total.update(evidence)
948
    if not (
949
        evidence["fixture_received_complete_request_before_disconnect"]
950
        and request_count == 1
951
        and evidence["local_head_preserved"]
952
        and evidence["local_work_preserved"]
953
        and not publication_attempt["timed_out"]
954
        and row_returncode(rows[-1]) == 0
955
    ):
956
        total["returncode"] = 1
957
        total["stderr"] = "interrupted_publication_evidence_incomplete"
958
    rows.append(total)
959
    return rows
960
961
962
def edit_step(
963
    subject: Subject,
964
    scenario: str,
965
    operation: str,
966
    workspace: Path,
967
    payload: str,
968
    run_index: int,
969
    metadata: dict[str, Any],
970
    admitted: int,
971
) -> dict[str, Any]:
972
    command = ["/bin/zsh", "-c", f"echo {payload!r} >> task-notes.md"]
973
    return step_row(
974
        subject, scenario, operation, command, workspace, run_index, metadata, admitted, kind="edit"
975
    )
976
977
978
def oak_virtual_branch(subject: Subject, mount_dir: Path) -> str | None:
979
    proc = run_untimed([str(subject.bin), "status"], mount_dir)
980
    match = VIRTUAL_BRANCH_RE.search(proc.stdout + proc.stderr)
981
    return match.group(1) if match else None
982
983
984
def run_oak_loop(
985
    subject: Subject,
986
    remote: oakbench_remotes.RemoteResolution,
987
    run_root: Path,
988
    run_index: int,
989
    metadata: dict[str, Any],
990
    admitted: int,
991
) -> list[dict[str, Any]]:
992
    scenario = "task_loop"
993
    meta = {**metadata, **remote.row_fields()}
994
    rows: list[dict[str, Any]] = []
995
    branches: dict[str, str | None] = {}
996
    payloads: dict[str, str] = {}
997
    mounts: list[Path] = []
998
    vcs = str(subject.bin)
999
    try:
1000
        for phase in PHASES:
1001
            workspace = run_root / f"tl-{subject.name}-r{run_index}-{phase}"
1002
            payload = f"payload {phase} run={run_index} bench={metadata['bench_id']}"
1003
            payloads[phase] = payload
1004
            acquire = step_row(
1005
                subject, scenario, f"{phase}.acquire", [vcs, "mount", remote.repo or "", str(workspace)],
1006
                run_root, run_index, meta, admitted,
1007
            )
1008
            rows.append(acquire)
1009
            if int(acquire["returncode"]) != 0:
1010
                return rows
1011
            mounts.append(workspace)
1012
            branches[phase] = oak_virtual_branch(subject, workspace)
1013
            rows.append(edit_step(subject, scenario, f"{phase}.edit", workspace, payload, run_index, meta, admitted))
1014
            rows.append(step_row(subject, scenario, f"{phase}.snapshot", [vcs, "commit", "--no-verify"], workspace, run_index, meta, admitted))
1015
            rows.append(step_row(subject, scenario, f"{phase}.publish", [vcs, "push"], workspace, run_index, meta, admitted))
1016
            if phase == "task":
1017
                rows.append(
1018
                    step_row(
1019
                        subject, scenario, "task.describe",
1020
                        [vcs, "desc", f"bench task loop {metadata['bench_id']} r{run_index}"],
1021
                        workspace, run_index, meta, admitted,
1022
                    )
1023
                )
1024
            end = step_row(
1025
                subject, scenario, f"{phase}.end", [vcs, "mount", "end", str(workspace)],
1026
                run_root, run_index, meta, admitted,
1027
            )
1028
            rows.append(end)
1029
            if int(end["returncode"]) == 0 and workspace in mounts:
1030
                mounts.remove(workspace)
1031
1032
        # Untimed oracle: each phase's payload must be on its pushed branch.
1033
        verifier = run_root / f"tl-verify-{subject.name}-r{run_index}"
1034
        clone = run_untimed([vcs, "clone", remote.repo or "", str(verifier)], run_root)
1035
        oracle_ok: bool | None = None
1036
        if clone.returncode == 0:
1037
            oracle_ok = True
1038
            for phase in PHASES:
1039
                branch = branches.get(phase)
1040
                if not branch:
1041
                    oracle_ok = None
1042
                    break
1043
                switch = run_untimed([vcs, "switch", branch], verifier)
1044
                content = (verifier / "task-notes.md")
1045
                seen = content.read_text() if content.exists() else ""
1046
                if switch.returncode != 0 or payloads[phase] not in seen:
1047
                    oracle_ok = False
1048
        total = {
1049
            **phase_total_row(rows, subject, scenario, run_index, meta, "task_loop.total"),
1050
            "oracle_payload_on_remote_branch": oracle_ok,
1051
            "workspace_branches": branches,
1052
            "measurement_source": "verifier_clone_switch_grep; null oracle means branch name unparsed",
1053
        }
1054
        rows.append(enforce_oracle_result(total, oracle_ok))
1055
        return rows
1056
    finally:
1057
        for mount_dir in mounts:
1058
            run_untimed([vcs, "mount", "end", str(mount_dir), "-f"], run_root)
1059
1060
1061
def ensure_git_remote_seeded(subject: Subject, remote: oakbench_remotes.RemoteResolution, scratch: Path) -> str | None:
1062
    """Untimed: make sure the git remote has a main with a seed commit."""
1063
    vcs = str(subject.bin)
1064
    probe = run_untimed([vcs, "ls-remote", str(remote.repo), "refs/heads/main"], None)
1065
    if probe.returncode == 0 and probe.stdout.strip():
1066
        if remote.transport == oakbench_remotes.TRANSPORT_LOCAL_FILE:
1067
            head = run_untimed([vcs, "symbolic-ref", "HEAD", "refs/heads/main"], Path(str(remote.repo)))
1068
            if head.returncode != 0:
1069
                return f"git remote HEAD setup failed: {head.stderr.strip()[:200]}"
1070
        return None
1071
    seed = scratch / "git-seed"
1072
    if seed.exists():
1073
        shutil.rmtree(seed)
1074
    seed.mkdir(parents=True)
1075
    (seed / "README.md").write_text("Task-loop benchmark repo (disposable).\n")
1076
    (seed / "task-notes.md").write_text("# task notes\n")
1077
    for command in (
1078
        [vcs, "init", "-q", "-b", "main"],
1079
        [vcs, "add", "."],
1080
        [vcs, "commit", "-q", "-m", "seed"],
1081
        [vcs, "push", "-q", str(remote.repo), "main:main"],
1082
    ):
1083
        proc = run_untimed(command, seed)
1084
        if proc.returncode != 0:
1085
            return f"git remote seed `{' '.join(command[1:3])}` failed: {proc.stderr.strip()[:200]}"
1086
    if remote.transport == oakbench_remotes.TRANSPORT_LOCAL_FILE:
1087
        head = run_untimed([vcs, "symbolic-ref", "HEAD", "refs/heads/main"], Path(str(remote.repo)))
1088
        if head.returncode != 0:
1089
            return f"git remote HEAD setup failed: {head.stderr.strip()[:200]}"
1090
    return None
1091
1092
1093
def run_git_agent_task_baseline(
1094
    subject: Subject,
1095
    remote: oakbench_remotes.RemoteResolution,
1096
    run_root: Path,
1097
    run_index: int,
1098
    metadata: dict[str, Any],
1099
    admitted: int,
1100
    remote_create_ms: float = 0.0,
1101
) -> list[dict[str, Any]]:
1102
    """Measure a cold clone and a warm isolated worktree on one local remote.
1103
1104
    This is the core-equivalent, local-file cell. Hosted CI is deliberately a
1105
    skip row: a local commit oracle is not relabelled as provider CI.
1106
    """
1107
    scenario = AGENT_TASK_SCENARIO
1108
    meta = {
1109
        **metadata,
1110
        **remote.row_fields(),
1111
        "benchmark_track": (
1112
            "core-equivalent"
1113
            if remote.transport == oakbench_remotes.TRANSPORT_LOCAL_FILE
1114
            else "agent-default"
1115
        ),
1116
    }
1117
    vcs = str(subject.bin)
1118
    rows: list[dict[str, Any]] = [
1119
        harness_step_row(
1120
            subject,
1121
            scenario,
1122
            "setup.remote_container",
1123
            run_index,
1124
            meta,
1125
            remote_create_ms,
1126
            harness_command_count=1,
1127
        )
1128
    ]
1129
    setup_start = time.perf_counter()
1130
    seed_error = ensure_git_remote_seeded(subject, remote, run_root)
1131
    rows.append(
1132
        harness_step_row(
1133
            subject,
1134
            scenario,
1135
            "setup.remote_seed",
1136
            run_index,
1137
            meta,
1138
            (time.perf_counter() - setup_start) * 1000,
1139
            returncode=1 if seed_error else 0,
1140
            harness_command_count=6,
1141
        )
1142
    )
1143
    if seed_error:
1144
        rows[-1]["stderr"] = seed_error
1145
        return rows
1146
1147
    base = run_root / f"agent-task-{subject.name}-r{run_index}-cold"
1148
    cold_branch = oakbench_remotes.disposable_branch(
1149
        str(metadata["bench_id"]), "agent-task-cold", subject.name, run_index
1150
    )
1151
    warm_branch = oakbench_remotes.disposable_branch(
1152
        str(metadata["bench_id"]), "agent-task-warm", subject.name, run_index
1153
    )
1154
    cold_payload = f"cold task run={run_index} bench={metadata['bench_id']}"
1155
    warm_payload = f"warm task run={run_index} bench={metadata['bench_id']}"
1156
1157
    rows.append(
1158
        step_row(
1159
            subject,
1160
            scenario,
1161
            "cold.acquire",
1162
            [vcs, "clone", "-q", str(remote.repo), str(base)],
1163
            run_root,
1164
            run_index,
1165
            meta,
1166
            admitted,
1167
        )
1168
    )
1169
    if row_returncode(rows[-1]) != 0:
1170
        return rows
1171
    rows.append(
1172
        step_row(
1173
            subject, scenario, "cold.branch", [vcs, "switch", "-q", "-c", cold_branch],
1174
            base, run_index, meta, admitted,
1175
        )
1176
    )
1177
    rows.append(
1178
        step_row(
1179
            subject, scenario, "cold.inspect", [vcs, "status", "--porcelain=v1"],
1180
            base, run_index, meta, admitted,
1181
        )
1182
    )
1183
    rows.append(edit_step(subject, scenario, "cold.edit", base, cold_payload, run_index, meta, admitted))
1184
    rows.append(step_row(subject, scenario, "cold.checkpoint.add", [vcs, "add", "."], base, run_index, meta, admitted))
1185
    rows.append(
1186
        step_row(
1187
            subject, scenario, "cold.checkpoint.commit",
1188
            [vcs, "commit", "-q", "-m", f"agent task cold {metadata['bench_id']} r{run_index}"],
1189
            base, run_index, meta, admitted,
1190
        )
1191
    )
1192
    rows.append(
1193
        step_row(
1194
            subject, scenario, "cold.publish", [vcs, "push", "-q", "-u", "origin", cold_branch],
1195
            base, run_index, meta, admitted,
1196
        )
1197
    )
1198
    rows.append(
1199
        unmeasured_step_row(
1200
            subject, scenario, "cold.ci.observe", run_index, meta,
1201
            "local-file transport has no hosted CI provider; exact CI observation is unmeasured",
1202
        )
1203
    )
1204
    rows.append(
1205
        step_row(
1206
            subject, scenario, "cold.review_handoff",
1207
            [vcs, "show", "--format=%H%n%s", "--stat", "--oneline", "HEAD"],
1208
            base, run_index, meta, admitted,
1209
        )
1210
    )
1211
1212
    warm = run_root / f"agent-task-{subject.name}-r{run_index}-warm"
1213
    rows.append(
1214
        step_row(
1215
            subject, scenario, "warm.acquire",
1216
            [vcs, "worktree", "add", "-q", str(warm), "-b", warm_branch, "origin/main"],
1217
            base, run_index, meta, admitted,
1218
        )
1219
    )
1220
    if row_returncode(rows[-1]) == 0:
1221
        rows.append(
1222
            step_row(
1223
                subject, scenario, "warm.inspect", [vcs, "status", "--porcelain=v1"],
1224
                warm, run_index, meta, admitted,
1225
            )
1226
        )
1227
        rows.append(edit_step(subject, scenario, "warm.edit", warm, warm_payload, run_index, meta, admitted))
1228
        rows.append(step_row(subject, scenario, "warm.checkpoint.add", [vcs, "add", "."], warm, run_index, meta, admitted))
1229
        rows.append(
1230
            step_row(
1231
                subject, scenario, "warm.checkpoint.commit",
1232
                [vcs, "commit", "-q", "-m", f"agent task warm {metadata['bench_id']} r{run_index}"],
1233
                warm, run_index, meta, admitted,
1234
            )
1235
        )
1236
        rows.append(
1237
            step_row(
1238
                subject, scenario, "warm.publish", [vcs, "push", "-q", "-u", "origin", warm_branch],
1239
                warm, run_index, meta, admitted,
1240
            )
1241
        )
1242
        rows.append(
1243
            unmeasured_step_row(
1244
                subject, scenario, "warm.ci.observe", run_index, meta,
1245
                "local-file transport has no hosted CI provider; exact CI observation is unmeasured",
1246
            )
1247
        )
1248
        rows.append(
1249
            step_row(
1250
                subject, scenario, "warm.review_handoff",
1251
                [vcs, "show", "--format=%H%n%s", "--stat", "--oneline", "HEAD"],
1252
                warm, run_index, meta, admitted,
1253
            )
1254
        )
1255
1256
    usage_before_cleanup = {
1257
        "cold": bounded_tree_usage(base),
1258
        "warm": bounded_tree_usage(warm),
1259
    }
1260
    oracle_ok = True
1261
    for branch, payload in ((cold_branch, cold_payload), (warm_branch, warm_payload)):
1262
        show = run_untimed([vcs, "--git-dir", str(remote.repo), "show", f"{branch}:task-notes.md"], run_root)
1263
        if show.returncode != 0 or payload not in show.stdout:
1264
            oracle_ok = False
1265
1266
    if warm.exists():
1267
        rows.append(
1268
            step_row(
1269
                subject, scenario, "cleanup.warm",
1270
                [vcs, "worktree", "remove", str(warm)], base, run_index, meta, admitted,
1271
            )
1272
        )
1273
    cleanup_start = time.perf_counter()
1274
    shutil.rmtree(base, ignore_errors=True)
1275
    rows.append(
1276
        harness_step_row(
1277
            subject, scenario, "cleanup.cold", run_index, meta,
1278
            (time.perf_counter() - cleanup_start) * 1000,
1279
        )
1280
    )
1281
1282
    total = phase_total_row(rows, subject, scenario, run_index, meta, "agent_task.total")
1283
    total.update(
1284
        {
1285
            "oracle_payload_on_remote_branch": oracle_ok,
1286
            "acquisition_models": {
1287
                "cold": "fresh_full_clone",
1288
                "warm": "shared_object_worktree",
1289
            },
1290
            "repository_cache_state": {
1291
                "cold": "fresh_destination_no_shared_object_store",
1292
                "warm": "explicit_shared_object_store",
1293
            },
1294
            "os_cache_state": "unknown",
1295
            "os_cache_state_reason": "page_cache_not_purged_or_instrumented",
1296
            "workspace_usage_before_cleanup": usage_before_cleanup,
1297
            "network_bytes": None,
1298
            "network_bytes_reason": "local_file_transport_not_observed_by_byte_proxy",
1299
            "partial_clone_comparator": None,
1300
            "partial_clone_reason": "local file transport does not establish a network partial-clone comparison",
1301
            "human_action_count": 0,
1302
            "parallel_evidence_lane": "parallel_contention.py/workspace_per_task",
1303
            "oracle_probe_calls": 2,
1304
            "harness_command_count": total["harness_command_count"] + 2,
1305
        }
1306
    )
1307
    if not oracle_ok:
1308
        total["returncode"] = 1
1309
        total["stderr"] = "agent_task_remote_payload_oracle_failed"
1310
    rows.append(total)
1311
    return rows
1312
1313
1314
def reserve_loopback_port() -> int:
1315
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
1316
        listener.bind(("127.0.0.1", 0))
1317
        return int(listener.getsockname()[1])
1318
1319
1320
def wait_for_loopback_port(
1321
    process: subprocess.Popen[Any], port: int, timeout_s: float = LOCAL_OAK_SERVER_READY_TIMEOUT_S
1322
) -> bool:
1323
    deadline = time.monotonic() + timeout_s
1324
    while time.monotonic() < deadline:
1325
        if process.poll() is not None:
1326
            return False
1327
        try:
1328
            with socket.create_connection(("127.0.0.1", port), timeout=0.2):
1329
                return True
1330
        except OSError:
1331
            time.sleep(0.05)
1332
    return False
1333
1334
1335
def harness_command_row(
1336
    subject: Subject,
1337
    scenario: str,
1338
    operation: str,
1339
    command: list[str],
1340
    cwd: Path,
1341
    run_index: int,
1342
    metadata: dict[str, Any],
1343
) -> tuple[dict[str, Any], subprocess.CompletedProcess[str]]:
1344
    started = time.perf_counter()
1345
    result = run_bounded_harness_command(command, cwd)
1346
    row = harness_step_row(
1347
        subject,
1348
        scenario,
1349
        operation,
1350
        run_index,
1351
        metadata,
1352
        (time.perf_counter() - started) * 1000,
1353
        returncode=0 if result.returncode == 0 else result.returncode,
1354
        harness_command_count=1,
1355
    )
1356
    if result.returncode != 0:
1357
        row["stderr"] = str(result.stderr)[-4000:]
1358
    return row, result
1359
1360
1361
def run_bounded_harness_command(
1362
    command: list[str], cwd: Path, *, env: dict[str, str] | None = None
1363
) -> subprocess.CompletedProcess[str]:
1364
    capture = run_timed(
1365
        command,
1366
        cwd,
1367
        env or base_env(),
1368
        admitted_output_chars=4_000,
1369
        timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
1370
    )
1371
    return subprocess.CompletedProcess(
1372
        command,
1373
        capture.returncode,
1374
        capture.stdout_text,
1375
        capture.stderr_text,
1376
    )
1377
1378
1379
def remote_branch_head(remote: str, repo_spec: str, branch: str) -> str | None:
1380
    owner, repo = repo_spec.split("/", 1)
1381
    branch_segment = urllib.parse.quote(branch, safe="")
1382
    url = f"{remote}/api/{owner}/{repo}/branches/{branch_segment}"
1383
    try:
1384
        with urllib.request.urlopen(url, timeout=2.0) as response:
1385
            value = json.loads(response.read())
1386
    except (OSError, ValueError, urllib.error.URLError):
1387
        return None
1388
    head = value.get("head") if isinstance(value, dict) else None
1389
    return head if isinstance(head, str) else None
1390
1391
1392
def run_oak_local_agent_task_baseline(
1393
    subject: Subject,
1394
    run_root: Path,
1395
    run_index: int,
1396
    metadata: dict[str, Any],
1397
    admitted: int,
1398
) -> list[dict[str, Any]]:
1399
    """Exercise two fresh Oak checkouts against an owned loopback Oak server."""
1400
    run_root = run_root.resolve()
1401
    scenario = AGENT_TASK_OAK_LOCAL_SCENARIO
1402
    meta = {
1403
        **metadata,
1404
        "benchmark_track": "oak-local-loopback",
1405
        "remote_transport": "loopback_http",
1406
        "remote_server": "127.0.0.1",
1407
    }
1408
    rows: list[dict[str, Any]] = []
1409
    vcs = str(subject.bin)
1410
    server_root = run_root / f"oak-local-server-{subject.name}-r{run_index}"
1411
    seed = run_root / f"oak-local-seed-{subject.name}-r{run_index}"
1412
    cold = run_root / f"oak-local-cold-{subject.name}-r{run_index}"
1413
    followup = run_root / f"oak-local-followup-{subject.name}-r{run_index}"
1414
    verifier = run_root / f"oak-local-verifier-{subject.name}-r{run_index}"
1415
    repo_spec = f"qa/task-loop-{subject.name}-r{run_index}"
1416
    port = reserve_loopback_port()
1417
    remote = f"http://127.0.0.1:{port}"
1418
    server_root.mkdir(parents=True)
1419
    server_started = time.perf_counter()
1420
    server: subprocess.Popen[Any] | None = None
1421
    server_start_error: str | None = None
1422
    try:
1423
        server = subprocess.Popen(
1424
            [vcs, "serve", "--dir", str(server_root), "--host", "127.0.0.1", "--port", str(port)],
1425
            cwd=run_root,
1426
            env=base_env(),
1427
            stdout=subprocess.DEVNULL,
1428
            stderr=subprocess.DEVNULL,
1429
            start_new_session=os.name == "posix",
1430
        )
1431
    except OSError as error:
1432
        server_start_error = type(error).__name__
1433
    ready = server is not None and wait_for_loopback_port(server, port)
1434
    rows.append(
1435
        harness_step_row(
1436
            subject,
1437
            scenario,
1438
            "setup.server",
1439
            run_index,
1440
            meta,
1441
            (time.perf_counter() - server_started) * 1000,
1442
            returncode=0 if ready else 1,
1443
            harness_command_count=1,
1444
        )
1445
    )
1446
    branch: str | None = None
1447
    expected_head: str | None = None
1448
    expected_payload = "# task notes\n"
1449
    failure: str | None = None if ready else (
1450
        f"local_oak_server_start_failed:{server_start_error}"
1451
        if server_start_error else "local_oak_server_not_ready"
1452
    )
1453
    try:
1454
        if failure is None:
1455
            seed.mkdir(parents=True)
1456
            (seed / "task-notes.md").write_text(expected_payload)
1457
            for operation, command in (
1458
                ("setup.seed_init", [vcs, "init", "."]),
1459
                ("setup.seed_commit", [vcs, "commit"]),
1460
                ("setup.seed_push", [vcs, "push", "--repo", repo_spec, "--remote", remote]),
1461
            ):
1462
                row, result = harness_command_row(
1463
                    subject, scenario, operation, command, seed, run_index, meta
1464
                )
1465
                rows.append(row)
1466
                if result.returncode != 0:
1467
                    failure = f"{operation}_failed"
1468
                    break
1469
1470
        def agent_step(operation: str, command: list[str], cwd: Path, *, preserve: bool = False) -> dict[str, Any]:
1471
            row = step_row(
1472
                subject,
1473
                scenario,
1474
                operation,
1475
                command,
1476
                cwd,
1477
                run_index,
1478
                meta,
1479
                admitted,
1480
                preserve_stdout=preserve,
1481
                timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
1482
            )
1483
            rows.append(row)
1484
            return row
1485
1486
        if failure is None:
1487
            row = agent_step(
1488
                "cold.acquire",
1489
                [vcs, "clone", repo_spec, str(cold), "--remote", remote, "--allow-legacy-scope"],
1490
                run_root,
1491
            )
1492
            if row_returncode(row) != 0:
1493
                failure = "cold.acquire_failed"
1494
        if failure is None:
1495
            status = agent_step("cold.inspect", [vcs, "status", "--json"], cold, preserve=True)
1496
            status_value = _parse_json_object(status.get("stdout"))
1497
            branch_value = status_value.get("branch") if status_value else None
1498
            branch = branch_value if isinstance(branch_value, str) else None
1499
            if not branch:
1500
                failure = "cold.branch_identity_missing"
1501
        if failure is None:
1502
            cold_payload = f"cold task run={run_index} bench={metadata['bench_id']}\n"
1503
            expected_payload += cold_payload
1504
            rows.append(edit_step(subject, scenario, "cold.edit", cold, cold_payload.strip(), run_index, meta, admitted))
1505
            if row_returncode(agent_step("cold.checkpoint", [vcs, "commit"], cold)) != 0:
1506
                failure = "cold.checkpoint_failed"
1507
            elif row_returncode(
1508
                agent_step("cold.publish", [vcs, "push", "--repo", repo_spec, "--remote", remote], cold)
1509
            ) != 0:
1510
                failure = "cold.publish_failed"
1511
            rows.append(
1512
                unmeasured_step_row(
1513
                    subject, scenario, "cold.ci.observe", run_index, meta,
1514
                    "owned local oak serve has no configured CI provider",
1515
                )
1516
            )
1517
            rows.append(
1518
                unmeasured_step_row(
1519
                    subject, scenario, "cold.review_handoff", run_index, meta,
1520
                    "owned local oak serve does not provide hosted review semantics",
1521
                )
1522
            )
1523
        if failure is None and branch is not None:
1524
            row = agent_step(
1525
                "followup.acquire",
1526
                [
1527
                    vcs, "clone", repo_spec, str(followup), "--remote", remote,
1528
                    "--branch", branch, "--allow-legacy-scope",
1529
                ],
1530
                run_root,
1531
            )
1532
            if row_returncode(row) != 0:
1533
                failure = "followup.acquire_failed"
1534
        if failure is None:
1535
            agent_step("followup.inspect", [vcs, "status", "--json"], followup)
1536
            followup_payload = f"followup task run={run_index} bench={metadata['bench_id']}\n"
1537
            expected_payload += followup_payload
1538
            rows.append(
1539
                edit_step(
1540
                    subject, scenario, "followup.edit", followup,
1541
                    followup_payload.strip(), run_index, meta, admitted,
1542
                )
1543
            )
1544
            if row_returncode(agent_step("followup.checkpoint", [vcs, "commit"], followup)) != 0:
1545
                failure = "followup.checkpoint_failed"
1546
            else:
1547
                head_result = run_bounded_harness_command([vcs, "hash"], followup)
1548
                expected_head = head_result.stdout.strip() if head_result.returncode == 0 else None
1549
                if row_returncode(
1550
                    agent_step(
1551
                        "followup.publish",
1552
                        [vcs, "push", "--repo", repo_spec, "--remote", remote],
1553
                        followup,
1554
                    )
1555
                ) != 0:
1556
                    failure = "followup.publish_failed"
1557
            rows.append(
1558
                unmeasured_step_row(
1559
                    subject, scenario, "followup.ci.observe", run_index, meta,
1560
                    "owned local oak serve has no configured CI provider",
1561
                )
1562
            )
1563
            rows.append(
1564
                unmeasured_step_row(
1565
                    subject, scenario, "followup.review_handoff", run_index, meta,
1566
                    "owned local oak serve does not provide hosted review semantics",
1567
                )
1568
            )
1569
1570
        payload_ok = False
1571
        verifier_head: str | None = None
1572
        advertised_head: str | None = None
1573
        if failure is None and branch is not None:
1574
            clone = run_bounded_harness_command(
1575
                [
1576
                    vcs, "clone", repo_spec, str(verifier), "--remote", remote,
1577
                    "--branch", branch, "--allow-legacy-scope",
1578
                ],
1579
                run_root,
1580
            )
1581
            if clone.returncode == 0:
1582
                payload_path = verifier / "task-notes.md"
1583
                payload_ok = payload_path.exists() and payload_path.read_text() == expected_payload
1584
                verifier_result = run_bounded_harness_command([vcs, "hash"], verifier)
1585
                verifier_head = verifier_result.stdout.strip() if verifier_result.returncode == 0 else None
1586
                advertised_head = remote_branch_head(remote, repo_spec, branch)
1587
            else:
1588
                failure = "verifier_clone_failed"
1589
        head_ok = bool(expected_head and expected_head == verifier_head == advertised_head)
1590
        if not payload_ok:
1591
            failure = failure or "local_oak_payload_oracle_failed"
1592
        if not head_ok:
1593
            failure = failure or "local_oak_head_oracle_failed"
1594
    finally:
1595
        if server is not None:
1596
            stop_owned_process(server)
1597
        server_stopped = server is None or server.poll() is not None
1598
        cleanup_started = time.perf_counter()
1599
        for path in (seed, cold, followup, verifier, server_root):
1600
            shutil.rmtree(path, ignore_errors=True)
1601
        rows.append(
1602
            harness_step_row(
1603
                subject,
1604
                scenario,
1605
                "cleanup.local_server_and_checkouts",
1606
                run_index,
1607
                meta,
1608
                (time.perf_counter() - cleanup_started) * 1000,
1609
            )
1610
        )
1611
1612
    total = phase_total_row(rows, subject, scenario, run_index, meta, "agent_task_oak_local.total")
1613
    total.update(
1614
        {
1615
            "oracle_payload_on_remote_branch": payload_ok and head_ok,
1616
            "oracle_payload_matches": payload_ok,
1617
            "oracle_branch_head_matches": head_ok,
1618
            "server_process_stopped": server_stopped,
1619
            "acquisition_models": {
1620
                "cold": "fresh_full_clone",
1621
                "followup": "second_fresh_full_clone",
1622
            },
1623
            "repository_cache_state": {
1624
                "cold": "fresh_destination_no_shared_object_store",
1625
                "followup": "no_shared_object_store",
1626
            },
1627
            "os_cache_state": "unknown",
1628
            "os_cache_state_reason": "page_cache_not_purged_or_instrumented",
1629
            "network_bytes": None,
1630
            "network_bytes_reason": "loopback transport not observed by byte proxy",
1631
            "hosted_ci_configured": False,
1632
            "oracle_probe_calls": 4,
1633
            # The expected-head hash plus verifier clone/hash/HTTP head probe
1634
            # are oracle/setup commands, not agent-facing tool calls.
1635
            "harness_command_count": total["harness_command_count"] + 4,
1636
        }
1637
    )
1638
    if failure or not server_stopped:
1639
        total["returncode"] = 1
1640
        total["stderr"] = failure or "local_oak_server_cleanup_failed"
1641
    rows.append(total)
1642
    return rows
1643
1644
1645
def run_legacy_selected_materialization_campaign(
1646
    subjects: list[Subject],
1647
    server_bin: Path | None,
1648
    server_source: Path | None,
1649
    server_source_head: str | None,
1650
    baseline_subject_name: str | None,
1651
    run_root: Path,
1652
    runs: int,
1653
    metadata: dict[str, Any],
1654
    admitted: int,
1655
    randomize_order: bool,
1656
    random_seed: int,
1657
    timing_controls_requested: bool,
1658
) -> list[dict[str, Any]]:
1659
    """Measure one legacy selected-branch clone against one fixed Serve fixture."""
1660
    scenario = LEGACY_SELECTED_MATERIALIZATION_SCENARIO
1661
    oak_subjects = [subject for subject in subjects if subject.kind == "oak"]
1662
    if not oak_subjects:
1663
        return []
1664
    if (
1665
        server_bin is None
1666
        or not server_bin.is_absolute()
1667
        or not server_bin.is_file()
1668
        or not os.access(server_bin, os.X_OK)
1669
    ):
1670
        return [
1671
            unmeasured_step_row(
1672
                subject,
1673
                scenario,
1674
                "scenario.skip",
1675
                0,
1676
                metadata,
1677
                "--oak-serve-bin must name an absolute executable used only as the fixed Serve fixture",
1678
            )
1679
            for subject in oak_subjects
1680
        ]
1681
    if server_source_head is None or re.fullmatch(r"[0-9a-f]{64}", server_source_head) is None:
1682
        return [
1683
            unmeasured_step_row(
1684
                subject,
1685
                scenario,
1686
                "scenario.skip",
1687
                0,
1688
                metadata,
1689
                "the fixed Serve binary's exact 64-hex --oak-serve-source-head is required",
1690
            )
1691
            for subject in oak_subjects
1692
        ]
1693
    baseline_subject = next(
1694
        (subject for subject in oak_subjects if subject.name == baseline_subject_name), None
1695
    )
1696
    if baseline_subject is None:
1697
        return [
1698
            unmeasured_step_row(
1699
                subject,
1700
                scenario,
1701
                "scenario.skip",
1702
                0,
1703
                metadata,
1704
                "--materialization-baseline-subject must name one enabled Oak subject",
1705
            )
1706
            for subject in oak_subjects
1707
        ]
1708
    git_bin = shutil.which("git")
1709
    if git_bin is None:
1710
        return [
1711
            unmeasured_step_row(
1712
                subject,
1713
                scenario,
1714
                "scenario.skip",
1715
                0,
1716
                metadata,
1717
                "git is required only to create the deterministic main-history fixture",
1718
            )
1719
            for subject in oak_subjects
1720
        ]
1721
1722
    server_bin = server_bin.resolve()
1723
    resolved_server_source: Path | None = None
1724
    if server_source is not None:
1725
        if not server_source.is_absolute() or not server_source.is_dir():
1726
            return [
1727
                unmeasured_step_row(
1728
                    subject,
1729
                    scenario,
1730
                    "scenario.skip",
1731
                    0,
1732
                    metadata,
1733
                    "--oak-serve-source must name an absolute read-only source checkout",
1734
                )
1735
                for subject in oak_subjects
1736
            ]
1737
        resolved_server_source = server_source.resolve()
1738
        source_head_result = run_bounded_harness_command(
1739
            [str(server_bin), "hash"],
1740
            resolved_server_source,
1741
            env=isolated_oak_fixture_env(run_root / "source-check"),
1742
        )
1743
        if source_head_result.returncode != 0 or source_head_result.stdout.strip() != server_source_head:
1744
            return [
1745
                unmeasured_step_row(
1746
                    subject,
1747
                    scenario,
1748
                    "scenario.skip",
1749
                    0,
1750
                    metadata,
1751
                    "fixed Serve source checkout does not match --oak-serve-source-head",
1752
                )
1753
                for subject in oak_subjects
1754
            ]
1755
    baseline_binary_sha256 = binary_sha256(baseline_subject.bin)
1756
    timing_controls_met = materialization_timing_controls_met(
1757
        metadata, runs, randomize_order, timing_controls_requested
1758
    )
1759
    fixture_root = run_root / "legacy-selected-materialization-fixture"
1760
    server_root = fixture_root / "server"
1761
    git_seed = fixture_root / "git-seed.git"
1762
    seed = fixture_root / "seed"
1763
    task_config_root = fixture_root / "config"
1764
    for path in (server_root, git_seed, task_config_root):
1765
        path.mkdir(parents=True, exist_ok=True)
1766
    fixture_env = isolated_oak_fixture_env(task_config_root)
1767
    fixture_meta = {
1768
        **metadata,
1769
        "benchmark_track": "oak-local-loopback-materialization",
1770
        "remote_transport": "loopback_http",
1771
        "remote_server": "127.0.0.1",
1772
        "serve_binary": str(server_bin),
1773
        "serve_binary_sha256": binary_sha256(server_bin),
1774
        "serve_source": str(resolved_server_source) if resolved_server_source else None,
1775
        "serve_source_checkout_verification": (
1776
            "exact_head_verified" if resolved_server_source else "unavailable_binary_hash_and_head_supplied"
1777
        ),
1778
        "serve_source_head": server_source_head,
1779
        "serve_version": subject_version(Subject("serve_fixture", "oak", "Serve fixture", server_bin)),
1780
        "baseline_subject": baseline_subject.name,
1781
        "baseline_binary_sha256": baseline_binary_sha256,
1782
        "fixture_id": "legacy-selected-materialization-192x16k-v1",
1783
        "fixture_regular_files": 192,
1784
        "fixture_regular_file_bytes": 16 * 1024,
1785
        "fixture_empty_files": 1,
1786
        "fixture_executable_files": 8,
1787
        "fixture_selected_only_files": 1,
1788
        "os_cache_state": "warm_or_unknown_after_required_fixture_probe",
1789
        "materialization_timing_controls_met": timing_controls_met,
1790
        "materialization_timing_controls_reason": (
1791
            "harness_controls_met_aa_and_quiet_host_evidence_still_required"
1792
            if timing_controls_met
1793
            else "correctness_only_or_harness_controls_not_met"
1794
        ),
1795
    }
1796
    rows: list[dict[str, Any]] = []
1797
    setup_subject = baseline_subject
1798
1799
    def setup_command(operation: str, command: list[str], cwd: Path) -> bool:
1800
        started = time.perf_counter()
1801
        result = run_bounded_harness_command(command, cwd, env=fixture_env)
1802
        rows.append(
1803
            harness_step_row(
1804
                setup_subject,
1805
                scenario,
1806
                operation,
1807
                0,
1808
                fixture_meta,
1809
                (time.perf_counter() - started) * 1000,
1810
                returncode=0 if result.returncode == 0 else result.returncode,
1811
                harness_command_count=1,
1812
            )
1813
        )
1814
        if result.returncode != 0:
1815
            rows[-1]["stderr"] = result.stderr[-4000:]
1816
        return result.returncode == 0
1817
1818
    server: subprocess.Popen[Any] | None = None
1819
    proxy: CountingProxy | None = None
1820
    failure: str | None = None
1821
    expected_artifacts: list[str] | None = None
1822
    baseline_cache: dict[str, Any] | None = None
1823
    baseline_receipt_contract: dict[str, Any] | None = None
1824
    baseline_network_contract: dict[str, Any] | None = None
1825
    expected_status_contract: dict[str, Any] | None = None
1826
    selected_branch = "selected-materialization"
1827
    repo_spec = "qa/materialization-bench"
1828
    try:
1829
        for index in range(192):
1830
            directory = git_seed / f"group-{index % 12:02d}"
1831
            directory.mkdir(parents=True, exist_ok=True)
1832
            digest = hashlib.sha256(f"oak-materialization-{index}".encode()).digest()
1833
            content = (digest * ((16 * 1024 + len(digest) - 1) // len(digest)))[: 16 * 1024]
1834
            path = directory / f"file-{index:03d}.bin"
1835
            path.write_bytes(content)
1836
            if index < 8:
1837
                path.chmod(0o755)
1838
        (git_seed / "empty.txt").write_bytes(b"")
1839
        for operation, command in (
1840
            ("setup.git_init", [git_bin, "init", "-q", "-b", "main"]),
1841
            ("setup.git_add", [git_bin, "add", "."]),
1842
            ("setup.git_commit", [git_bin, "commit", "-q", "-m", "fixture main"]),
1843
        ):
1844
            if not setup_command(operation, command, git_seed):
1845
                failure = f"{operation}_failed"
1846
                break
1847
1848
        port = reserve_loopback_port()
1849
        direct_remote = f"http://127.0.0.1:{port}"
1850
        if failure is None:
1851
            server = subprocess.Popen(
1852
                [
1853
                    str(server_bin),
1854
                    "serve",
1855
                    "--dir",
1856
                    str(server_root),
1857
                    "--host",
1858
                    "127.0.0.1",
1859
                    "--port",
1860
                    str(port),
1861
                ],
1862
                cwd=fixture_root,
1863
                env=fixture_env,
1864
                stdout=subprocess.DEVNULL,
1865
                stderr=subprocess.DEVNULL,
1866
                start_new_session=os.name == "posix",
1867
            )
1868
            if not wait_for_loopback_port(server, port):
1869
                failure = "setup.server_not_ready"
1870
        if failure is None:
1871
            for operation, command, cwd in (
1872
                ("setup.oak_import", [str(server_bin), "clone", str(git_seed), str(seed)], fixture_root),
1873
                ("setup.branch", [str(server_bin), "switch", "-c", selected_branch], seed),
1874
            ):
1875
                if not setup_command(operation, command, cwd):
1876
                    failure = f"{operation}_failed"
1877
                    break
1878
        if failure is None:
1879
            (seed / "selected.txt").write_text("selected branch\n")
1880
            for operation, command in (
1881
                ("setup.oak_commit", [str(server_bin), "commit"]),
1882
                (
1883
                    "setup.oak_push",
1884
                    [str(server_bin), "push", "--repo", repo_spec, "--remote", direct_remote],
1885
                ),
1886
            ):
1887
                if not setup_command(operation, command, seed):
1888
                    failure = f"{operation}_failed"
1889
                    break
1890
1891
        expected_head_result = (
1892
            run_bounded_harness_command([str(server_bin), "hash"], seed, env=fixture_env)
1893
            if failure is None
1894
            else None
1895
        )
1896
        expected_head = (
1897
            expected_head_result.stdout.strip()
1898
            if expected_head_result is not None and expected_head_result.returncode == 0
1899
            else None
1900
        )
1901
        expected_history = (
1902
            normalized_history(json_command([str(server_bin), "log", "--json"], seed, fixture_env))
1903
            if failure is None
1904
            else None
1905
        )
1906
        expected_worktree = normalized_worktree_snapshot(seed) if failure is None else None
1907
        if not expected_head or expected_history is None or expected_worktree is None:
1908
            failure = failure or "setup.fixture_oracle_failed"
1909
1910
        if failure is None:
1911
            proxy = CountingProxy("127.0.0.1", port).start()
1912
            measured_remote = f"http://127.0.0.1:{proxy.address[1]}"
1913
            probe = fixture_root / "baseline-probe"
1914
            probe_env = isolated_oak_fixture_env(fixture_root / "baseline-probe-config")
1915
            proxy.reset_counts()
1916
            result = run_bounded_harness_command(
1917
                [
1918
                    str(baseline_subject.bin),
1919
                    "clone",
1920
                    "--json",
1921
                    "--allow-legacy-scope",
1922
                    "--remote",
1923
                    measured_remote,
1924
                    "--branch",
1925
                    selected_branch,
1926
                    repo_spec,
1927
                    str(probe),
1928
                ],
1929
                fixture_root,
1930
                env=probe_env,
1931
            )
1932
            receipt = _parse_json_object(result.stdout)
1933
            evidence = legacy_selected_receipt_evidence(receipt or {})
1934
            baseline_proxy_stats = proxy.snapshot()
1935
            baseline_cache = stat_cache_snapshot(probe)
1936
            expected_artifacts = normalized_oak_artifacts(probe)
1937
            baseline_receipt_contract = legacy_selected_receipt_contract(evidence)
1938
            baseline_network_contract = {
1939
                "connections": baseline_proxy_stats.connections,
1940
                "receipt": baseline_receipt_contract,
1941
            }
1942
            expected_status_contract = normalized_status(
1943
                json_command([str(baseline_subject.bin), "status", "--json"], probe, probe_env)
1944
            )
1945
            probe_checks = {
1946
                "returncode": result.returncode == 0,
1947
                "legacy_route": evidence["legacy_route_observed"],
1948
                "passes_two": evidence["materialization_passes_observed"] == 2,
1949
                "head": evidence["head"] == expected_head,
1950
                "manifest": bool(evidence["manifest"]),
1951
                "artifacts": expected_artifacts is not None,
1952
                "cache": normalized_cache_contract(baseline_cache) is not None,
1953
                "status": expected_status_contract is not None,
1954
                "network": baseline_proxy_stats.connection_errors == 0,
1955
            }
1956
            if not all(probe_checks.values()):
1957
                failed_checks = ",".join(key for key, passed in probe_checks.items() if not passed)
1958
                failure = f"baseline_probe_failed:{failed_checks}"
1959
                if isinstance(baseline_cache, dict) and baseline_cache.get("error"):
1960
                    failure += f":{baseline_cache['error']}"
1961
            shutil.rmtree(probe, ignore_errors=True)
1962
1963
        if failure is None and proxy is not None:
1964
            measured_remote = f"http://127.0.0.1:{proxy.address[1]}"
1965
            order_rng = random.Random(random_seed)
1966
            for run_index in range(runs):
1967
                run_subjects = list(oak_subjects)
1968
                if randomize_order:
1969
                    order_rng.shuffle(run_subjects)
1970
                for subject in run_subjects:
1971
                    destination = run_root / f"materialization-{subject.name}-r{run_index}"
1972
                    subject_env = isolated_oak_fixture_env(
1973
                        run_root / f"materialization-config-{subject.name}-r{run_index}"
1974
                    )
1975
                    proxy.reset_counts()
1976
                    row = step_row(
1977
                        subject,
1978
                        scenario,
1979
                        "legacy_selected.acquire",
1980
                        [
1981
                            str(subject.bin),
1982
                            "clone",
1983
                            "--json",
1984
                            "--allow-legacy-scope",
1985
                            "--remote",
1986
                            measured_remote,
1987
                            "--branch",
1988
                            selected_branch,
1989
                            repo_spec,
1990
                            str(destination),
1991
                        ],
1992
                        fixture_root,
1993
                        run_index,
1994
                        {
1995
                            **fixture_meta,
1996
                            "run_subject_order": [item.name for item in run_subjects],
1997
                        },
1998
                        admitted,
1999
                        preserve_stdout=True,
2000
                        env=subject_env,
2001
                        timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
2002
                    )
2003
                    receipt = _parse_json_object(row.get("stdout"))
2004
                    evidence = legacy_selected_receipt_evidence(receipt or {})
2005
                    proxy_stats = proxy.snapshot()
2006
                    cache_before_status = stat_cache_snapshot(destination)
2007
                    artifacts = normalized_oak_artifacts(destination)
2008
                    trial_meta = {
2009
                        **fixture_meta,
2010
                        "run_subject_order": [item.name for item in run_subjects],
2011
                    }
2012
                    status_row = step_row(
2013
                        subject,
2014
                        scenario,
2015
                        "legacy_selected.first_status",
2016
                        [str(subject.bin), "status", "--json"],
2017
                        destination,
2018
                        run_index,
2019
                        trial_meta,
2020
                        admitted,
2021
                        preserve_stdout=True,
2022
                        env=subject_env,
2023
                        timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
2024
                    )
2025
                    status = normalized_status(_parse_json_object(status_row.get("stdout")) or {})
2026
                    commit_row = step_row(
2027
                        subject,
2028
                        scenario,
2029
                        "legacy_selected.first_noop_commit",
2030
                        [str(subject.bin), "commit"],
2031
                        destination,
2032
                        run_index,
2033
                        trial_meta,
2034
                        admitted,
2035
                        env=subject_env,
2036
                        timeout_seconds=INTERRUPTED_PUBLICATION_PROCESS_TIMEOUT_S,
2037
                    )
2038
                    history = normalized_history(
2039
                        json_command([str(subject.bin), "log", "--json"], destination, subject_env)
2040
                    )
2041
                    head_result = run_bounded_harness_command(
2042
                        [str(subject.bin), "hash"], destination, env=subject_env
2043
                    )
2044
                    local_head = head_result.stdout.strip() if head_result.returncode == 0 else None
2045
                    worktree = normalized_worktree_snapshot(destination) if destination.exists() else None
2046
                    advertised_head = remote_branch_head(direct_remote, repo_spec, selected_branch)
2047
                    subject_sha256 = binary_sha256(subject.bin)
2048
                    expected_passes = 2 if subject_sha256 == baseline_binary_sha256 else 1
2049
                    receipt_contract = legacy_selected_receipt_contract(evidence)
2050
                    network_contract = {
2051
                        "connections": proxy_stats.connections,
2052
                        "receipt": receipt_contract,
2053
                    }
2054
                    cache_contract = normalized_cache_contract(cache_before_status)
2055
                    baseline_cache_contract = normalized_cache_contract(baseline_cache)
2056
                    expected_cache_valid = subject_sha256 != baseline_binary_sha256
2057
                    cache_valid = bool(
2058
                        isinstance(cache_before_status, dict)
2059
                        and cache_before_status.get("all_metadata_valid") is expected_cache_valid
2060
                    )
2061
                    status_ok = status == expected_status_contract
2062
                    artifacts_ok = artifacts == expected_artifacts
2063
                    network_ok = network_contract == baseline_network_contract
2064
                    oracle_ok = bool(
2065
                        row_returncode(row) == 0
2066
                        and row_returncode(status_row) == 0
2067
                        and row_returncode(commit_row) == 0
2068
                        and evidence["legacy_route_observed"]
2069
                        and evidence["snapshot_bound"] is False
2070
                        and evidence["materialization_passes_observed"] == expected_passes
2071
                        and evidence["head"] == expected_head == local_head == advertised_head
2072
                        and evidence["manifest"] == baseline_receipt_contract["manifest"]
2073
                        and evidence["written_paths"] == len(expected_worktree)
2074
                        and receipt_contract == baseline_receipt_contract
2075
                        and history == expected_history
2076
                        and worktree == expected_worktree
2077
                        and status_ok
2078
                        and artifacts_ok
2079
                        and cache_contract == baseline_cache_contract
2080
                        and cache_valid
2081
                        and network_ok
2082
                        and proxy_stats.connection_errors == 0
2083
                    )
2084
                    row.update(
2085
                        {
2086
                            **evidence,
2087
                            "network_connections_observed": proxy_stats.connections,
2088
                            "network_client_to_server_bytes_observed": proxy_stats.client_to_upstream_bytes,
2089
                            "network_server_to_client_bytes_observed": proxy_stats.upstream_to_client_bytes,
2090
                            "network_connection_errors": proxy_stats.connection_errors,
2091
                            "normalized_network_work": network_contract,
2092
                            "oracle_network_work_matches_baseline": network_ok,
2093
                            "oracle_head_matches": expected_head == local_head == advertised_head,
2094
                            "oracle_manifest_matches": evidence["manifest"]
2095
                            == baseline_receipt_contract["manifest"],
2096
                            "oracle_receipt_contract_matches": receipt_contract
2097
                            == baseline_receipt_contract,
2098
                            "oracle_history_matches": history == expected_history,
2099
                            "oracle_worktree_bytes_and_modes_match": worktree == expected_worktree,
2100
                            "oracle_clean_status": status_ok,
2101
                            "oracle_local_artifact_names_match": artifacts_ok,
2102
                            "oracle_stat_cache_contract_matches": cache_contract
2103
                            == baseline_cache_contract,
2104
                            "stat_cache_metadata_valid_before_status": (
2105
                                cache_before_status.get("all_metadata_valid")
2106
                                if isinstance(cache_before_status, dict)
2107
                                else None
2108
                            ),
2109
                            "stat_cache_rows_before_status": (
2110
                                len(cache_before_status.get("rows", []))
2111
                                if isinstance(cache_before_status, dict)
2112
                                and isinstance(cache_before_status.get("rows"), list)
2113
                                else None
2114
                            ),
2115
                            "stat_cache_expected_valid_before_status": expected_cache_valid,
2116
                            "oracle_clone_residue_absent": artifacts_ok,
2117
                            "oracle_passed": oracle_ok,
2118
                            "repository_cache_state": "fresh_destination_no_shared_object_store",
2119
                        }
2120
                    )
2121
                    for followup_row in (status_row, commit_row):
2122
                        followup_row.update(
2123
                            {
2124
                                "materialization_timing_controls_met": timing_controls_met,
2125
                                "stat_cache_expected_valid_before_first_status": expected_cache_valid,
2126
                                "oracle_passed": oracle_ok,
2127
                            }
2128
                        )
2129
                    if not oracle_ok:
2130
                        row["returncode"] = 1
2131
                        row["stderr"] = "legacy_selected_materialization_oracle_failed"
2132
                        status_row["returncode"] = 1
2133
                        commit_row["returncode"] = 1
2134
                    total = phase_total_row(
2135
                        [row, status_row, commit_row],
2136
                        subject,
2137
                        scenario,
2138
                        run_index,
2139
                        trial_meta,
2140
                        "legacy_selected_clone_materialization.total",
2141
                    )
2142
                    total.update(
2143
                        {
2144
                            key: row[key]
2145
                            for key in (
2146
                                "materialization_passes_observed",
2147
                                "network_connections_observed",
2148
                                "network_client_to_server_bytes_observed",
2149
                                "network_server_to_client_bytes_observed",
2150
                                "oracle_passed",
2151
                                "oracle_head_matches",
2152
                                "oracle_manifest_matches",
2153
                                "oracle_receipt_contract_matches",
2154
                                "oracle_history_matches",
2155
                                "oracle_worktree_bytes_and_modes_match",
2156
                                "oracle_clean_status",
2157
                                "oracle_local_artifact_names_match",
2158
                                "oracle_stat_cache_contract_matches",
2159
                                "stat_cache_metadata_valid_before_status",
2160
                                "stat_cache_rows_before_status",
2161
                                "oracle_clone_residue_absent",
2162
                                "oracle_network_work_matches_baseline",
2163
                                "serve_binary_sha256",
2164
                                "serve_source_head",
2165
                                "fixture_id",
2166
                                "os_cache_state",
2167
                                "materialization_timing_controls_met",
2168
                            )
2169
                        }
2170
                    )
2171
                    total.update(
2172
                        {
2173
                            "clone_elapsed_ms": row.get("elapsed_ms"),
2174
                            "first_status_elapsed_ms": status_row.get("elapsed_ms"),
2175
                            "first_noop_commit_elapsed_ms": commit_row.get("elapsed_ms"),
2176
                        }
2177
                    )
2178
                    rows.extend([row, status_row, commit_row, total])
2179
                    shutil.rmtree(destination, ignore_errors=True)
2180
    finally:
2181
        if proxy is not None:
2182
            proxy.stop()
2183
        if server is not None:
2184
            stop_owned_process(server)
2185
        shutil.rmtree(fixture_root, ignore_errors=True)
2186
2187
    if failure is not None:
2188
        rows.append(
2189
            harness_step_row(
2190
                setup_subject,
2191
                scenario,
2192
                "scenario.fixture_failure",
2193
                0,
2194
                fixture_meta,
2195
                0.0,
2196
                returncode=1,
2197
            )
2198
        )
2199
        rows[-1]["stderr"] = failure
2200
    return rows
2201
2202
2203
def run_git_loop(
2204
    subject: Subject,
2205
    remote: oakbench_remotes.RemoteResolution,
2206
    run_root: Path,
2207
    run_index: int,
2208
    metadata: dict[str, Any],
2209
    admitted: int,
2210
) -> list[dict[str, Any]]:
2211
    scenario = "task_loop"
2212
    meta = {
2213
        **metadata,
2214
        **remote.row_fields(),
2215
        "benchmark_track": (
2216
            "core-equivalent"
2217
            if remote.transport == oakbench_remotes.TRANSPORT_LOCAL_FILE
2218
            else "agent-default"
2219
        ),
2220
    }
2221
    vcs = str(subject.bin)
2222
    rows: list[dict[str, Any]] = []
2223
2224
    seed_error = ensure_git_remote_seeded(subject, remote, run_root)
2225
    if seed_error:
2226
        return [skip_row(subject, scenario, run_index, meta, seed_error)]
2227
2228
    # Untimed: the base clone is the long-lived checkout worktrees hang off;
2229
    # an agent fleet pays it once, not per task.
2230
    base = run_root / f"tl-git-base-r{run_index}"
2231
    clone = run_untimed([vcs, "clone", "-q", str(remote.repo), str(base)], run_root)
2232
    if clone.returncode != 0:
2233
        return [skip_row(subject, scenario, run_index, meta, f"base clone failed: {clone.stderr.strip()[:200]}")]
2234
2235
    branches: dict[str, str] = {}
2236
    payloads: dict[str, str] = {}
2237
    for phase in PHASES:
2238
        workspace = run_root / f"tl-{subject.name}-r{run_index}-{phase}"
2239
        branch = oakbench_remotes.disposable_branch(
2240
            str(metadata["bench_id"]), f"taskloop-{phase}", subject.name, run_index
2241
        )
2242
        branches[phase] = branch
2243
        payload = f"payload {phase} run={run_index} bench={metadata['bench_id']}"
2244
        payloads[phase] = payload
2245
        acquire = step_row(
2246
            subject, scenario, f"{phase}.acquire",
2247
            [vcs, "worktree", "add", str(workspace), "-b", branch], base, run_index, meta, admitted,
2248
        )
2249
        rows.append(acquire)
2250
        if int(acquire["returncode"]) != 0:
2251
            return rows
2252
        rows.append(edit_step(subject, scenario, f"{phase}.edit", workspace, payload, run_index, meta, admitted))
2253
        # Agent-default snapshot: git pays stage+commit as two calls.
2254
        rows.append(step_row(subject, scenario, f"{phase}.snapshot.add", [vcs, "add", "."], workspace, run_index, meta, admitted))
2255
        rows.append(
2256
            step_row(
2257
                subject, scenario, f"{phase}.snapshot.commit",
2258
                [vcs, "commit", "-q", "-m", f"bench task loop {metadata['bench_id']} r{run_index} {phase}"],
2259
                workspace, run_index, meta, admitted,
2260
            )
2261
        )
2262
        rows.append(
2263
            step_row(
2264
                subject, scenario, f"{phase}.publish",
2265
                [vcs, "push", "-q", "-u", "origin", branch], workspace, run_index, meta, admitted,
2266
            )
2267
        )
2268
        # No task.describe: git's description is the commit message already
2269
        # paid in {phase}.snapshot.commit (asymmetry documented in the lane doc).
2270
        rows.append(
2271
            step_row(
2272
                subject, scenario, f"{phase}.end",
2273
                [vcs, "worktree", "remove", str(workspace)], base, run_index, meta, admitted,
2274
            )
2275
        )
2276
2277
    # Untimed oracle: payloads visible on the remote branches.
2278
    oracle_ok: bool | None = True
2279
    for phase in PHASES:
2280
        fetch = run_untimed([vcs, "fetch", "-q", "origin", branches[phase]], base)
2281
        show = run_untimed([vcs, "show", f"FETCH_HEAD:task-notes.md"], base)
2282
        if fetch.returncode != 0 or payloads[phase] not in show.stdout:
2283
            oracle_ok = False
2284
    rows.append(
2285
        {
2286
            **phase_total_row(rows, subject, scenario, run_index, meta, "task_loop.total"),
2287
            "oracle_payload_on_remote_branch": oracle_ok,
2288
            "workspace_branches": branches,
2289
            "measurement_source": "fetch_plus_show_grep",
2290
        }
2291
    )
2292
    return rows
2293
2294
2295
def loop_summary_fields(rows: list[dict[str, Any]]) -> dict[str, Any]:
2296
    """followup-vs-task warm deltas computed from this run's step rows."""
2297
    def op_ms(operation: str) -> float | None:
2298
        for row in rows:
2299
            if row.get("operation") == operation:
2300
                return float(row.get("elapsed_ms", 0.0))
2301
        return None
2302
2303
    def phase_ms(prefix: str) -> float:
2304
        return sum(
2305
            float(row.get("elapsed_ms", 0.0))
2306
            for row in rows
2307
            if str(row.get("operation", "")).startswith(prefix + ".")
2308
        )
2309
2310
    task_acquire = op_ms("task.acquire")
2311
    followup_acquire = op_ms("followup.acquire")
2312
    fields: dict[str, Any] = {
2313
        "task_phase_ms": round(phase_ms("task"), 3),
2314
        "followup_phase_ms": round(phase_ms("followup"), 3),
2315
    }
2316
    if task_acquire is not None and followup_acquire is not None:
2317
        fields["followup_warm_delta_ms"] = round(followup_acquire - task_acquire, 3)
2318
        fields["acquire_ms_task"] = round(task_acquire, 3)
2319
        fields["acquire_ms_followup"] = round(followup_acquire, 3)
2320
    else:
2321
        fields["followup_warm_delta_ms"] = None
2322
    return fields
2323
2324
2325
def summary_text(rows: list[dict[str, Any]]) -> str:
2326
    totals = [row for row in rows if row.get("operation") == "task_loop.total"]
2327
    lines = [
2328
        "# Task Loop Summary",
2329
        "",
2330
        "Whole-task cost of the signature motion: acquire workspace -> edit -> snapshot ->",
2331
        "publish -> describe -> end, then a follow-up task in a fresh workspace.",
2332
        "Command sequences are NOT step-equivalent across subjects",
2333
        "(oak pays desc; git's description is its commit message). Latency across unlike",
2334
        "transports is never comparable; scripted call equivalents and estimated token proxies are.",
2335
        "",
2336
        "| Subject | Run | Transport | Wall ms | Scripted call equivalents | Cost-wt tokens (env) | Warm acquire delta ms | Oracle |",
2337
        "| --- | ---: | --- | ---: | ---: | ---: | ---: | --- |",
2338
    ]
2339
    for row in totals:
2340
        oracle = row.get("oracle_payload_on_remote_branch")
2341
        lines.append(
2342
            "| `{}` | {} | {} | {} | {} | {} | {} | {} |".format(
2343
                row["subject"],
2344
                row["run"],
2345
                row.get("remote_transport", "unmeasured"),
2346
                fmt_ms(row.get("elapsed_ms")),
2347
                row.get("tool_call_count", "unmeasured"),
2348
                row.get("estimated_cost_weighted_tokens_with_envelope", "unmeasured"),
2349
                row.get("followup_warm_delta_ms", "unmeasured"),
2350
                "pass" if oracle else ("FAIL" if oracle is False else "unmeasured"),
2351
            )
2352
        )
2353
    skips = [
2354
        row for row in rows
2355
        if row.get("skipped") and row.get("scenario") == TASK_LOOP_SCENARIO
2356
    ]
2357
    for row in skips:
2358
        lines.append(f"| `{row['subject']}` | {row['run']} | skipped | | | | | {row['skip_reason'][:70]} |")
2359
2360
    agent_totals = [
2361
        row
2362
        for row in rows
2363
        if row.get("operation") in ("agent_task.total", "agent_task_oak_local.total")
2364
    ]
2365
    if agent_totals:
2366
        lines.extend(
2367
            [
2368
                "",
2369
                "## Cold clone and warm isolated task",
2370
                "",
2371
                "Local-file Git rows compare a fresh full clone with a worktree sharing",
2372
                "the first clone's object store. Oak local rows use two independent fresh",
2373
                "clones against an owned loopback server; they do not claim a shared cache.",
2374
                "OS cache and network bytes remain unknown. Hosted CI/review are skips.",
2375
                "",
2376
                "| Subject | Run | Wall ms | Scripted call equivalents | Output bytes | Coverage | Oracle |",
2377
                "| --- | ---: | ---: | ---: | ---: | --- | --- |",
2378
            ]
2379
        )
2380
        for row in agent_totals:
2381
            lines.append(
2382
                "| `{}` | {} | {} | {} | {} | {} | {} |".format(
2383
                    row["subject"],
2384
                    row["run"],
2385
                    fmt_ms(row.get("elapsed_ms")),
2386
                    row.get("tool_call_count", "unmeasured"),
2387
                    row.get("raw_output_bytes", "unmeasured"),
2388
                    "complete" if row.get("coverage_complete") else "partial",
2389
                    "pass" if row.get("oracle_payload_on_remote_branch") else "FAIL",
2390
                )
2391
            )
2392
2393
    materialization_totals = [
2394
        row
2395
        for row in rows
2396
        if row.get("operation") == "legacy_selected_clone_materialization.total"
2397
    ]
2398
    if materialization_totals:
2399
        lines.extend(
2400
            [
2401
                "",
2402
                "## Legacy selected-branch materialization",
2403
                "",
2404
                "One fixed loopback Serve and one immutable fixture are shared across subjects.",
2405
                "Fresh destinations do not imply a cold OS page cache. Timing claims require",
2406
                "separate interleaved A/A noise and A/B campaigns.",
2407
                "",
2408
                "| Subject | Run | Clone ms | First status ms | First no-op commit ms | Passes | Harness controls met | Oracle |",
2409
                "| --- | ---: | ---: | ---: | ---: | ---: | --- | --- |",
2410
            ]
2411
        )
2412
        for row in materialization_totals:
2413
            lines.append(
2414
                "| `{}` | {} | {} | {} | {} | {} | {} | {} |".format(
2415
                    row["subject"],
2416
                    row["run"],
2417
                    fmt_ms(row.get("clone_elapsed_ms")),
2418
                    fmt_ms(row.get("first_status_elapsed_ms")),
2419
                    fmt_ms(row.get("first_noop_commit_elapsed_ms")),
2420
                    row.get("materialization_passes_observed", "unmeasured"),
2421
                    "yes" if row.get("materialization_timing_controls_met") else "no",
2422
                    "pass" if row.get("oracle_passed") else "FAIL",
2423
                )
2424
            )
2425
2426
    interrupted_totals = [
2427
        row for row in rows if row.get("operation") == "interrupted_publication.total"
2428
    ]
2429
    if interrupted_totals:
2430
        lines.extend(
2431
            [
2432
                "",
2433
                "## Interrupted publication",
2434
                "",
2435
                "A loopback fixture validates the request framing and drops the response.",
2436
                "Fixture delivery knowledge is reported separately from the CLI's own outcome",
2437
                "claim. Request receipt is not durability proof; the harness never retries.",
2438
                "",
2439
                "| Subject | Run | Wall ms | Scripted call equivalents | Output bytes | CLI outcome | Fixture delivery | POSTs | Request bytes | Head/work preserved |",
2440
                "| --- | ---: | ---: | ---: | ---: | --- | --- | ---: | ---: | --- |",
2441
            ]
2442
        )
2443
        for row in interrupted_totals:
2444
            lines.append(
2445
                "| `{}` | {} | {} | {} | {} | `{}` | `{}` | {} | {} | {}/{} |".format(
2446
                    row["subject"],
2447
                    row["run"],
2448
                    fmt_ms(row.get("elapsed_ms")),
2449
                    row.get("tool_call_count", "unmeasured"),
2450
                    row.get("raw_output_bytes", "unmeasured"),
2451
                    row.get("publication_outcome", "unmeasured"),
2452
                    row.get("fixture_delivery_state", "unmeasured"),
2453
                    row.get("mutation_requests_observed", "unmeasured"),
2454
                    row.get("network_request_bytes_observed", "unmeasured"),
2455
                    "yes" if row.get("local_head_preserved") else "no",
2456
                    "yes" if row.get("local_work_preserved") else "no",
2457
                )
2458
            )
2459
    return "\n".join(lines) + "\n"
2460
2461
2462
def fmt_ms(value: Any) -> str:
2463
    try:
2464
        return f"{float(value):.0f}"
2465
    except (TypeError, ValueError):
2466
        return "unmeasured"
2467
2468
2469
def main() -> int:
2470
    args = parse_args()
2471
    scenarios = [item.strip() for item in args.scenarios.split(",") if item.strip()]
2472
    known_scenarios = {
2473
        TASK_LOOP_SCENARIO,
2474
        AGENT_TASK_SCENARIO,
2475
        AGENT_TASK_OAK_LOCAL_SCENARIO,
2476
        LEGACY_SELECTED_MATERIALIZATION_SCENARIO,
2477
        INTERRUPTED_PUBLICATION_SCENARIO,
2478
    }
2479
    unknown_scenarios = sorted(set(scenarios) - known_scenarios)
2480
    if unknown_scenarios:
2481
        raise SystemExit(
2482
            "Unknown scenarios: " + ", ".join(unknown_scenarios)
2483
            + "; valid: " + ", ".join(sorted(known_scenarios))
2484
        )
2485
    subjects = load_subjects(args)
2486
    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
2487
    args.results.mkdir(parents=True, exist_ok=True)
2488
    args.workdir.mkdir(parents=True, exist_ok=True)
2489
    run_root = args.workdir / "runs" / timestamp
2490
    run_root.mkdir(parents=True, exist_ok=True)
2491
2492
    metadata = {
2493
        "bench_id": timestamp,
2494
        "profile": "task-loop",
2495
        "benchmark_track": "agent-default",
2496
        "timestamp_utc": timestamp,
2497
        "host": platform.node(),
2498
        "platform": platform.platform(),
2499
        "machine": platform.machine(),
2500
        "python": platform.python_version(),
2501
        "env_isolation_version": ENV_ISOLATION_VERSION,
2502
        "subject_versions": subject_versions(subjects),
2503
        "subject_details": subject_details(subjects),
2504
        "source": source_metadata(args.oak_repo),
2505
        "git_remote_mode": args.git_remote,
2506
        "trial_order_randomized": args.randomize_subject_order,
2507
        "trial_order_seed": args.random_seed if args.randomize_subject_order else None,
2508
    }
2509
2510
    rows: list[dict[str, Any]] = []
2511
    with measurement_lock("task_loop") as lock_info:
2512
        metadata["measurement_lock_wait_ms"] = lock_info.wait_ms
2513
        metadata["measurement_lock"] = "held" if lock_info.enabled else "disabled"
2514
        order_rng = random.Random(args.random_seed)
2515
        if LEGACY_SELECTED_MATERIALIZATION_SCENARIO in scenarios:
2516
            rows.extend(
2517
                run_legacy_selected_materialization_campaign(
2518
                    subjects,
2519
                    args.oak_serve_bin,
2520
                    args.oak_serve_source,
2521
                    args.oak_serve_source_head,
2522
                    args.materialization_baseline_subject,
2523
                    run_root,
2524
                    args.runs,
2525
                    metadata,
2526
                    args.admitted_output_chars,
2527
                    args.randomize_subject_order,
2528
                    args.random_seed,
2529
                    args.materialization_timing_controls_met,
2530
                )
2531
            )
2532
        ordinary_scenarios = [
2533
            scenario for scenario in scenarios if scenario != LEGACY_SELECTED_MATERIALIZATION_SCENARIO
2534
        ]
2535
        for run_index in range(args.runs):
2536
            run_subjects = list(subjects)
2537
            if args.randomize_subject_order:
2538
                order_rng.shuffle(run_subjects)
2539
            run_metadata = {
2540
                **metadata,
2541
                "run_subject_order": [subject.name for subject in run_subjects],
2542
            }
2543
            for subject in run_subjects:
2544
                for scenario in ordinary_scenarios:
2545
                    print(f"[run] {scenario} run={run_index} subject={subject.name}", flush=True)
2546
                    if scenario == INTERRUPTED_PUBLICATION_SCENARIO:
2547
                        rows.extend(
2548
                            run_interrupted_publication(
2549
                                subject, run_root, run_index, run_metadata, args.admitted_output_chars
2550
                            )
2551
                        )
2552
                        continue
2553
                    if scenario == AGENT_TASK_SCENARIO:
2554
                        if subject.kind != "git":
2555
                            rows.append(
2556
                                unmeasured_step_row(
2557
                                    subject,
2558
                                    scenario,
2559
                                    "agent_task.skipped",
2560
                                    run_index,
2561
                                    run_metadata,
2562
                                    "Oak end-to-end publication and exact CI require an explicitly configured disposable staging lane; production mutation is forbidden",
2563
                                )
2564
                            )
2565
                            continue
2566
                        remote_create_start = time.perf_counter()
2567
                        remote = oakbench_remotes.make_git_bare_remote(
2568
                            subject.bin, run_root, f"agent-task-remote-r{run_index}.git"
2569
                        )
2570
                        remote_create_ms = (time.perf_counter() - remote_create_start) * 1000
2571
                        rows.extend(
2572
                            run_git_agent_task_baseline(
2573
                                subject,
2574
                                remote,
2575
                                run_root,
2576
                                run_index,
2577
                                run_metadata,
2578
                                args.admitted_output_chars,
2579
                                remote_create_ms,
2580
                            )
2581
                        )
2582
                        continue
2583
                    if scenario == AGENT_TASK_OAK_LOCAL_SCENARIO:
2584
                        if subject.kind != "oak":
2585
                            rows.append(
2586
                                unmeasured_step_row(
2587
                                    subject,
2588
                                    scenario,
2589
                                    "agent_task_oak_local.skipped",
2590
                                    run_index,
2591
                                    run_metadata,
2592
                                    "owned local oak serve scenario applies only to Oak subjects",
2593
                                )
2594
                            )
2595
                            continue
2596
                        rows.extend(
2597
                            run_oak_local_agent_task_baseline(
2598
                                subject,
2599
                                run_root,
2600
                                run_index,
2601
                                run_metadata,
2602
                                args.admitted_output_chars,
2603
                            )
2604
                        )
2605
                        continue
2606
                    if subject.kind == "git":
2607
                        if args.git_remote == "github":
2608
                            remote = oakbench_remotes.resolve_git_github_remote()
2609
                        else:
2610
                            remote = oakbench_remotes.make_git_bare_remote(
2611
                                subject.bin, run_root, f"task-loop-remote-r{run_index}.git"
2612
                            )
2613
                        if not remote.resolved:
2614
                            rows.append(skip_row(subject, TASK_LOOP_SCENARIO, run_index, run_metadata, remote.skip_reason or ""))
2615
                            continue
2616
                        subject_rows = run_git_loop(subject, remote, run_root, run_index, run_metadata, args.admitted_output_chars)
2617
                    else:
2618
                        remote = oakbench_remotes.resolve_oak_remote("sync")
2619
                        if not remote.resolved:
2620
                            rows.append(skip_row(subject, TASK_LOOP_SCENARIO, run_index, run_metadata, remote.skip_reason or ""))
2621
                            continue
2622
                        subject_rows = run_oak_loop(subject, remote, run_root, run_index, run_metadata, args.admitted_output_chars)
2623
                    total = next((row for row in subject_rows if row.get("operation") == "task_loop.total"), None)
2624
                    if total is not None:
2625
                        total.update(loop_summary_fields(subject_rows))
2626
                    rows.extend(subject_rows)
2627
2628
    store = ResultsStore(args.results, lane="task-loop")
2629
    raw_path, summary_path = store.write(timestamp, rows, summary_text(rows))
2630
    if not args.keep_workdirs:
2631
        shutil.rmtree(run_root, ignore_errors=True)
2632
    print(f"[result] {raw_path}")
2633
    print(f"[summary] {summary_path}")
2634
    return 1 if has_lane_failures(rows) else 0
2635
2636
2637
if __name__ == "__main__":
2638
    raise SystemExit(main())