Viewing files at commit ada5e175e5bfCommit detailsView latest

suites/scale/scalesuite/ops.py 930 lines · 41.9 KB · python
1
"""The scale-lane operation catalog: every user-facing op, git and oak.
2
3
Each op is one function per subject kind. Inside it, ONLY ``ctx.timed(...)``
4
calls are measured (their elapsed times are summed into the row); setup,
5
oracle and teardown use ``ctx.untimed(...)`` and run outside the timed path
6
(ADR-0011: oracles untimed). An op whose timed command fails emits no oracle
7
verdict (the row carries the failure); an op with no equivalent for a kind
8
returns a skip (returncode 77 with a reason) so coverage gaps are rows, never
9
silent omissions.
10
11
Oracle evidence classes (``oracle_reader``):
12
- ``git_fixture_instrument``: expected values read from the generated
13
  fixture with git plumbing, compared against the subject's worktree bytes
14
  hashed by the harness (independent of the subject under test);
15
- ``subject_output``: the subject's own output parsed by the harness
16
  (counts, names, JSON fields);
17
- ``subject_file_inspect``: Oak's ``oak file inspect`` (subject-produced)
18
  or git's ``git show``/``cat-file`` of the subject repository.
19
20
t/perf references cite upstream git test FILE NAMES only (GPL-2.0); the ops
21
are re-implemented here, nothing is copied (docs/prior-art.md).
22
"""
23
24
from __future__ import annotations
25
26
import hashlib
27
import json
28
import os
29
import random
30
import shutil
31
from dataclasses import dataclass, field
32
from pathlib import Path
33
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
34
35
from . import generator as gen
36
37
UNIT_ORDER: Tuple[Tuple[str, ...], ...] = (
38
    ("status.clean",),
39
    ("log.full",),
40
    ("log.n",),
41
    ("log.path",),
42
    ("blame",),
43
    ("file.show",),
44
    ("branch.list",),
45
    ("review.branch_diff",),
46
    ("merge.preview",),
47
    ("merge.preview.conflict",),
48
    ("switch.existing", "switch.back"),
49
    ("branch.create",),
50
    ("status.dirty", "diff.dirty", "commit.dirty"),
51
    ("merge.local",),
52
    ("merge.conflict",),
53
    ("push.branch", "pull.ff"),
54
    ("pull.diverged",),
55
    ("clone.full",),
56
    ("gc",),
57
)
58
59
60
@dataclass(frozen=True)
61
class OpInfo:
62
    name: str
63
    description: str
64
    tperf: Tuple[str, ...] = ()
65
    core_yaml: Tuple[str, ...] = ()
66
    mutates: bool = False
67
68
69
OP_INFO: Dict[str, OpInfo] = {info.name: info for info in (
70
    OpInfo("status.clean", "Status of a clean working tree (porcelain).",
71
           ("p0005-status.sh", "p7519-fsmonitor.sh"), ("status.clean",)),
72
    OpInfo("status.dirty", "Status after the tier's dirty shape (modify/add/delete/rename).",
73
           ("p0005-status.sh", "p7519-fsmonitor.sh"), ("status.dirty",)),
74
    OpInfo("diff.dirty", "Full worktree diff of the dirty shape.",
75
           ("p4000-diff-algorithms.sh", "p0009-diff-pathspec.sh"), ("diff.dirty",)),
76
    OpInfo("commit.dirty", "Commit the dirty shape (git: add -A + commit, summed).",
77
           ("p0007-write-cache.sh", "p0090-cache-tree.sh"), ("snapshot.dirty",), mutates=True),
78
    OpInfo("log.full", "Entire history reachable from the current branch, one line per commit.",
79
           ("p0001-rev-list.sh", "p4205-log-pretty-formats.sh"), ("history.log.range",)),
80
    OpInfo("log.n", "The newest 20 commits.", ("p4205-log-pretty-formats.sh",), ("history.log.tip",)),
81
    OpInfo("log.path", "History of one path (the fixture's probe path).",
82
           ("p4211-line-log.sh", "p0001-rev-list.sh"), ("history.log.range",)),
83
    OpInfo("blame", "Line authorship of the probe path.", ("p8020-last-modified.sh",), ()),
84
    OpInfo("file.show", "Committed bytes of the probe path at HEAD.", ("p1006-cat-file.sh",), ()),
85
    OpInfo("branch.list", "Enumerate branches.", ("p6300-for-each-ref.sh",), ("branch.list",)),
86
    OpInfo("branch.create", "Create and switch to a new branch.", ("p1400-update-ref.sh",),
87
           ("branch.create",), mutates=True),
88
    OpInfo("switch.existing", "Switch to the fixture's side branch b00000.",
89
           ("p0006-read-tree-checkout.sh", "p2000-sparse-operations.sh"), ("branch.switch",), mutates=True),
90
    OpInfo("switch.back", "Switch back to the base branch.", ("p0006-read-tree-checkout.sh",),
91
           ("history.restore_head",), mutates=True),
92
    OpInfo("review.branch_diff", "Checkout-free diff of b00000 against main.",
93
           ("p4000-diff-algorithms.sh", "p6010-merge-base.sh"), ()),
94
    OpInfo("merge.preview", "Dry-run merge prediction of b00000 into main.",
95
           ("p6010-merge-base.sh",), ()),
96
    OpInfo("merge.preview.conflict", "Dry-run merge prediction of the conflict-shape branches.",
97
           ("p6010-merge-base.sh",), ("merge.conflict",)),
98
    OpInfo("merge.local", "Local (non-fast-forward) merge of b00000 into a scratch branch.",
99
           ("p6010-merge-base.sh", "p3400-rebase.sh"), (), mutates=True),
100
    OpInfo("merge.conflict", "Local merge of the conflict-shape branches (expected to conflict), then abort.",
101
           ("p6010-merge-base.sh",), ("merge.conflict", "merge.abort"), mutates=True),
102
    OpInfo("push.branch", "Push one new commit over loopback HTTP.",
103
           ("p5313-pack-objects.sh", "p5310-pack-bitmaps.sh"), (), mutates=True),
104
    OpInfo("pull.ff", "Pull one upstream commit (fast-forward).",
105
           ("p5311-pack-bitmaps-fetch.sh", "p5551-fetch-rescan.sh"), (), mutates=True),
106
    OpInfo("pull.diverged", "Pull when local and upstream both have one new commit.",
107
           ("p5311-pack-bitmaps-fetch.sh", "p3400-rebase.sh"), (), mutates=True),
108
    OpInfo("clone.full", "Full clone over loopback HTTP into a new directory.",
109
           ("p5600-partial-clone.sh", "p5601-clone-reference.sh"), ()),
110
    OpInfo("gc", "Repository maintenance (git gc / oak maintenance compact).",
111
           ("p5304-prune.sh", "p5313-pack-objects.sh", "p5303-many-packs.sh"), (), mutates=True),
112
)}
113
114
ALL_OPS: Tuple[str, ...] = tuple(op for unit in UNIT_ORDER for op in unit)
115
assert set(ALL_OPS) == set(OP_INFO), "UNIT_ORDER and OP_INFO must list the same operations"
116
117
118
@dataclass
119
class Timed:
120
    label: str
121
    command: List[str]
122
    capture: Any
123
124
125
@dataclass
126
class Outcome:
127
    timed: List[Timed] = field(default_factory=list)
128
    failed: Optional[str] = None
129
    skip_reason: Optional[str] = None
130
    oracle_ok: Optional[bool] = None
131
    oracle_method: Optional[str] = None
132
    oracle_reader: Optional[str] = None
133
    oracle_detail: Dict[str, Any] = field(default_factory=dict)
134
    semantic_class: Optional[str] = None
135
    extra: Dict[str, Any] = field(default_factory=dict)
136
137
138
class OpFailed(Exception):
139
    """A timed command failed; the row records it and the unit aborts."""
140
141
142
class OpSkipped(Exception):
143
    pass
144
145
146
class Ctx:
147
    """Per (subject, run, op) execution context."""
148
149
    def __init__(self, prepared: Any, subject: Any, run: int, runner: Callable[..., Any],
150
                 scratch: Path, timeout_s: float) -> None:
151
        self.p = prepared
152
        self.subject = subject
153
        self.kind = subject.kind
154
        self.bin = str(subject.bin)
155
        self.run = run
156
        self._run_timed = runner
157
        self.scratch = scratch
158
        self.timeout_s = timeout_s
159
        self.outcome = Outcome()
160
        self.tag = f"{subject.name}-r{run}"
161
162
    # -- command helpers ----------------------------------------------------
163
164
    def timed(self, args: Sequence[str], cwd: Path, *, label: str = "main", accept: Sequence[int] = (0,),
165
              raw: bool = False) -> Any:
166
        command = [self.bin, *args]
167
        capture = self._run_timed(command, cwd, self.p.env, 4000, full_output_bytes=64 << 20,
168
                                  timeout_seconds=self.timeout_s, raw_output_bytes=(64 << 20) if raw else None)
169
        self.outcome.timed.append(Timed(label, command, capture))
170
        if capture.returncode not in accept:
171
            tail = (capture.stderr_text or capture.stdout_text or "")[-400:]
172
            raise OpFailed(f"{label}: exit {capture.returncode}: {tail}")
173
        return capture
174
175
    def untimed(self, args: Sequence[str], cwd: Path, *, check: bool = True, binary: Optional[str] = None
176
                ) -> Any:
177
        from .workspace import run
178
        return run([binary or self.bin, *args], cwd=cwd, env=self.p.env, check=check)
179
180
    def git(self, args: Sequence[str], cwd: Path, check: bool = True) -> Any:
181
        """The oracle's git instrument (the resolved real git), never the subject binary."""
182
        return self.untimed(args, cwd, check=check, binary=self.p.git_bin)
183
184
    def oracle(self, ok: bool, method: str, reader: str, **detail: Any) -> None:
185
        self.outcome.oracle_ok = None if "reader_unavailable" in detail else bool(ok)
186
        self.outcome.oracle_method = method
187
        self.outcome.oracle_reader = reader
188
        self.outcome.oracle_detail.update(detail)
189
190
    # -- workspace paths ----------------------------------------------------
191
192
    @property
193
    def work(self) -> Path:
194
        return self.p.path(self.kind, "work")
195
196
    def client(self, name: str) -> Path:
197
        return self.p.path(self.kind, name)
198
199
    @property
200
    def state(self) -> Dict[str, Any]:
201
        return self.p.state.setdefault(self.kind, {"harness_commits": 0, "deleted": set(), "server_extra": 0,
202
                                                     "branch_seq": 0})
203
204
205
# ---------------------------------------------------------------------------
206
# Shared oracle helpers (harness-side, independent of the subject)
207
# ---------------------------------------------------------------------------
208
209
210
def worktree_files(root: Path) -> List[str]:
211
    out = []
212
    for base, dirs, files in os.walk(root):
213
        dirs[:] = [d for d in dirs if d not in (".git", ".oak")]
214
        rel = os.path.relpath(base, root)
215
        for name in files:
216
            out.append(name if rel == "." else f"{rel}/{name}".replace(os.sep, "/"))
217
    return out
218
219
220
def blob_id_of(path: Path) -> str:
221
    digest = hashlib.sha1()
222
    size = path.stat().st_size
223
    digest.update(b"blob %d\0" % size)
224
    with path.open("rb") as fh:
225
        for chunk in iter(lambda: fh.read(1 << 20), b""):
226
            digest.update(chunk)
227
    return digest.hexdigest()
228
229
230
def check_worktree(ctx: Ctx, root: Path, expected_count: Optional[int], ignore_prefixes: Sequence[str] = ()
231
                   ) -> Tuple[bool, Dict[str, Any]]:
232
    """Sampled content check against the fixture plus an exact file count."""
233
    exp = ctx.p.expected
234
    files = [f for f in worktree_files(root) if not any(f.startswith(prefix) for prefix in ignore_prefixes)]
235
    mismatched = []
236
    for rel, blob in exp.sample.items():
237
        if rel in ctx.state["deleted"]:
238
            continue
239
        target = root / rel
240
        if not target.is_file() or blob_id_of(target) != blob:
241
            mismatched.append(rel)
242
    detail = {"files_seen": len(files), "files_expected": expected_count, "sample_checked": len(exp.sample),
243
              "sample_mismatched": mismatched[:10], "sample_mismatch_count": len(mismatched)}
244
    ok = not mismatched and (expected_count is None or len(files) == expected_count)
245
    return ok, detail
246
247
248
def porcelain_paths(text: str) -> List[str]:
249
    paths = []
250
    for line in text.splitlines():
251
        line = line.rstrip()
252
        if len(line) < 3:
253
            continue
254
        rest = line[2:].strip() if line[2] == " " or line[1] == " " else line.split(None, 1)[-1]
255
        if " -> " in rest:
256
            old, new = rest.split(" -> ", 1)
257
            paths.extend([old.strip('"'), new.strip('"')])
258
        else:
259
            paths.append(rest.strip('"'))
260
    return paths
261
262
263
def committed_bytes(ctx: Ctx, repo: Path, paths: Sequence[str]) -> Tuple[Dict[str, bytes], str, Dict[str, Any]]:
264
    """Committed HEAD bytes of ``paths`` read WITHOUT the subject binary.
265
266
    git: ``git cat-file`` with the oracle's own git. oak: the independent
267
    stdlib reader of Oak's SQLite store (oakbench.oak_sqlite_reader). Returns
268
    (payloads, reader label, meta). A reader bound (Unavailable) puts
269
    ``reader_unavailable`` in meta; Ctx.oracle then records oracle_ok None
270
    (unmeasured, ADR-0002) instead of a guess.
271
    """
272
    if _is_git(ctx):
273
        out: Dict[str, bytes] = {}
274
        for rel in paths:
275
            proc = ctx.git(["cat-file", "blob", f"HEAD:{rel}"], repo, check=False)
276
            if proc.returncode == 0:
277
                out[rel] = proc.stdout
278
        return out, "git_cat_file_oracle_instrument", {}
279
    from oakbench import oak_sqlite_reader as reader
280
    try:
281
        payloads, meta = reader.committed_payloads(Path(repo), list(paths))
282
    except reader.Unavailable as exc:
283
        # A harness/reader bound, not a property of Oak's data: the oracle
284
        # becomes unmeasured (Ctx.oracle turns this into oracle_ok None).
285
        return {}, "oak_sqlite_reader_independent", {"reader_unavailable": str(exc)[:300]}
286
    except reader.Invalid as exc:
287
        return {}, "oak_sqlite_reader_independent", {"invalid": str(exc)[:300]}
288
    return payloads, "oak_sqlite_reader_independent", {"head": meta.get("head"),
289
                                                       "reader_version": meta.get("reader_version")}
290
291
292
# ---------------------------------------------------------------------------
293
# Dirty shape (untimed setup)
294
# ---------------------------------------------------------------------------
295
296
297
def apply_dirty(ctx: Ctx) -> Dict[str, Any]:
298
    spec = ctx.p.tier.spec
299
    shape = ctx.p.tier.dirty
300
    cache = gen.layout(spec)
301
    factory = gen.ContentFactory(spec)
302
    protected = set(ctx.p.expected.protected)
303
    rng = random.Random(f"{spec.seed}:dirty:{ctx.tag}")
304
    deleted: set = ctx.state["deleted"]
305
    wanted = shape.modified + shape.deleted + shape.renamed
306
    pool: List[int] = []
307
    attempts = 0
308
    seen = set()
309
    while len(pool) < wanted and attempts < wanted * 20 + 100:
310
        attempts += 1
311
        index = rng.randrange(spec.files)
312
        path = gen.file_path(spec, index, cache)
313
        if index in seen or path in protected or path in deleted:
314
            continue
315
        seen.add(index)
316
        pool.append(index)
317
    modified = pool[:shape.modified]
318
    removed = pool[shape.modified:shape.modified + shape.deleted]
319
    renamed = pool[shape.modified + shape.deleted:]
320
    root = ctx.work
321
    mod_paths, del_paths, ren_pairs, add_paths = [], [], [], []
322
    for index in modified:
323
        path = gen.file_path(spec, index, cache)
324
        (root / path).write_bytes(factory.text(index, f"dirty-{ctx.tag}"))
325
        mod_paths.append(path)
326
    for index in removed:
327
        path = gen.file_path(spec, index, cache)
328
        (root / path).unlink()
329
        del_paths.append(path)
330
    for index in renamed:
331
        path = gen.file_path(spec, index, cache)
332
        target = f"renamed/{ctx.tag}/f{index}.txt"
333
        (root / target).parent.mkdir(parents=True, exist_ok=True)
334
        os.replace(root / path, root / target)
335
        ren_pairs.append((path, target))
336
    for number in range(shape.added):
337
        rel = f"added/{ctx.tag}/n{number}.txt"
338
        (root / rel).parent.mkdir(parents=True, exist_ok=True)
339
        (root / rel).write_bytes(factory.text(10_000_000 + number, f"add-{ctx.tag}"))
340
        add_paths.append(rel)
341
    dirty = {"modified": mod_paths, "deleted": del_paths, "renamed": ren_pairs, "added": add_paths,
342
             "requested": [shape.modified, shape.added, shape.deleted, shape.renamed],
343
             "applied": [len(mod_paths), len(add_paths), len(del_paths), len(ren_pairs)]}
344
    ctx.state["dirty"] = dirty
345
    return dirty
346
347
348
# ---------------------------------------------------------------------------
349
# Operations. Each returns None; results land in ctx.outcome.
350
# ---------------------------------------------------------------------------
351
352
353
def _is_git(ctx: Ctx) -> bool:
354
    return ctx.kind == "git"
355
356
357
def op_status_clean(ctx: Ctx) -> None:
358
    args = ["status", "--porcelain=v1"] if _is_git(ctx) else ["status", "--porcelain"]
359
    cap = ctx.timed(args, ctx.work)
360
    paths = porcelain_paths(cap.stdout_full_text or "")
361
    ctx.oracle(not paths, "clean_status_reports_no_paths", "subject_output", reported_paths=paths[:10])
362
363
364
def _dirty_branch_enter(ctx: Ctx) -> None:
365
    """The dirty cycle runs on a throwaway branch (untimed switch), so the base
366
    branch never accumulates harness commits: every run's switch/log/merge ops
367
    see the same repository state (no run-over-run drift)."""
368
    name = f"dirty-{ctx.tag}"
369
    if _is_git(ctx):
370
        ctx.git(["switch", "-q", "-c", name, "main"], ctx.work)
371
    else:
372
        ctx.untimed(["switch", "-c", name], ctx.work)
373
    ctx.state["dirty_branch"] = name
374
375
376
def _dirty_branch_leave(ctx: Ctx) -> None:
377
    name = ctx.state.pop("dirty_branch", None)
378
    if name is None:
379
        return
380
    if _is_git(ctx):
381
        ctx.git(["switch", "-q", "-f", "main"], ctx.work)
382
        ctx.git(["branch", "-q", "-D", name], ctx.work, check=False)
383
    else:
384
        ctx.untimed(["switch", _base_branch(ctx)], ctx.work)
385
386
387
def recover(ctx: Ctx) -> Optional[str]:
388
    """Untimed repair after a failed unit: discard worktree changes, return to base."""
389
    try:
390
        ctx.state.pop("dirty", None)
391
        ctx.state.pop("dirty_branch", None)
392
        if _is_git(ctx):
393
            ctx.git(["merge", "--abort"], ctx.work, check=False)
394
            ctx.git(["reset", "-q", "--hard"], ctx.work, check=False)
395
            ctx.git(["clean", "-fdq", "-e", "build/"], ctx.work, check=False)
396
            ctx.git(["switch", "-q", "-f", "main"], ctx.work)
397
        else:
398
            ctx.untimed(["reset", "-f"], ctx.work, check=False)
399
            for extra in ("added", "renamed"):
400
                shutil.rmtree(ctx.work / extra, ignore_errors=True)
401
            ctx.untimed(["switch", _base_branch(ctx)], ctx.work)
402
        return None
403
    except Exception as exc:  # noqa: BLE001 - reported on the row, never hidden
404
        return f"{type(exc).__name__}: {str(exc)[-300:]}"
405
406
407
def op_status_dirty(ctx: Ctx) -> None:
408
    _dirty_branch_enter(ctx)
409
    dirty = apply_dirty(ctx)
410
    ctx.outcome.extra["dirty_applied"] = dirty["applied"]
411
    args = ["status", "--porcelain=v1"] if _is_git(ctx) else ["status", "--porcelain"]
412
    cap = ctx.timed(args, ctx.work)
413
    text = cap.stdout_full_text or ""
414
    reported = set(porcelain_paths(text))
415
    must = set(dirty["modified"]) | set(dirty["deleted"]) | {old for old, _ in dirty["renamed"]}
416
    missing = sorted(must - reported)
417
    # Git collapses untracked directories ("?? added/<tag>/"); accept a
418
    # reported ancestor directory for added/renamed targets.
419
    new_paths = dirty["added"] + [new for _, new in dirty["renamed"]]
420
    uncovered = [p for p in new_paths if p not in reported and not any(
421
        p.startswith(r.rstrip("/") + "/") for r in reported if r.endswith("/") or r in ("added", "renamed"))]
422
    ctx.oracle(not missing and not uncovered, "dirty_paths_reported", "subject_output",
423
               missing=missing[:10], missing_count=len(missing), uncovered_new=uncovered[:10],
424
               uncovered_new_count=len(uncovered), stdout_bytes=cap.stdout_bytes)
425
426
427
def op_diff_dirty(ctx: Ctx) -> None:
428
    dirty = ctx.state.get("dirty")
429
    if not dirty:
430
        raise OpFailed("diff.dirty needs status.dirty's dirty shape")
431
    args = ["diff", "--no-ext-diff"] if _is_git(ctx) else ["diff", "--print"]
432
    cap = ctx.timed(args, ctx.work)
433
    text = cap.stdout_full_text
434
    if text is None:
435
        ctx.oracle(cap.stdout_bytes > 0, "diff_output_nonempty_over_8MiB", "subject_output",
436
                   stdout_bytes=cap.stdout_bytes)
437
        return
438
    must = dirty["modified"] + dirty["deleted"]
439
    missing = [p for p in must if p not in text]
440
    ctx.oracle(not missing, "diff_mentions_modified_and_deleted_paths", "subject_output",
441
               missing=missing[:10], missing_count=len(missing), stdout_bytes=cap.stdout_bytes)
442
443
444
def op_commit_dirty(ctx: Ctx) -> None:
445
    dirty = ctx.state.pop("dirty", None)
446
    if not dirty:
447
        raise OpFailed("commit.dirty needs status.dirty's dirty shape")
448
    if _is_git(ctx):
449
        ctx.timed(["add", "-A"], ctx.work, label="add")
450
        ctx.timed(["commit", "-q", "--no-verify", "-m", f"dirty {ctx.tag}"], ctx.work, label="commit")
451
    else:
452
        ctx.timed(["commit", "--json"], ctx.work)
453
    probe = dirty["modified"][0] if dirty["modified"] else None
454
    gone = dirty["deleted"][0] if dirty["deleted"] else None
455
    status_args = ["status", "--porcelain=v1"] if _is_git(ctx) else ["status", "--porcelain"]
456
    left = porcelain_paths(ctx.untimed(status_args, ctx.work).stdout.decode(errors="replace"))
457
    wanted = [p for p in (probe, gone) if p]
458
    payloads, reader, meta = committed_bytes(ctx, ctx.work, wanted)
459
    committed_ok = probe is None or payloads.get(probe) == (ctx.work / probe).read_bytes()
460
    deleted_ok = gone is None or gone not in payloads
461
    ctx.oracle(not left and committed_ok and deleted_ok and "invalid" not in meta and "reader_unavailable" not in meta,
462
               "tree_clean_modified_bytes_committed_deleted_path_absent", reader,
463
               left_after_commit=left[:10], committed_bytes_match=committed_ok, deleted_absent=deleted_ok, **meta)
464
    _dirty_branch_leave(ctx)
465
466
467
def _log_count(text: str) -> int:
468
    return sum(1 for line in text.splitlines() if line.strip() and not line.startswith("No matching commits"))
469
470
471
def op_log_full(ctx: Ctx) -> None:
472
    args = ["log", "--oneline"] if _is_git(ctx) else ["log", "--oneline", "-n", "100000000"]
473
    cap = ctx.timed(args, ctx.work)
474
    count = _log_count(cap.stdout_full_text) if cap.stdout_full_text is not None else None
475
    expected = ctx.p.expected.main_commits + ctx.state["harness_commits"]
476
    ctx.oracle(count == expected, "log_line_count_equals_history_length", "subject_output",
477
               lines=count, expected=expected, stdout_bytes=cap.stdout_bytes,
478
               output_over_capture_cap=count is None)
479
480
481
def op_log_n(ctx: Ctx) -> None:
482
    cap = ctx.timed(["log", "--oneline", "-n", "20"], ctx.work)
483
    count = _log_count(cap.stdout_full_text or "")
484
    expected = min(20, ctx.p.expected.main_commits + ctx.state["harness_commits"])
485
    ctx.oracle(count == expected, "log_n_line_count", "subject_output", lines=count, expected=expected)
486
487
488
def op_log_path(ctx: Ctx) -> None:
489
    probe = ctx.p.expected.probe_path
490
    args = ["log", "--oneline", "--", probe] if _is_git(ctx) else ["log", "--oneline", "-n", "100000000", probe]
491
    cap = ctx.timed(args, ctx.work)
492
    count = _log_count(cap.stdout_full_text or "")
493
    expected = ctx.p.expected.probe_commit_count
494
    ctx.oracle(count == expected, "path_log_count_equals_generator_touch_count", "subject_output",
495
               lines=count, expected=expected, probe_path=probe)
496
497
498
def op_blame(ctx: Ctx) -> None:
499
    if not _is_git(ctx):
500
        raise OpSkipped("oak_cli_has_no_blame_or_annotate_command")
501
    probe = ctx.p.expected.probe_path
502
    cap = ctx.timed(["blame", "-s", "--", probe], ctx.work)
503
    lines = _log_count(cap.stdout_full_text or "")
504
    ctx.oracle(lines == ctx.p.expected.probe_lines, "blame_line_count_equals_file_lines", "subject_output",
505
               lines=lines, expected=ctx.p.expected.probe_lines)
506
507
508
def op_file_show(ctx: Ctx) -> None:
509
    probe = ctx.p.expected.probe_path
510
    if _is_git(ctx):
511
        cap = ctx.timed(["show", f"HEAD:{probe}"], ctx.work, raw=True)
512
        got = cap.stdout_raw or b""
513
        reader = "subject_output"
514
    else:
515
        out = ctx.scratch / f"show-{ctx.tag}.bin"
516
        ctx.timed(["file", "inspect", "--at", "HEAD", "--json", probe, "--output", str(out)], ctx.work)
517
        got = out.read_bytes() if out.exists() else b""
518
        if out.exists():
519
            out.unlink()
520
        reader = "subject_file_inspect"
521
    ctx.oracle(got == ctx.p.expected.probe_bytes, "shown_bytes_equal_fixture_blob", reader,
522
               bytes=len(got), expected_bytes=len(ctx.p.expected.probe_bytes))
523
524
525
def op_branch_list(ctx: Ctx) -> None:
526
    if _is_git(ctx):
527
        cap = ctx.timed(["branch", "--list"], ctx.work)
528
        names = [line[2:].strip() for line in (cap.stdout_full_text or "").splitlines() if line.strip()]
529
        reader = "subject_output"
530
    else:
531
        cap = ctx.timed(["branch", "list", "--json"], ctx.work)
532
        try:
533
            names = [item.get("name") for item in json.loads(cap.stdout_full_text or "[]")]
534
        except ValueError:
535
            names = []
536
        reader = "subject_output_json"
537
    spec = ctx.p.tier.spec
538
    fixture_branches = spec.branches if _is_git(ctx) else ctx.p.oak_branches_created
539
    need = [gen.branch_name(0)] if fixture_branches else []
540
    ok = len(names) >= fixture_branches and all(n in names for n in need)
541
    ctx.oracle(ok, "branch_count_at_least_fixture_branches", reader, listed=len(names),
542
               fixture_branches=fixture_branches)
543
544
545
def _branch_paths_match(ctx: Ctx, root: Path, want: Dict[str, bytes]) -> Tuple[bool, List[str]]:
546
    bad = [rel for rel, data in want.items() if not (root / rel).is_file() or (root / rel).read_bytes() != data]
547
    return not bad, bad
548
549
550
def op_switch_existing(ctx: Ctx) -> None:
551
    exp = ctx.p.expected
552
    args = ["switch", "-q", exp.branch] if _is_git(ctx) else ["switch", exp.branch]
553
    ctx.timed(args, ctx.work)
554
    ok, bad = _branch_paths_match(ctx, ctx.work, exp.branch_bytes)
555
    ctx.oracle(ok, "branch_paths_bytes_equal_fixture", "git_fixture_instrument", mismatched=bad[:10])
556
557
558
def _base_branch(ctx: Ctx) -> str:
559
    return "main" if _is_git(ctx) else str(ctx.p.oak_base_branch)
560
561
562
def op_switch_back(ctx: Ctx) -> None:
563
    exp = ctx.p.expected
564
    args = ["switch", "-q", _base_branch(ctx)] if _is_git(ctx) else ["switch", _base_branch(ctx)]
565
    ctx.timed(args, ctx.work)
566
    ok, bad = _branch_paths_match(ctx, ctx.work, exp.main_bytes_of_branch_paths)
567
    ctx.oracle(ok, "base_paths_bytes_equal_fixture_main", "git_fixture_instrument", mismatched=bad[:10])
568
569
570
def current_branch(ctx: Ctx, root: Path) -> str:
571
    if _is_git(ctx):
572
        return ctx.git(["symbolic-ref", "--short", "HEAD"], root, check=False).stdout.decode().strip()
573
    out = ctx.untimed(["status", "--json"], root, check=False).stdout
574
    try:
575
        return str(json.loads(out).get("branch"))
576
    except ValueError:
577
        return ""
578
579
580
def op_branch_create(ctx: Ctx) -> None:
581
    ctx.state["branch_seq"] += 1
582
    name = f"new-{ctx.tag}-{ctx.state['branch_seq']}"
583
    args = ["switch", "-q", "-c", name] if _is_git(ctx) else ["switch", "-c", name]
584
    ctx.timed(args, ctx.work)
585
    now = current_branch(ctx, ctx.work)
586
    ctx.outcome.semantic_class = "git_branch_from_HEAD" if _is_git(ctx) else "oak_branch_from_main"
587
    # teardown: back to base (and drop the git branch so ref counts stay fixed)
588
    if _is_git(ctx):
589
        ctx.git(["switch", "-q", "main"], ctx.work)
590
        ctx.git(["branch", "-q", "-D", name], ctx.work)
591
    else:
592
        ctx.untimed(["switch", _base_branch(ctx)], ctx.work)
593
    ctx.oracle(now == name, "current_branch_is_new_branch", "subject_output", current=now, wanted=name)
594
595
596
def op_review_branch_diff(ctx: Ctx) -> None:
597
    exp = ctx.p.expected
598
    if _is_git(ctx):
599
        cap = ctx.timed(["diff", "--name-status", f"main...{exp.branch}"], ctx.work)
600
        paths = [line.split("\t", 1)[-1] for line in (cap.stdout_full_text or "").splitlines() if "\t" in line]
601
        reader = "subject_output"
602
    else:
603
        cap = ctx.timed(["branch", "diff", exp.branch, "--json"], ctx.work)
604
        try:
605
            paths = [item["path"] for item in json.loads(cap.stdout_full_text or "{}").get("changed_files", [])]
606
        except (ValueError, KeyError, AttributeError):
607
            paths = []
608
        reader = "subject_output_json"
609
    ctx.oracle(sorted(paths) == sorted(exp.branch_paths), "changed_paths_equal_generator_branch_paths", reader,
610
               reported=sorted(paths)[:10], expected=sorted(exp.branch_paths)[:10])
611
612
613
def op_merge_preview(ctx: Ctx) -> None:
614
    exp = ctx.p.expected
615
    if _is_git(ctx):
616
        cap = ctx.timed(["merge-tree", "--write-tree", "main", exp.branch], ctx.work)
617
        tree = (cap.stdout_full_text or "").split("\n", 1)[0].strip()
618
        bad = []
619
        for rel, data in exp.branch_bytes.items():
620
            got = ctx.git(["cat-file", "blob", f"{tree}:{rel}"], ctx.work, check=False).stdout
621
            if got != data:
622
                bad.append(rel)
623
        ctx.oracle(bool(tree) and not bad, "clean_prediction_tree_has_branch_bytes", "git_fixture_instrument",
624
                   mismatched=bad[:10])
625
        ctx.outcome.semantic_class = "git_merge_tree_two_refs"
626
        return
627
    ctx.untimed(["switch", exp.branch], ctx.work)
628
    try:
629
        # Exit 7 = prediction computed but not certified against the remote
630
        # target head (documented exit code); the prediction itself is timed.
631
        cap = ctx.timed(["merge", "--dry-run", "--json"], ctx.work, accept=(0, 7))
632
    finally:
633
        ctx.untimed(["switch", _base_branch(ctx)], ctx.work)
634
    try:
635
        data = json.loads((cap.stdout_full_text or "").splitlines()[0])
636
    except (ValueError, IndexError):
637
        data = {}
638
    changed = sorted(item.get("path") if isinstance(item, dict) else item for item in data.get("changed_files", []))
639
    ctx.outcome.semantic_class = "oak_merge_dry_run_current_into_main"
640
    ctx.outcome.extra["merge_certified"] = (data.get("merge_safety") or {}).get("certified")
641
    ctx.oracle(data.get("clean") is True and changed == sorted(exp.branch_paths),
642
               "clean_prediction_lists_branch_paths", "subject_output_json", clean=data.get("clean"),
643
               changed=changed[:10], expected=sorted(exp.branch_paths)[:10])
644
645
646
def op_merge_preview_conflict(ctx: Ctx) -> None:
647
    conflict = ctx.p.expected.conflict
648
    if not conflict:
649
        raise OpSkipped("fixture_has_no_conflict_shape")
650
    if not _is_git(ctx):
651
        raise OpSkipped("oak_merge_dry_run_predicts_current_branch_into_main_only")
652
    cap = ctx.timed(["merge-tree", "--write-tree", "--name-only", conflict["ours"], conflict["theirs"]],
653
                    ctx.work, accept=(1,))
654
    lines = (cap.stdout_full_text or "").split("\n\n", 1)[0].splitlines()[1:]
655
    ctx.oracle(len(lines) == conflict["paths"], "conflicted_path_count_equals_fixture", "subject_output",
656
               conflicted=len(lines), expected=conflict["paths"])
657
658
659
def op_merge_local(ctx: Ctx) -> None:
660
    if not _is_git(ctx):
661
        raise OpSkipped("oak_local_merge_needs_non_main_parent_unreachable_from_cli; "
662
                        "oak_serve_lacks_merge_to_main")
663
    exp = ctx.p.expected
664
    scratch = f"merge-{ctx.tag}"
665
    ctx.git(["switch", "-q", "-c", scratch, "main"], ctx.work)
666
    try:
667
        ctx.timed(["merge", "-q", "--no-ff", "--no-edit", exp.branch], ctx.work)
668
        parents = ctx.git(["rev-list", "--parents", "-n", "1", "HEAD"], ctx.work).stdout.split()
669
        ok, bad = _branch_paths_match(ctx, ctx.work, exp.branch_bytes)
670
        ctx.oracle(ok and len(parents) == 3, "merge_commit_two_parents_and_branch_bytes", "git_fixture_instrument",
671
                   parents=len(parents) - 1, mismatched=bad[:10])
672
    finally:
673
        ctx.git(["merge", "--abort"], ctx.work, check=False)
674
        ctx.git(["switch", "-q", "-f", "main"], ctx.work)
675
        ctx.git(["branch", "-q", "-D", scratch], ctx.work, check=False)
676
677
678
def op_merge_conflict(ctx: Ctx) -> None:
679
    conflict = ctx.p.expected.conflict
680
    if not conflict:
681
        raise OpSkipped("fixture_has_no_conflict_shape")
682
    if not _is_git(ctx):
683
        raise OpSkipped("oak_local_merge_needs_non_main_parent_unreachable_from_cli")
684
    scratch = f"conflict-{ctx.tag}"
685
    ctx.git(["switch", "-q", "-c", scratch, conflict["ours"]], ctx.work)
686
    try:
687
        ctx.timed(["merge", "-q", "--no-edit", conflict["theirs"]], ctx.work, accept=(1,))
688
        unmerged = ctx.git(["diff", "--name-only", "--diff-filter=U"], ctx.work).stdout.decode().split()
689
        ctx.oracle(len(unmerged) == conflict["paths"], "unmerged_path_count_equals_fixture", "subject_output",
690
                   unmerged=len(unmerged), expected=conflict["paths"])
691
    finally:
692
        ctx.git(["merge", "--abort"], ctx.work, check=False)
693
        ctx.git(["switch", "-q", "-f", "main"], ctx.work)
694
        ctx.git(["branch", "-q", "-D", scratch], ctx.work, check=False)
695
696
697
def _client_commit(ctx: Ctx, client: Path, rel: str, content: bytes) -> str:
698
    target = client / rel
699
    target.parent.mkdir(parents=True, exist_ok=True)
700
    target.write_bytes(content)
701
    if _is_git(ctx):
702
        ctx.git(["add", "-A"], client)
703
        ctx.git(["commit", "-q", "--no-verify", "-m", f"sync {rel}"], client)
704
        return ctx.git(["rev-parse", "HEAD"], client).stdout.decode().strip()
705
    ctx.untimed(["commit", "--json"], client)
706
    return json.loads(ctx.untimed(["status", "--json"], client).stdout)["head"]
707
708
709
def _head(ctx: Ctx, client: Path) -> str:
710
    if _is_git(ctx):
711
        return ctx.git(["rev-parse", "HEAD"], client).stdout.decode().strip()
712
    return json.loads(ctx.untimed(["status", "--json"], client).stdout)["head"]
713
714
715
def _oak_remote(ctx: Ctx) -> None:
716
    """Sync ops need seeded loopback servers; label an Oak snapshot seed."""
717
    if not getattr(ctx.p.tier, "sync", True):
718
        raise OpSkipped("tier_excludes_sync_clients_disk_budget")
719
    if _is_git(ctx):
720
        return
721
    if getattr(ctx.p, "oak_remote_unavailable", None):
722
        raise OpSkipped(str(ctx.p.oak_remote_unavailable))
723
    seed = getattr(ctx.p, "oak_seed", None)
724
    ctx.outcome.extra["oak_server_seed"] = seed
725
    if seed and seed != "full_history":
726
        ctx.outcome.semantic_class = f"oak_server_seed_{seed}"
727
728
729
def op_push_branch(ctx: Ctx) -> None:
730
    _oak_remote(ctx)
731
    client_b = ctx.client("clientB")
732
    rel = f"sync/push-{ctx.tag}.txt"
733
    head = _client_commit(ctx, client_b, rel, f"push {ctx.tag}\n".encode())
734
    ctx.state["pushed"] = (rel, head)
735
    if _is_git(ctx):
736
        ctx.timed(["push", "-q", "origin", "HEAD:main"], client_b)
737
        remote = ctx.git(["ls-remote", "origin", "refs/heads/main"], client_b).stdout.decode().split()
738
        ctx.state["server_extra"] += 1
739
        ctx.oracle(bool(remote) and remote[0] == head, "server_ref_equals_pushed_head", "git_ls_remote",
740
                   remote=remote[:1], pushed=head)
741
    else:
742
        cap = ctx.timed(["push", "--json"], client_b)
743
        try:
744
            receipt = json.loads((cap.stdout_full_text or "").splitlines()[-1])
745
        except (ValueError, IndexError):
746
            receipt = {}
747
        # Server-side confirmation comes from clientA's pull.ff oracle in the
748
        # same unit (independent checkout sees the pushed commit and bytes).
749
        ctx.outcome.extra["push_receipt_pushed"] = receipt.get("pushed")
750
        ctx.oracle(receipt.get("remote_contacted") is True, "push_receipt_remote_contacted_confirmed_by_pull_ff",
751
                   "subject_output_json", pushed=receipt.get("pushed"))
752
753
754
def op_pull_ff(ctx: Ctx) -> None:
755
    _oak_remote(ctx)
756
    pushed = ctx.state.pop("pushed", None)
757
    if not pushed:
758
        raise OpFailed("pull.ff needs push.branch in the same unit")
759
    rel, head = pushed
760
    client_a = ctx.client("clientA")
761
    if _is_git(ctx):
762
        ctx.timed(["pull", "-q", "--ff-only"], client_a)
763
    else:
764
        ctx.timed(["pull", "--json"], client_a)
765
    now = _head(ctx, client_a)
766
    want = f"push {ctx.tag}\n".encode()
767
    data_ok = (client_a / rel).is_file() and (client_a / rel).read_bytes() == want
768
    payloads, reader, meta = committed_bytes(ctx, client_a, [rel])
769
    committed_ok = payloads.get(rel) == want
770
    ctx.oracle(now == head and data_ok and committed_ok, "puller_head_worktree_and_committed_bytes_equal_pusher",
771
               reader, puller_head=now, pusher_head=head, bytes_ok=data_ok, committed_ok=committed_ok, **meta)
772
773
774
def op_pull_diverged(ctx: Ctx) -> None:
775
    _oak_remote(ctx)
776
    client_a, client_b = ctx.client("clientA"), ctx.client("clientB")
777
    local_rel, up_rel = f"sync/local-{ctx.tag}.txt", f"sync/up-{ctx.tag}.txt"
778
    _client_commit(ctx, client_a, local_rel, b"local\n")
779
    _client_commit(ctx, client_b, up_rel, b"upstream\n")
780
    if _is_git(ctx):
781
        ctx.git(["push", "-q", "origin", "HEAD:main"], client_b)
782
        ctx.state["server_extra"] += 1
783
        ctx.timed(["pull", "-q", "--no-rebase", "--no-edit"], client_a)
784
        ctx.outcome.semantic_class = "git_merge_commit"
785
    else:
786
        ctx.untimed(["push", "--json"], client_b)
787
        cap = ctx.timed(["pull", "--json"], client_a)
788
        try:
789
            receipt = json.loads((cap.stdout_full_text or "").splitlines()[-1])
790
        except (ValueError, IndexError):
791
            receipt = {}
792
        ctx.outcome.semantic_class = (ctx.outcome.semantic_class or "") + f"|oak_pull_{receipt.get('convergence', 'unknown')}"
793
    both = (client_a / local_rel).is_file() and (client_a / up_rel).is_file()
794
    status_args = ["status", "--porcelain=v1"] if _is_git(ctx) else ["status", "--porcelain"]
795
    left = porcelain_paths(ctx.untimed(status_args, client_a).stdout.decode(errors="replace"))
796
    payloads, reader, meta = committed_bytes(ctx, client_a, [local_rel, up_rel])
797
    committed_ok = payloads.get(local_rel) == b"local\n" and payloads.get(up_rel) == b"upstream\n"
798
    ctx.oracle(both and not left and committed_ok, "both_sides_committed_present_and_tree_clean", reader,
799
               local_present=(client_a / local_rel).is_file(), upstream_present=(client_a / up_rel).is_file(),
800
               committed_ok=committed_ok, left=left[:10], **meta)
801
    # Re-converge (untimed): A publishes its merge, B catches up. A failure
802
    # here is an observed subject defect in an UNTIMED step: it is recorded on
803
    # the row (teardown_error) without discarding the timed sample.
804
    if _is_git(ctx):
805
        ctx.git(["push", "-q", "origin", "HEAD:main"], client_a)
806
        ctx.state["server_extra"] += 1
807
        ctx.git(["pull", "-q", "--ff-only"], client_b)
808
        return
809
    ctx.untimed(["push", "--json"], client_a)
810
    done = ctx.untimed(["pull", "--json"], client_b, check=False)
811
    if done.returncode != 0:
812
        try:
813
            receipt = json.loads(done.stdout.decode(errors="replace").splitlines()[-1])
814
        except (ValueError, IndexError):
815
            receipt = {}
816
        ctx.outcome.extra["teardown_error"] = (
817
            f"peer pull after reparent exited {done.returncode}: branch_update={receipt.get('branch_update')} "
818
            f"parent_sync_error_code={receipt.get('parent_sync_error_code')}")
819
        if _head(ctx, client_b) != _head(ctx, client_a):
820
            raise OpFailed("peer checkout did not converge after reparent: " + ctx.outcome.extra["teardown_error"])
821
822
823
def op_clone_full(ctx: Ctx) -> None:
824
    _oak_remote(ctx)
825
    dest = ctx.scratch / f"clone-{ctx.tag}"
826
    if dest.exists():
827
        shutil.rmtree(dest)
828
    try:
829
        if _is_git(ctx):
830
            ctx.timed(["clone", "-q", str(ctx.p.git_url), str(dest)], ctx.scratch)
831
            # server main = fixture + files pushed to main by the sync ops
832
            expected = ctx.p.expected.main_file_count + ctx.state["server_extra"]
833
            extra_prefix: Tuple[str, ...] = ()
834
        else:
835
            ctx.timed(["clone", "-r", str(ctx.p.oak_url), "bench/fx", str(dest)], ctx.scratch)
836
            expected = ctx.p.expected.main_file_count  # oak sync ops publish to a branch, not main
837
            # (a snapshot seed has the same main tree, one commit of history)
838
            extra_prefix = ()
839
        ok, detail = check_worktree(ctx, dest, expected, extra_prefix)
840
        ctx.oracle(ok, "clone_file_count_and_sampled_blob_ids_equal_fixture", "git_fixture_instrument", **detail)
841
    finally:
842
        shutil.rmtree(dest, ignore_errors=True)
843
844
845
def op_gc(ctx: Ctx) -> None:
846
    status_args = ["status", "--porcelain=v1"] if _is_git(ctx) else ["status", "--porcelain"]
847
    if _is_git(ctx):
848
        before = ctx.git(["rev-list", "--count", "HEAD"], ctx.work).stdout.strip()
849
        ctx.timed(["gc", "--quiet"], ctx.work)
850
        after = ctx.git(["rev-list", "--count", "HEAD"], ctx.work).stdout.strip()
851
        ctx.outcome.semantic_class = "git_gc"
852
    else:
853
        before = ctx.untimed(["hash"], ctx.work).stdout.strip()
854
        ctx.timed(["maintenance", "compact"], ctx.work)
855
        after = ctx.untimed(["hash"], ctx.work).stdout.strip()
856
        ctx.outcome.semantic_class = "oak_maintenance_compact"
857
    left = porcelain_paths(ctx.untimed(status_args, ctx.work).stdout.decode(errors="replace"))
858
    probe = ctx.p.expected.probe_path
859
    payloads, reader, meta = committed_bytes(ctx, ctx.work, [probe])
860
    probe_ok = payloads.get(probe) == ctx.p.expected.probe_bytes
861
    ctx.oracle(before == after and not left and probe_ok, "head_status_and_committed_probe_unchanged",
862
               reader, before=before.decode() if isinstance(before, bytes) else before,
863
               after=after.decode() if isinstance(after, bytes) else after, left=left[:10],
864
               committed_probe_ok=probe_ok, **meta)
865
866
867
OPS: Dict[str, Callable[[Ctx], None]] = {
868
    "status.clean": op_status_clean,
869
    "status.dirty": op_status_dirty,
870
    "diff.dirty": op_diff_dirty,
871
    "commit.dirty": op_commit_dirty,
872
    "log.full": op_log_full,
873
    "log.n": op_log_n,
874
    "log.path": op_log_path,
875
    "blame": op_blame,
876
    "file.show": op_file_show,
877
    "branch.list": op_branch_list,
878
    "branch.create": op_branch_create,
879
    "switch.existing": op_switch_existing,
880
    "switch.back": op_switch_back,
881
    "review.branch_diff": op_review_branch_diff,
882
    "merge.preview": op_merge_preview,
883
    "merge.preview.conflict": op_merge_preview_conflict,
884
    "merge.local": op_merge_local,
885
    "merge.conflict": op_merge_conflict,
886
    "push.branch": op_push_branch,
887
    "pull.ff": op_pull_ff,
888
    "pull.diverged": op_pull_diverged,
889
    "clone.full": op_clone_full,
890
    "gc": op_gc,
891
}
892
assert set(OPS) == set(OP_INFO)
893
894
895
def catalog_yaml() -> str:
896
    """scenarios/scale.yaml, generated from this catalog (a test keeps them equal)."""
897
    lines = [
898
        "schema_version: 1",
899
        "scenario_set: scale",
900
        "description: Scale and coverage lane (suites/scale). Every user-facing VCS operation timed for git and oak"
901
        " across repository tiers. Generated from suites/scale/scalesuite/ops.py (catalog_yaml); "
902
        "tests/test_scale_suite.py keeps the two equal.",
903
        "",
904
        "operation_catalog:",
905
    ]
906
    for unit in UNIT_ORDER:
907
        for op in unit:
908
            info = OP_INFO[op]
909
            lines += [f"  {op}:", f"    description: {info.description}", "    timed: true",
910
                      f"    mutates: {str(info.mutates).lower()}"]
911
            for key, values in (("tperf_refs", info.tperf), ("core_yaml_ops", info.core_yaml)):
912
                if values:
913
                    lines.append(f"    {key}:")
914
                    lines.extend(f"      - {value}" for value in values)
915
                else:
916
                    lines.append(f"    {key}: []")
917
    return "\n".join(lines) + "\n"
918
919
920
def execute(ctx: Ctx, op: str) -> Outcome:
921
    try:
922
        OPS[op](ctx)
923
    except OpSkipped as exc:
924
        ctx.outcome.skip_reason = str(exc)
925
    except OpFailed as exc:
926
        ctx.outcome.failed = str(exc)
927
    except Exception as exc:  # noqa: BLE001 - harness errors are recorded, never hidden
928
        ctx.outcome.failed = f"harness_error: {type(exc).__name__}: {str(exc)[-800:]}"
929
        ctx.outcome.extra["harness_error"] = True
930
    return ctx.outcome