Viewing files at commit 1a441edafc71Commit detailsView latest

suites/robustness/robustness/engine.py 396 lines · 18.4 KB · python
1
"""Trial engine: copy a pristine fixture, inject one fault, judge invariants.
2
3
Invariants (each True / False / None=unmeasured, ADR-0002):
4
5
- ``op_no_hang``            the faulted operation ended (killed or exited) within its timeout
6
- ``repo_opens``            independent open: SQLite integrity_check (Oak) / git fsck (Git)
7
- ``acked_local_reachable`` every commit a local ref named before the op is still reachable
8
                            from a local ref (branch head / HEAD / tag), per the independent reader
9
- ``server_intact``         remote store opens, its pre-op heads stay reachable, and the branch
10
                            head is either the pre-op head or the operation's intended head
11
- ``worktree_consistent``   the worktree equals the pre- or post-op state, or a committed tree,
12
                            or the subject's status reports it dirty/in-progress (never "clean"
13
                            while differing from HEAD)
14
- ``next_op_ok_or_actionable`` rerunning the operation succeeds, or fails with a typed/actionable
15
                            message (not a hang, panic, or signal)
16
- ``no_false_success``      a reported success (exit 0) leaves exactly the calibrated outcome
17
- ``acked_after_next``      pre-op local commits are still reachable after the next operation
18
"""
19
20
from __future__ import annotations
21
22
import json
23
import re
24
import shutil
25
import time
26
from dataclasses import dataclass, field
27
from pathlib import Path
28
from typing import Any, Callable, Optional
29
30
from . import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, VIOLATED
31
from .adapters import GitAdapter, OakAdapter, Server
32
from .fixtures import Fixture
33
from .procs import Result, copy_tree
34
from .readers import OakGraph
35
36
# A failure is actionable only if it tells the user what to do next:
37
# a concrete recovery command (a real subcommand of the subject, or a recovery
38
# flag), an explicit manual step, a documented retryable condition, or a typed,
39
# specific JSON error code / recommended command list. Mentioning "oak" or
40
# "git" is not enough ("failed to open oak database", "not a git repository").
41
RECOVERY_SUBCOMMANDS = (
42
    "pull", "push", "commit", "fetch", "clone", "switch", "checkout", "merge", "rebase", "reset", "restore",
43
    "stash", "add", "rm", "status", "init", "close", "am", "cherry-pick", "revert", "gc", "fsck", "prune",
44
    "doctor", "sparse", "conflict", "login", "finish", "desc")
45
RECOVERY_COMMAND_RE = re.compile(
46
    r"\b(?:oak|git)\s+(?:" + "|".join(re.escape(c) for c in RECOVERY_SUBCOMMANDS) + r")\b"
47
    r"|(?:^|\s)--(?:continue|abort|skip|force)\b", re.I)
48
RECOVERY_STEP_RE = re.compile(
49
    r"remove the file manually|remove this directory|move or remove|delete (?:it|this directory|the directory)"
50
    r"|commit your changes|stash them|\bpull (?:changes )?first\b|\((?:fetch first|non-fast-forward)\)", re.I)
51
RETRYABLE_RE = re.compile(r"\btry again\b|\bretry\b|\bre-?run\b|repository (?:is )?locked|lock(?:ed)? by (?:another|pid)",
52
                          re.I)
53
# Corruption / internal-failure wording: never actionable on retry wording or a generic code alone.
54
NON_ACTIONABLE_RE = re.compile(
55
    r"panicked at|RUST_BACKTRACE|not a database|malformed|corrupt|internal (?:server )?error|\binternal\b"
56
    r"|integrity check failed|unexpected (?:eof|end)", re.I)
57
GENERIC_ERROR_CODES = {"error", "internal", "internal_error", "unknown", "unknown_error", "io", "io_error", "other",
58
                       "panic", "database", "database_error", "sqlite", "unexpected", "server_error", "http_error"}
59
60
61
def _json_error(stdout: str) -> Optional[dict]:
62
    try:
63
        data = json.loads(stdout)
64
    except (ValueError, TypeError):
65
        return None
66
    if isinstance(data, dict) and isinstance(data.get("error"), dict):
67
        return data["error"]
68
    return None
69
70
71
def actionable(res: Result) -> bool:
72
    """True when a failed command tells the user a concrete next step."""
73
    text = res.text()
74
    if res.returncode is None or res.returncode <= 0:
75
        return False
76
    if "panicked at" in text or "RUST_BACKTRACE" in text:
77
        return False
78
    if RECOVERY_COMMAND_RE.search(text) or RECOVERY_STEP_RE.search(text):
79
        return True
80
    broken = bool(NON_ACTIONABLE_RE.search(text))
81
    error = _json_error(res.stdout)
82
    if error is not None and not broken:
83
        code = str(error.get("code") or "").strip().lower()
84
        if error.get("recommended_next_commands"):
85
            return True
86
        if code and code not in GENERIC_ERROR_CODES and not code.startswith(("internal", "unknown", "io_")):
87
            return True
88
    return not broken and bool(RETRYABLE_RE.search(text))
89
90
91
@dataclass
92
class Calibration:
93
    elapsed_ms: float
94
    post_tree: Optional[dict]
95
    post_worktree: Optional[dict]
96
    post_head: Optional[str]
97
    post_server_head: Optional[str]
98
    boundaries: int = 0
99
    boundary_kinds: list = field(default_factory=list)
100
    requests: list = field(default_factory=list)
101
    ok: bool = True
102
    detail: str = ""
103
104
105
class Trial:
106
    """One copied fixture + (for Oak) its loopback server, ready for one faulted op."""
107
108
    def __init__(self, fixture: Fixture, adapter_bin: str, work: Path, name: str):
109
        self.fixture = fixture
110
        self.dir = work / name
111
        copy_tree(fixture.pristine, self.dir)
112
        self.home = self.dir / "home"
113
        self.kind = fixture.kind
114
        if self.kind == "oak":
115
            self.adapter = OakAdapter(adapter_bin, self.home)
116
            self.server: Optional[Server] = Server(adapter_bin, self.dir / "srv", fixture.port or 0,
117
                                                   self.dir / "serve.log")
118
        else:
119
            self.adapter = GitAdapter(adapter_bin, self.home)
120
            self.server = None
121
        self.client = self.dir / "client"
122
123
    # op plumbing ---------------------------------------------------------------
124
    def op_cwd(self) -> Path:
125
        return self.dir if self.fixture.op == "clone" else self.client
126
127
    def op_argv(self, bin_override: Optional[str] = None, dest: str = "cloned", url: Optional[str] = None) -> list[str]:
128
        kw: dict[str, Any] = {"bin": bin_override}
129
        if self.fixture.op == "clone":
130
            kw.update(dest=dest, branch=self.fixture.branch, url=url or self.fixture.meta["url"])
131
        if self.fixture.op == "switch":
132
            kw["target"] = self.fixture.meta["target"]
133
        return self.adapter.argv(self.fixture.op, **kw)
134
135
    def pre(self) -> None:
136
        for argv in self.adapter.pre_argv(self.fixture.op):
137
            self.adapter.must(argv, self.client)
138
139
    def start_server(self, port: Optional[int] = None) -> None:
140
        if self.server is not None:
141
            self.server.start(port)
142
143
    def stop_server(self) -> None:
144
        if self.server is not None:
145
            self.server.stop()
146
147
    def settle(self) -> None:
148
        """Let orphaned server-side children (Git local receive-pack) finish."""
149
        if self.kind == "git":
150
            remote = self.dir / "srv" / "remote.git"
151
            deadline = time.monotonic() + 10
152
            while time.monotonic() < deadline and any(remote.rglob("*.lock")):
153
                time.sleep(0.05)
154
            time.sleep(0.1)
155
156
    def target_repo(self, dest: str = "cloned") -> Path:
157
        return self.dir / dest if self.fixture.op == "clone" else self.client
158
159
    def cleanup(self) -> None:
160
        self.stop_server()
161
        shutil.rmtree(self.dir, ignore_errors=True)
162
163
    # independent observations ---------------------------------------------------
164
    def server_head(self) -> Optional[str]:
165
        if self.kind == "oak":
166
            db = self.server.db() if self.server else None
167
            if not db or not db.exists():
168
                return None
169
            with OakGraph(db) as g:
170
                return g.branch_heads().get(self.fixture.branch)
171
        refs = self.adapter.oracle.refs(self.dir / "srv" / "remote.git")
172
        return refs.get(f"refs/heads/{self.fixture.branch}")
173
174
    def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
175
        if self.kind == "oak":
176
            from .readers import oak_commit_tree
177
            return oak_commit_tree(self.server.db(), commit)
178
        return self.adapter.oracle.commit_tree(self.dir / "srv" / "remote.git", commit)
179
180
    def repo_opens(self, repo: Path) -> Optional[bool]:
181
        if self.kind == "oak":
182
            db = repo / ".oak" / "oak.db"
183
            if not db.exists():
184
                return False
185
            try:
186
                with OakGraph(db) as g:
187
                    return g.integrity() == "ok" and g.metadata("current_branch") is not None
188
            except Exception:  # noqa: BLE001 - any failure to open is the finding
189
                return False
190
        if not (repo / ".git").exists():
191
            return False
192
        return self.adapter.oracle.fsck(repo) == "ok"
193
194
    def acked_reachable(self, repo: Path, heads: list[str]) -> tuple[Optional[bool], list[str]]:
195
        if not heads:
196
            return None, []
197
        try:
198
            if self.kind == "oak":
199
                with OakGraph(repo / ".oak" / "oak.db") as g:
200
                    reach = g.reachable()
201
                lost = [h for h in heads if h not in reach]
202
            else:
203
                lost = [h for h in heads if not self.adapter.oracle.reachable_from_refs(repo, h)]
204
        except Exception as exc:  # noqa: BLE001 - reader failure is unmeasured, never a subject violation
205
            return None, [f"reader error: {type(exc).__name__}: {exc}"]
206
        return not lost, lost
207
208
    def server_intact(self, intended: Optional[str]) -> tuple[Optional[bool], dict]:
209
        pre = self.fixture.meta.get("pre_server_heads", {})
210
        detail: dict = {}
211
        try:
212
            if self.kind == "oak":
213
                with OakGraph(self.server.db()) as g:
214
                    detail["integrity"] = g.integrity()
215
                    heads = g.branch_heads()
216
                    reach = g.ancestors(heads.values())
217
                    missing = g.missing_parents([h for h in heads.values()])
218
            else:
219
                remote = self.dir / "srv" / "remote.git"
220
                detail["integrity"] = self.adapter.oracle.fsck(remote)
221
                refs = self.adapter.oracle.refs(remote)
222
                heads = {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
223
                reach = set()
224
                for v in pre.values():
225
                    if any(self.adapter.oracle.is_ancestor(remote, v, h) for h in heads.values()):
226
                        reach.add(v)
227
                missing = []
228
        except Exception as exc:  # noqa: BLE001 - reader failure is unmeasured, never a subject violation
229
            return None, {"reader_error": f"{type(exc).__name__}: {exc}"}
230
        head = heads.get(self.fixture.branch)
231
        detail.update(server_head=head, lost_pre_heads=[v for v in pre.values() if v not in reach],
232
                      torn_graph=missing[:5])
233
        allowed = {pre.get(self.fixture.branch), intended}
234
        detail["head_in_allowed"] = head in allowed
235
        ok = detail["integrity"] == "ok" and not detail["lost_pre_heads"] and not missing and head in allowed
236
        return ok, detail
237
238
239
def worktree_verdict(trial: Trial, repo: Path, allowed: list[Optional[dict]]) -> tuple[Optional[bool], dict]:
240
    from .readers import worktree_manifest
241
242
    wt = worktree_manifest(repo)
243
    if wt is None:
244
        return None, {"worktree": "unreadable"}
245
    if any(a is not None and wt == a for a in allowed):
246
        return True, {"worktree": "pre_or_post"}
247
    head_tree, status = trial.adapter.tree(repo)
248
    if head_tree is not None and wt == head_tree:
249
        return True, {"worktree": "matches_head_tree"}
250
    dirty, res = trial.adapter.status_dirty(repo)
251
    if dirty is None:
252
        # status refused with an error: acceptable only if it is actionable
253
        ok = actionable(res)
254
        return ok, {"worktree": "differs; status errored", "status_actionable": ok, "status": res.brief(300),
255
                    "head_tree_reader": status}
256
    if dirty:
257
        return True, {"worktree": "differs; reported dirty", "head_tree_reader": status}
258
    if head_tree is None:
259
        return None, {"worktree": "differs; status clean; head tree unreadable", "head_tree_reader": status}
260
    return False, {"worktree": "SILENT_TORN: differs from HEAD tree while status reports clean"}
261
262
263
def torn_worktree(wt: Optional[dict], pre: Optional[dict], post: Optional[dict]) -> Optional[dict]:
264
    """Split a crashed worktree against the interrupted operation's two sides.
265
266
    Returns None when unmeasurable, else lists of paths the op changes that
267
    already hold the post-op (target) side (``op_output``) or still hold the
268
    pre-op side (``pre_side``), plus op artifacts in neither side (``stray``,
269
    e.g. temp files).
270
    """
271
    if wt is None or pre is None or post is None:
272
        return None
273
    changed_by_op = {p for p in set(pre) | set(post) if pre.get(p) != post.get(p)}
274
    op_output, pre_side, stray = [], [], []
275
    for path in sorted(set(wt) | changed_by_op):
276
        have = wt.get(path)
277
        if path in changed_by_op:
278
            if have == post.get(path):
279
                op_output.append(path)
280
            elif have == pre.get(path):
281
                pre_side.append(path)
282
        elif path not in pre and path not in post and have is not None:
283
            stray.append(path)
284
    return {"op_output": op_output, "pre_side": pre_side, "stray": stray}
285
286
287
def torn_state(split: dict, ref_side: Optional[str]) -> bool:
288
    """Is the worktree out of step with where the ref says the operation is?
289
290
    ref_side "pre": any of the op's output (or its temp files) is present.
291
    ref_side "post": any changed path still on the pre side (or temp files).
292
    unknown ref: the worktree mixes both sides, or has temp files.
293
    """
294
    if split["stray"]:
295
        return True
296
    if ref_side == "pre":
297
        return bool(split["op_output"])
298
    if ref_side == "post":
299
        return bool(split["pre_side"])
300
    return bool(split["op_output"]) and bool(split["pre_side"])
301
302
303
def torn_verdict(trial: "Trial", repo: Path, pre: Optional[dict], post: Optional[dict],
304
                 pre_head: Optional[str], op_res: Optional[Result] = None) -> tuple[Optional[bool], dict]:
305
    """Invariant ``worktree_not_torn``.
306
307
    Fails when, after the crash, the worktree holds the interrupted operation's
308
    own output out of step with the ref (e.g. files already on the target side
309
    while HEAD still names the old side, or the op's temp files) and the
310
    subject presents it as ordinary uncommitted user changes: status lists
311
    changes, reports no operation in progress, and does not refuse. Committing
312
    that state would record the op's partial output as the user's own work.
313
    Passing: worktree in step with the ref, an in-progress flag (MERGE_HEAD /
314
    progress_state / a stale index.lock that blocks every mutation), a status
315
    refusal (judged by ``worktree_consistent``), or the failing op itself naming
316
    a recovery command (e.g. git's "checkout failed ... retry with git restore").
317
    A killed op prints nothing, so crash rows rely on the persistent flags only.
318
    """
319
    from .readers import worktree_manifest
320
321
    wt = worktree_manifest(repo)
322
    split = torn_worktree(wt, pre, post)
323
    if split is None:
324
        return None, {"torn": "unmeasurable"}
325
    head = trial.adapter.head(repo)
326
    # The op either leaves HEAD where it was or moves it to its result (a merge /
327
    # re-parent commit's hash differs per run, so "moved" is the post side).
328
    ref_side = None if head is None else ("pre" if head == pre_head else "post")
329
    detail = {"ref_side": ref_side, "op_output_files": len(split["op_output"]), "stray_files": len(split["stray"]),
330
              "pre_side_files": len(split["pre_side"]), "sample": (split["stray"] + split["op_output"])[:5]}
331
    if not torn_state(split, ref_side):
332
        return True, {**detail, "torn": "no"}
333
    if op_res is not None and (op_res.returncode or 0) > 0 and RECOVERY_COMMAND_RE.search(op_res.text()):
334
        return True, {**detail, "torn": "failing op named a recovery command"}
335
    changes, in_progress, res = trial.adapter.status_state(repo)
336
    detail["status_rc"] = res.returncode
337
    if changes is None:
338
        return True, {**detail, "torn": "status refused (judged by worktree_consistent)"}
339
    if in_progress:
340
        return True, {**detail, "torn": "flagged in progress"}
341
    if changes:
342
        return False, {**detail, "torn": "PARTIAL_OP_OUTPUT_AS_USER_CHANGES"}
343
    return True, {**detail, "torn": "status clean (judged by worktree_consistent)"}
344
345
346
def classify(invariants: dict[str, Optional[bool]], harness_error: Optional[str] = None,
347
             fault_fired: Optional[bool] = None, reader_errors: Optional[list] = None) -> str:
348
    """held / violated / harness_error / fault_not_fired.
349
350
    ``fault_fired=False`` (the fault demonstrably never took effect) is never
351
    ``held``: the row is a clean run. ``None`` means the family has no single
352
    fault to fire (concurrency). Reader exceptions are harness errors unless a
353
    subject-level invariant already failed independently of the reader.
354
    """
355
    if harness_error:
356
        return HARNESS_ERROR
357
    if fault_fired is False:
358
        return FAULT_NOT_FIRED
359
    if any(v is False for v in invariants.values()):
360
        return VIOLATED
361
    if reader_errors:
362
        return HARNESS_ERROR
363
    return HELD
364
365
366
def converge_check(trial: Trial, calib: Calibration, repo: Path) -> tuple[Optional[bool], dict]:
367
    """After a successful next op, is the state exactly the calibrated outcome?"""
368
    from .readers import worktree_manifest
369
370
    op = trial.fixture.op
371
    detail: dict = {}
372
    if op == "push":
373
        local = trial.adapter.head(repo)
374
        remote = trial.server_head()
375
        detail.update(local_head=local, server_head=remote)
376
        if remote is None or local is None or remote != local:
377
            return False, detail
378
        tree, status = trial.server_tree(remote)
379
        detail["server_tree_reader"] = status
380
        if tree is None:
381
            return None, detail
382
        return tree == calib.post_tree, detail
383
    if op == "switch":
384
        branch = trial.adapter.current_branch(repo)
385
        detail["branch"] = branch
386
        return branch == trial.fixture.meta["target"] and worktree_manifest(repo) == calib.post_worktree, detail
387
    tree, status = trial.adapter.tree(repo)
388
    detail["head_tree_reader"] = status
389
    if tree is None:
390
        return (False if status.startswith("corrupt") else None), detail
391
    same_tree = tree == calib.post_tree
392
    same_wt = worktree_manifest(repo) == calib.post_worktree
393
    detail.update(head_tree_matches=same_tree, worktree_matches=same_wt)
394
    if op == "clone":
395
        detail["head"] = trial.adapter.head(repo)
396
    return same_tree and same_wt, detail