Viewing files at commit 5717cc9fc7edCommit detailsView latest

suites/robustness/robustness/engine.py 407 lines · 19.1 KB · python
1
"""Trial engine: copy a pristine fixture, inject one fault, judge invariants.
2
3
Invariants (each True / False / None=unmeasured, ADR-0002):
4
5
- ``op_no_hang``            the faulted operation ended (killed or exited) within its timeout
6
- ``repo_opens``            independent open: SQLite integrity_check (Oak) / git fsck (Git)
7
- ``acked_local_reachable`` every commit a local ref named before the op is still reachable
8
                            from a local ref (branch head / HEAD / tag), per the independent reader
9
- ``server_intact``         remote store opens, its pre-op heads stay reachable, and the branch
10
                            head is either the pre-op head or the operation's intended head
11
- ``worktree_consistent``   the worktree equals the pre- or post-op state, or a committed tree,
12
                            or the subject's status reports it dirty/in-progress (never "clean"
13
                            while differing from HEAD)
14
- ``next_op_ok_or_actionable`` rerunning the operation succeeds, or fails with a typed/actionable
15
                            message (not a hang, panic, or signal)
16
- ``no_false_success``      a reported success (exit 0) leaves exactly the calibrated outcome
17
- ``acked_after_next``      pre-op local commits are still reachable after the next operation
18
"""
19
20
from __future__ import annotations
21
22
import json
23
import re
24
import shutil
25
import time
26
from dataclasses import dataclass, field
27
from pathlib import Path
28
from typing import Any, Callable, Optional
29
30
from . import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, VIOLATED
31
from .adapters import GitAdapter, OakAdapter, Server
32
from .fixtures import Fixture
33
from .procs import Result, copy_tree
34
from .readers import OakGraph
35
36
# A failure is actionable only if it tells the user what to do next:
37
# a concrete recovery command (a real subcommand of the subject, or a recovery
38
# flag), an explicit manual step, a documented retryable condition, or a typed,
39
# specific JSON error code / recommended command list. Mentioning "oak" or
40
# "git" is not enough ("failed to open oak database", "not a git repository").
41
RECOVERY_SUBCOMMANDS = (
42
    "pull", "push", "commit", "fetch", "clone", "switch", "checkout", "merge", "rebase", "reset", "restore",
43
    "stash", "add", "rm", "status", "init", "close", "am", "cherry-pick", "revert", "gc", "fsck", "prune",
44
    "doctor", "sparse", "conflict", "login", "finish", "desc")
45
RECOVERY_COMMAND_RE = re.compile(
46
    r"\b(?:oak|git)\s+(?:" + "|".join(re.escape(c) for c in RECOVERY_SUBCOMMANDS) + r")\b"
47
    r"|(?:^|\s)--(?:continue|abort|skip|force)\b", re.I)
48
RECOVERY_STEP_RE = re.compile(
49
    r"remove the file manually|remove this directory|move or remove|delete (?:it|this directory|the directory)"
50
    r"|commit your changes|stash them|\bpull (?:changes )?first\b|\((?:fetch first|non-fast-forward)\)", re.I)
51
RETRYABLE_RE = re.compile(r"\btry again\b|\bretry\b|\bre-?run\b|repository (?:is )?locked|lock(?:ed)? by (?:another|pid)",
52
                          re.I)
53
# Git's normal outcome messages for a conflicted merge / refused fast-forward. They
54
# name the next step (resolve and commit; merge or rebase instead of --ff-only), but
55
# are accepted only when nothing in the output reports corruption or an internal /
56
# server failure (see NON_ACTIONABLE_RE), so they cannot launder a broken repository.
57
GIT_OUTCOME_HINT_RE = re.compile(
58
    r"Automatic merge failed; fix conflicts and then commit the result\."
59
    r"|Not possible to fast-forward, aborting\."
60
    r"|hint: Diverging branches can't be fast-forwarded", re.I)
61
# Corruption / internal-failure wording: never actionable on retry wording or a generic code alone.
62
NON_ACTIONABLE_RE = re.compile(
63
    r"panicked at|RUST_BACKTRACE|not a database|malformed|corrupt|internal (?:server )?error|\binternal\b"
64
    r"|integrity check failed|unexpected (?:eof|end)|not a git repository|HTTP 500\b", re.I)
65
GENERIC_ERROR_CODES = {"error", "internal", "internal_error", "unknown", "unknown_error", "io", "io_error", "other",
66
                       "panic", "database", "database_error", "sqlite", "unexpected", "server_error", "http_error"}
67
68
69
def _json_error(stdout: str) -> Optional[dict]:
70
    try:
71
        data = json.loads(stdout)
72
    except (ValueError, TypeError):
73
        return None
74
    if isinstance(data, dict) and isinstance(data.get("error"), dict):
75
        return data["error"]
76
    return None
77
78
79
def actionable(res: Result) -> bool:
80
    """True when a failed command tells the user a concrete next step."""
81
    text = res.text()
82
    if res.returncode is None or res.returncode <= 0:
83
        return False
84
    if "panicked at" in text or "RUST_BACKTRACE" in text:
85
        return False
86
    if RECOVERY_COMMAND_RE.search(text) or RECOVERY_STEP_RE.search(text):
87
        return True
88
    broken = bool(NON_ACTIONABLE_RE.search(text))
89
    if not broken and GIT_OUTCOME_HINT_RE.search(text):
90
        return True
91
    error = _json_error(res.stdout)
92
    if error is not None and not broken:
93
        code = str(error.get("code") or "").strip().lower()
94
        if error.get("recommended_next_commands"):
95
            return True
96
        if code and code not in GENERIC_ERROR_CODES and not code.startswith(("internal", "unknown", "io_")):
97
            return True
98
    return not broken and bool(RETRYABLE_RE.search(text))
99
100
101
@dataclass
102
class Calibration:
103
    elapsed_ms: float
104
    post_tree: Optional[dict]
105
    post_worktree: Optional[dict]
106
    post_head: Optional[str]
107
    post_server_head: Optional[str]
108
    boundaries: int = 0
109
    boundary_kinds: list = field(default_factory=list)
110
    requests: list = field(default_factory=list)
111
    ok: bool = True
112
    detail: str = ""
113
114
115
class Trial:
116
    """One copied fixture + (for Oak) its loopback server, ready for one faulted op."""
117
118
    def __init__(self, fixture: Fixture, adapter_bin: str, work: Path, name: str):
119
        self.fixture = fixture
120
        self.dir = work / name
121
        copy_tree(fixture.pristine, self.dir)
122
        self.home = self.dir / "home"
123
        self.kind = fixture.kind
124
        if self.kind == "oak":
125
            self.adapter = OakAdapter(adapter_bin, self.home)
126
            self.server: Optional[Server] = Server(adapter_bin, self.dir / "srv", fixture.port or 0,
127
                                                   self.dir / "serve.log")
128
        else:
129
            self.adapter = GitAdapter(adapter_bin, self.home)
130
            self.server = None
131
        self.client = self.dir / "client"
132
133
    # op plumbing ---------------------------------------------------------------
134
    def op_cwd(self) -> Path:
135
        return self.dir if self.fixture.op == "clone" else self.client
136
137
    def op_argv(self, bin_override: Optional[str] = None, dest: str = "cloned", url: Optional[str] = None) -> list[str]:
138
        kw: dict[str, Any] = {"bin": bin_override}
139
        if self.fixture.op == "clone":
140
            kw.update(dest=dest, branch=self.fixture.branch, url=url or self.fixture.meta["url"])
141
        if self.fixture.op == "switch":
142
            kw["target"] = self.fixture.meta["target"]
143
        return self.adapter.argv(self.fixture.op, **kw)
144
145
    def pre(self) -> None:
146
        for argv in self.adapter.pre_argv(self.fixture.op):
147
            self.adapter.must(argv, self.client)
148
149
    def start_server(self, port: Optional[int] = None) -> None:
150
        if self.server is not None:
151
            self.server.start(port)
152
153
    def stop_server(self) -> None:
154
        if self.server is not None:
155
            self.server.stop()
156
157
    def settle(self) -> None:
158
        """Let orphaned server-side children (Git local receive-pack) finish."""
159
        if self.kind == "git":
160
            remote = self.dir / "srv" / "remote.git"
161
            deadline = time.monotonic() + 10
162
            while time.monotonic() < deadline and any(remote.rglob("*.lock")):
163
                time.sleep(0.05)
164
            time.sleep(0.1)
165
166
    def target_repo(self, dest: str = "cloned") -> Path:
167
        return self.dir / dest if self.fixture.op == "clone" else self.client
168
169
    def cleanup(self) -> None:
170
        self.stop_server()
171
        shutil.rmtree(self.dir, ignore_errors=True)
172
173
    # independent observations ---------------------------------------------------
174
    def server_head(self) -> Optional[str]:
175
        if self.kind == "oak":
176
            db = self.server.db() if self.server else None
177
            if not db or not db.exists():
178
                return None
179
            with OakGraph(db) as g:
180
                return g.branch_heads().get(self.fixture.branch)
181
        refs = self.adapter.oracle.refs(self.dir / "srv" / "remote.git")
182
        return refs.get(f"refs/heads/{self.fixture.branch}")
183
184
    def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
185
        if self.kind == "oak":
186
            from .readers import oak_commit_tree
187
            return oak_commit_tree(self.server.db(), commit)
188
        return self.adapter.oracle.commit_tree(self.dir / "srv" / "remote.git", commit)
189
190
    def repo_opens(self, repo: Path) -> Optional[bool]:
191
        if self.kind == "oak":
192
            db = repo / ".oak" / "oak.db"
193
            if not db.exists():
194
                return False
195
            try:
196
                with OakGraph(db) as g:
197
                    return g.integrity() == "ok" and g.metadata("current_branch") is not None
198
            except Exception:  # noqa: BLE001 - any failure to open is the finding
199
                return False
200
        if not (repo / ".git").exists():
201
            return False
202
        return self.adapter.oracle.fsck(repo) == "ok"
203
204
    def acked_reachable(self, repo: Path, heads: list[str]) -> tuple[Optional[bool], list[str]]:
205
        if not heads:
206
            return None, []
207
        try:
208
            if self.kind == "oak":
209
                with OakGraph(repo / ".oak" / "oak.db") as g:
210
                    reach = g.reachable()
211
                lost = [h for h in heads if h not in reach]
212
            else:
213
                lost = [h for h in heads if not self.adapter.oracle.reachable_from_refs(repo, h)]
214
        except Exception as exc:  # noqa: BLE001 - reader failure is unmeasured, never a subject violation
215
            return None, [f"reader error: {type(exc).__name__}: {exc}"]
216
        return not lost, lost
217
218
    def server_intact(self, intended: Optional[str]) -> tuple[Optional[bool], dict]:
219
        pre = self.fixture.meta.get("pre_server_heads", {})
220
        detail: dict = {}
221
        try:
222
            if self.kind == "oak":
223
                with OakGraph(self.server.db()) as g:
224
                    detail["integrity"] = g.integrity()
225
                    heads = g.branch_heads()
226
                    reach = g.ancestors(heads.values())
227
                    missing = g.missing_parents([h for h in heads.values()])
228
            else:
229
                remote = self.dir / "srv" / "remote.git"
230
                detail["integrity"] = self.adapter.oracle.fsck(remote)
231
                refs = self.adapter.oracle.refs(remote)
232
                heads = {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
233
                reach = set()
234
                for v in pre.values():
235
                    if any(self.adapter.oracle.is_ancestor(remote, v, h) for h in heads.values()):
236
                        reach.add(v)
237
                missing = []
238
        except Exception as exc:  # noqa: BLE001 - reader failure is unmeasured, never a subject violation
239
            return None, {"reader_error": f"{type(exc).__name__}: {exc}"}
240
        head = heads.get(self.fixture.branch)
241
        detail.update(server_head=head, lost_pre_heads=[v for v in pre.values() if v not in reach],
242
                      torn_graph=missing[:5])
243
        allowed = {pre.get(self.fixture.branch), intended}
244
        detail["head_in_allowed"] = head in allowed
245
        ok = detail["integrity"] == "ok" and not detail["lost_pre_heads"] and not missing and head in allowed
246
        return ok, detail
247
248
249
def worktree_verdict(trial: Trial, repo: Path, allowed: list[Optional[dict]]) -> tuple[Optional[bool], dict]:
250
    from .readers import worktree_manifest
251
252
    wt = worktree_manifest(repo)
253
    if wt is None:
254
        return None, {"worktree": "unreadable"}
255
    if any(a is not None and wt == a for a in allowed):
256
        return True, {"worktree": "pre_or_post"}
257
    head_tree, status = trial.adapter.tree(repo)
258
    if head_tree is not None and wt == head_tree:
259
        return True, {"worktree": "matches_head_tree"}
260
    dirty, res = trial.adapter.status_dirty(repo)
261
    if dirty is None:
262
        # status refused with an error: acceptable only if it is actionable
263
        ok = actionable(res)
264
        return ok, {"worktree": "differs; status errored", "status_actionable": ok, "status": res.brief(300),
265
                    "head_tree_reader": status}
266
    if dirty:
267
        return True, {"worktree": "differs; reported dirty", "head_tree_reader": status}
268
    if head_tree is None:
269
        return None, {"worktree": "differs; status clean; head tree unreadable", "head_tree_reader": status}
270
    return False, {"worktree": "SILENT_TORN: differs from HEAD tree while status reports clean"}
271
272
273
def torn_worktree(wt: Optional[dict], pre: Optional[dict], post: Optional[dict]) -> Optional[dict]:
274
    """Split a crashed worktree against the interrupted operation's two sides.
275
276
    Returns None when unmeasurable, else lists of paths the op changes that
277
    already hold the post-op (target) side (``op_output``) or still hold the
278
    pre-op side (``pre_side``), plus op artifacts in neither side (``stray``,
279
    e.g. temp files).
280
    """
281
    if wt is None or pre is None or post is None:
282
        return None
283
    changed_by_op = {p for p in set(pre) | set(post) if pre.get(p) != post.get(p)}
284
    op_output, pre_side, stray = [], [], []
285
    for path in sorted(set(wt) | changed_by_op):
286
        have = wt.get(path)
287
        if path in changed_by_op:
288
            if have == post.get(path):
289
                op_output.append(path)
290
            elif have == pre.get(path):
291
                pre_side.append(path)
292
        elif path not in pre and path not in post and have is not None:
293
            stray.append(path)
294
    return {"op_output": op_output, "pre_side": pre_side, "stray": stray}
295
296
297
def torn_state(split: dict, ref_side: Optional[str]) -> bool:
298
    """Is the worktree out of step with where the ref says the operation is?
299
300
    ref_side "pre": any of the op's output (or its temp files) is present.
301
    ref_side "post": any changed path still on the pre side (or temp files).
302
    unknown ref: the worktree mixes both sides, or has temp files.
303
    """
304
    if split["stray"]:
305
        return True
306
    if ref_side == "pre":
307
        return bool(split["op_output"])
308
    if ref_side == "post":
309
        return bool(split["pre_side"])
310
    return bool(split["op_output"]) and bool(split["pre_side"])
311
312
313
def torn_verdict(trial: "Trial", repo: Path, pre: Optional[dict], post: Optional[dict],
314
                 pre_head: Optional[str], op_res: Optional[Result] = None) -> tuple[Optional[bool], dict]:
315
    """Invariant ``worktree_not_torn``.
316
317
    Fails when, after the crash, the worktree holds the interrupted operation's
318
    own output out of step with the ref (e.g. files already on the target side
319
    while HEAD still names the old side, or the op's temp files) and the
320
    subject presents it as ordinary uncommitted user changes: status lists
321
    changes, reports no operation in progress, and does not refuse. Committing
322
    that state would record the op's partial output as the user's own work.
323
    Passing: worktree in step with the ref, an in-progress flag (MERGE_HEAD /
324
    progress_state / a stale git lock - index.lock, HEAD.lock or the current
325
    branch's ref lock - that makes the next commit refuse), a status
326
    refusal (judged by ``worktree_consistent``), or the failing op itself naming
327
    a recovery command (e.g. git's "checkout failed ... retry with git restore").
328
    A killed op prints nothing, so crash rows rely on the persistent flags only.
329
    """
330
    from .readers import worktree_manifest
331
332
    wt = worktree_manifest(repo)
333
    split = torn_worktree(wt, pre, post)
334
    if split is None:
335
        return None, {"torn": "unmeasurable"}
336
    head = trial.adapter.head(repo)
337
    # The op either leaves HEAD where it was or moves it to its result (a merge /
338
    # re-parent commit's hash differs per run, so "moved" is the post side).
339
    ref_side = None if head is None else ("pre" if head == pre_head else "post")
340
    detail = {"ref_side": ref_side, "op_output_files": len(split["op_output"]), "stray_files": len(split["stray"]),
341
              "pre_side_files": len(split["pre_side"]), "sample": (split["stray"] + split["op_output"])[:5]}
342
    if not torn_state(split, ref_side):
343
        return True, {**detail, "torn": "no"}
344
    if op_res is not None and (op_res.returncode or 0) > 0 and RECOVERY_COMMAND_RE.search(op_res.text()):
345
        return True, {**detail, "torn": "failing op named a recovery command"}
346
    changes, in_progress, res = trial.adapter.status_state(repo)
347
    detail["status_rc"] = res.returncode
348
    if changes is None:
349
        return True, {**detail, "torn": "status refused (judged by worktree_consistent)"}
350
    if in_progress:
351
        return True, {**detail, "torn": "flagged in progress"}
352
    if changes:
353
        return False, {**detail, "torn": "PARTIAL_OP_OUTPUT_AS_USER_CHANGES"}
354
    return True, {**detail, "torn": "status clean (judged by worktree_consistent)"}
355
356
357
def classify(invariants: dict[str, Optional[bool]], harness_error: Optional[str] = None,
358
             fault_fired: Optional[bool] = None, reader_errors: Optional[list] = None) -> str:
359
    """held / violated / harness_error / fault_not_fired.
360
361
    ``fault_fired=False`` (the fault demonstrably never took effect) is never
362
    ``held``: the row is a clean run. ``None`` means the family has no single
363
    fault to fire (concurrency). Reader exceptions are harness errors unless a
364
    subject-level invariant already failed independently of the reader.
365
    """
366
    if harness_error:
367
        return HARNESS_ERROR
368
    if fault_fired is False:
369
        return FAULT_NOT_FIRED
370
    if any(v is False for v in invariants.values()):
371
        return VIOLATED
372
    if reader_errors:
373
        return HARNESS_ERROR
374
    return HELD
375
376
377
def converge_check(trial: Trial, calib: Calibration, repo: Path) -> tuple[Optional[bool], dict]:
378
    """After a successful next op, is the state exactly the calibrated outcome?"""
379
    from .readers import worktree_manifest
380
381
    op = trial.fixture.op
382
    detail: dict = {}
383
    if op == "push":
384
        local = trial.adapter.head(repo)
385
        remote = trial.server_head()
386
        detail.update(local_head=local, server_head=remote)
387
        if remote is None or local is None or remote != local:
388
            return False, detail
389
        tree, status = trial.server_tree(remote)
390
        detail["server_tree_reader"] = status
391
        if tree is None:
392
            return None, detail
393
        return tree == calib.post_tree, detail
394
    if op == "switch":
395
        branch = trial.adapter.current_branch(repo)
396
        detail["branch"] = branch
397
        return branch == trial.fixture.meta["target"] and worktree_manifest(repo) == calib.post_worktree, detail
398
    tree, status = trial.adapter.tree(repo)
399
    detail["head_tree_reader"] = status
400
    if tree is None:
401
        return (False if status.startswith("corrupt") else None), detail
402
    same_tree = tree == calib.post_tree
403
    same_wt = worktree_manifest(repo) == calib.post_worktree
404
    detail.update(head_tree_matches=same_tree, worktree_matches=same_wt)
405
    if op == "clone":
406
        detail["head"] = trial.adapter.head(repo)
407
    return same_tree and same_wt, detail