Viewing files at commit 369435870869Commit detailsView latest

suites/robustness/robustness/engine.py 430 lines · 20.1 KB · python
1
"""Trial engine: copy a pristine fixture, inject one fault, judge invariants.
2
3
Invariants (each True / False / None=unmeasured, ADR-0002):
4
5
- ``op_no_hang``            the faulted operation ended (killed or exited) within its timeout
6
- ``repo_opens``            independent open: SQLite integrity_check (Oak) / git fsck (Git)
7
- ``acked_local_reachable`` every commit a local ref named before the op is still reachable
8
                            from a local ref (branch head / HEAD / tag), per the independent reader
9
- ``server_intact``         remote store opens, its pre-op heads stay reachable, and the branch
10
                            head is either the pre-op head or the operation's intended head
11
- ``worktree_consistent``   the worktree equals the pre- or post-op state, or a committed tree,
12
                            or the subject's status reports it dirty/in-progress (never "clean"
13
                            while differing from HEAD)
14
- ``next_op_ok_or_actionable`` rerunning the operation succeeds, or fails with a typed/actionable
15
                            message (not a hang, panic, or signal)
16
- ``no_false_success``      a reported success (exit 0) leaves exactly the calibrated outcome
17
- ``acked_after_next``      pre-op local commits are still reachable after the next operation
18
"""
19
20
from __future__ import annotations
21
22
import json
23
import re
24
import shutil
25
import time
26
from dataclasses import dataclass, field
27
from pathlib import Path
28
from typing import Any, Callable, Optional
29
30
from . import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, VIOLATED
31
from .adapters import GitAdapter, OakAdapter, Server
32
from .fixtures import Fixture
33
from .procs import Result, copy_tree
34
from .readers import OakGraph
35
36
# A failure is actionable only if it tells the user what to do next:
37
# a concrete recovery command (a real subcommand of the subject, or a recovery
38
# flag), an explicit manual step, a documented retryable condition, or a typed,
39
# specific JSON error code / recommended command list. Mentioning "oak" or
40
# "git" is not enough ("failed to open oak database", "not a git repository").
41
RECOVERY_SUBCOMMANDS = (
42
    "pull", "push", "commit", "fetch", "clone", "switch", "checkout", "merge", "rebase", "reset", "restore",
43
    "stash", "add", "rm", "status", "init", "close", "am", "cherry-pick", "revert", "gc", "fsck", "prune",
44
    "doctor", "sparse", "conflict", "login", "finish", "desc")
45
RECOVERY_COMMAND_RE = re.compile(
46
    r"\b(?:oak|git)\s+(?:" + "|".join(re.escape(c) for c in RECOVERY_SUBCOMMANDS) + r")\b"
47
    r"|(?:^|\s)--(?:continue|abort|skip|force)\b", re.I)
48
RECOVERY_STEP_RE = re.compile(
49
    r"remove the file manually|remove this directory|move or remove|delete (?:it|this directory|the directory)"
50
    r"|commit your changes|stash them|\bpull (?:changes )?first\b|\((?:fetch first|non-fast-forward)\)", re.I)
51
RETRYABLE_RE = re.compile(r"\btry again\b|\bretry\b|\bre-?run\b|repository (?:is )?locked|lock(?:ed)? by (?:another|pid)",
52
                          re.I)
53
# Git's normal outcome messages for a conflicted merge / refused fast-forward. They
54
# name the next step (resolve and commit; merge or rebase instead of --ff-only). They
55
# are accepted only from the git subject and only when EVERY output line is one of
56
# git's ordinary merge/pull lines (an allowlist, not a denylist of corruption words):
57
# any other line - "fatal: bad object HEAD", "unable to read sha1 file", a segfault,
58
# "unable to open database file" - means something else went wrong, so the hint
59
# cannot launder it.
60
GIT_OUTCOME_HINT_RE = re.compile(
61
    r"^(?:fatal: )?Not possible to fast-forward, aborting\.$"
62
    r"|^Automatic merge failed; fix conflicts and then commit the result\.$", re.M)
63
GIT_OUTCOME_LINE_RE = re.compile(
64
    r"Automatic merge failed; fix conflicts and then commit the result\."
65
    r"|(?:fatal: )?Not possible to fast-forward, aborting\."
66
    r"|hint:.*"
67
    r"|Auto-merging [^\n]+"
68
    r"|CONFLICT \([a-z/ -]+\): [^\n]+"
69
    r"|From [^\s]+"
70
    r"|\s*\* branch\s+\S+\s+-> FETCH_HEAD"
71
    r"|\s*(?:\* \[new branch\]|[0-9a-f]{7,40}\.\.[0-9a-f]{7,40}|\+ [0-9a-f]{7,40}\.\.\.[0-9a-f]{7,40})"
72
    r"\s+\S+\s+-> \S+(?: \(forced update\))?"
73
    r"|Updating [0-9a-f]{7,40}\.\.[0-9a-f]{7,40}")
74
75
76
def git_outcome_hint(res: Result) -> bool:
77
    """git's conflict / refused-fast-forward outcome with nothing else in the output."""
78
    if not res.argv or Path(res.argv[0]).name != "git":
79
        return False
80
    text = res.text()
81
    if not GIT_OUTCOME_HINT_RE.search(text):
82
        return False
83
    return all(GIT_OUTCOME_LINE_RE.fullmatch(line.rstrip()) for line in text.splitlines() if line.strip())
84
# Corruption / internal-failure wording: never actionable on retry wording or a generic code alone.
85
NON_ACTIONABLE_RE = re.compile(
86
    r"panicked at|RUST_BACKTRACE|not a database|malformed|corrupt|internal (?:server )?error|\binternal\b"
87
    r"|integrity check failed|unexpected (?:eof|end)|not a git repository|HTTP 500\b", re.I)
88
GENERIC_ERROR_CODES = {"error", "internal", "internal_error", "unknown", "unknown_error", "io", "io_error", "other",
89
                       "panic", "database", "database_error", "sqlite", "unexpected", "server_error", "http_error"}
90
91
92
def _json_error(stdout: str) -> Optional[dict]:
93
    try:
94
        data = json.loads(stdout)
95
    except (ValueError, TypeError):
96
        return None
97
    if isinstance(data, dict) and isinstance(data.get("error"), dict):
98
        return data["error"]
99
    return None
100
101
102
def actionable(res: Result) -> bool:
103
    """True when a failed command tells the user a concrete next step."""
104
    text = res.text()
105
    if res.returncode is None or res.returncode <= 0:
106
        return False
107
    if "panicked at" in text or "RUST_BACKTRACE" in text:
108
        return False
109
    if RECOVERY_COMMAND_RE.search(text) or RECOVERY_STEP_RE.search(text):
110
        return True
111
    broken = bool(NON_ACTIONABLE_RE.search(text))
112
    if not broken and git_outcome_hint(res):
113
        return True
114
    error = _json_error(res.stdout)
115
    if error is not None and not broken:
116
        code = str(error.get("code") or "").strip().lower()
117
        if error.get("recommended_next_commands"):
118
            return True
119
        if code and code not in GENERIC_ERROR_CODES and not code.startswith(("internal", "unknown", "io_")):
120
            return True
121
    return not broken and bool(RETRYABLE_RE.search(text))
122
123
124
@dataclass
125
class Calibration:
126
    elapsed_ms: float
127
    post_tree: Optional[dict]
128
    post_worktree: Optional[dict]
129
    post_head: Optional[str]
130
    post_server_head: Optional[str]
131
    boundaries: int = 0
132
    boundary_kinds: list = field(default_factory=list)
133
    requests: list = field(default_factory=list)
134
    ok: bool = True
135
    detail: str = ""
136
137
138
class Trial:
139
    """One copied fixture + (for Oak) its loopback server, ready for one faulted op."""
140
141
    def __init__(self, fixture: Fixture, adapter_bin: str, work: Path, name: str):
142
        self.fixture = fixture
143
        self.dir = work / name
144
        copy_tree(fixture.pristine, self.dir)
145
        self.home = self.dir / "home"
146
        self.kind = fixture.kind
147
        if self.kind == "oak":
148
            self.adapter = OakAdapter(adapter_bin, self.home)
149
            self.server: Optional[Server] = Server(adapter_bin, self.dir / "srv", fixture.port or 0,
150
                                                   self.dir / "serve.log")
151
        else:
152
            self.adapter = GitAdapter(adapter_bin, self.home)
153
            self.server = None
154
        self.client = self.dir / "client"
155
156
    # op plumbing ---------------------------------------------------------------
157
    def op_cwd(self) -> Path:
158
        return self.dir if self.fixture.op == "clone" else self.client
159
160
    def op_argv(self, bin_override: Optional[str] = None, dest: str = "cloned", url: Optional[str] = None) -> list[str]:
161
        kw: dict[str, Any] = {"bin": bin_override}
162
        if self.fixture.op == "clone":
163
            kw.update(dest=dest, branch=self.fixture.branch, url=url or self.fixture.meta["url"])
164
        if self.fixture.op == "switch":
165
            kw["target"] = self.fixture.meta["target"]
166
        return self.adapter.argv(self.fixture.op, **kw)
167
168
    def pre(self) -> None:
169
        for argv in self.adapter.pre_argv(self.fixture.op):
170
            self.adapter.must(argv, self.client)
171
172
    def start_server(self, port: Optional[int] = None) -> None:
173
        if self.server is not None:
174
            self.server.start(port)
175
176
    def stop_server(self) -> None:
177
        if self.server is not None:
178
            self.server.stop()
179
180
    def settle(self) -> None:
181
        """Let orphaned server-side children (Git local receive-pack) finish."""
182
        if self.kind == "git":
183
            remote = self.dir / "srv" / "remote.git"
184
            deadline = time.monotonic() + 10
185
            while time.monotonic() < deadline and any(remote.rglob("*.lock")):
186
                time.sleep(0.05)
187
            time.sleep(0.1)
188
189
    def target_repo(self, dest: str = "cloned") -> Path:
190
        return self.dir / dest if self.fixture.op == "clone" else self.client
191
192
    def cleanup(self) -> None:
193
        self.stop_server()
194
        shutil.rmtree(self.dir, ignore_errors=True)
195
196
    # independent observations ---------------------------------------------------
197
    def server_head(self) -> Optional[str]:
198
        if self.kind == "oak":
199
            db = self.server.db() if self.server else None
200
            if not db or not db.exists():
201
                return None
202
            with OakGraph(db) as g:
203
                return g.branch_heads().get(self.fixture.branch)
204
        refs = self.adapter.oracle.refs(self.dir / "srv" / "remote.git")
205
        return refs.get(f"refs/heads/{self.fixture.branch}")
206
207
    def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
208
        if self.kind == "oak":
209
            from .readers import oak_commit_tree
210
            return oak_commit_tree(self.server.db(), commit)
211
        return self.adapter.oracle.commit_tree(self.dir / "srv" / "remote.git", commit)
212
213
    def repo_opens(self, repo: Path) -> Optional[bool]:
214
        if self.kind == "oak":
215
            db = repo / ".oak" / "oak.db"
216
            if not db.exists():
217
                return False
218
            try:
219
                with OakGraph(db) as g:
220
                    return g.integrity() == "ok" and g.metadata("current_branch") is not None
221
            except Exception:  # noqa: BLE001 - any failure to open is the finding
222
                return False
223
        if not (repo / ".git").exists():
224
            return False
225
        return self.adapter.oracle.fsck(repo) == "ok"
226
227
    def acked_reachable(self, repo: Path, heads: list[str]) -> tuple[Optional[bool], list[str]]:
228
        if not heads:
229
            return None, []
230
        try:
231
            if self.kind == "oak":
232
                with OakGraph(repo / ".oak" / "oak.db") as g:
233
                    reach = g.reachable()
234
                lost = [h for h in heads if h not in reach]
235
            else:
236
                lost = [h for h in heads if not self.adapter.oracle.reachable_from_refs(repo, h)]
237
        except Exception as exc:  # noqa: BLE001 - reader failure is unmeasured, never a subject violation
238
            return None, [f"reader error: {type(exc).__name__}: {exc}"]
239
        return not lost, lost
240
241
    def server_intact(self, intended: Optional[str]) -> tuple[Optional[bool], dict]:
242
        pre = self.fixture.meta.get("pre_server_heads", {})
243
        detail: dict = {}
244
        try:
245
            if self.kind == "oak":
246
                with OakGraph(self.server.db()) as g:
247
                    detail["integrity"] = g.integrity()
248
                    heads = g.branch_heads()
249
                    reach = g.ancestors(heads.values())
250
                    missing = g.missing_parents([h for h in heads.values()])
251
            else:
252
                remote = self.dir / "srv" / "remote.git"
253
                detail["integrity"] = self.adapter.oracle.fsck(remote)
254
                refs = self.adapter.oracle.refs(remote)
255
                heads = {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
256
                reach = set()
257
                for v in pre.values():
258
                    if any(self.adapter.oracle.is_ancestor(remote, v, h) for h in heads.values()):
259
                        reach.add(v)
260
                missing = []
261
        except Exception as exc:  # noqa: BLE001 - reader failure is unmeasured, never a subject violation
262
            return None, {"reader_error": f"{type(exc).__name__}: {exc}"}
263
        head = heads.get(self.fixture.branch)
264
        detail.update(server_head=head, lost_pre_heads=[v for v in pre.values() if v not in reach],
265
                      torn_graph=missing[:5])
266
        allowed = {pre.get(self.fixture.branch), intended}
267
        detail["head_in_allowed"] = head in allowed
268
        ok = detail["integrity"] == "ok" and not detail["lost_pre_heads"] and not missing and head in allowed
269
        return ok, detail
270
271
272
def worktree_verdict(trial: Trial, repo: Path, allowed: list[Optional[dict]]) -> tuple[Optional[bool], dict]:
273
    from .readers import worktree_manifest
274
275
    wt = worktree_manifest(repo)
276
    if wt is None:
277
        return None, {"worktree": "unreadable"}
278
    if any(a is not None and wt == a for a in allowed):
279
        return True, {"worktree": "pre_or_post"}
280
    head_tree, status = trial.adapter.tree(repo)
281
    if head_tree is not None and wt == head_tree:
282
        return True, {"worktree": "matches_head_tree"}
283
    dirty, res = trial.adapter.status_dirty(repo)
284
    if dirty is None:
285
        # status refused with an error: acceptable only if it is actionable
286
        ok = actionable(res)
287
        return ok, {"worktree": "differs; status errored", "status_actionable": ok, "status": res.brief(300),
288
                    "head_tree_reader": status}
289
    if dirty:
290
        return True, {"worktree": "differs; reported dirty", "head_tree_reader": status}
291
    if head_tree is None:
292
        return None, {"worktree": "differs; status clean; head tree unreadable", "head_tree_reader": status}
293
    return False, {"worktree": "SILENT_TORN: differs from HEAD tree while status reports clean"}
294
295
296
def torn_worktree(wt: Optional[dict], pre: Optional[dict], post: Optional[dict]) -> Optional[dict]:
297
    """Split a crashed worktree against the interrupted operation's two sides.
298
299
    Returns None when unmeasurable, else lists of paths the op changes that
300
    already hold the post-op (target) side (``op_output``) or still hold the
301
    pre-op side (``pre_side``), plus op artifacts in neither side (``stray``,
302
    e.g. temp files).
303
    """
304
    if wt is None or pre is None or post is None:
305
        return None
306
    changed_by_op = {p for p in set(pre) | set(post) if pre.get(p) != post.get(p)}
307
    op_output, pre_side, stray = [], [], []
308
    for path in sorted(set(wt) | changed_by_op):
309
        have = wt.get(path)
310
        if path in changed_by_op:
311
            if have == post.get(path):
312
                op_output.append(path)
313
            elif have == pre.get(path):
314
                pre_side.append(path)
315
        elif path not in pre and path not in post and have is not None:
316
            stray.append(path)
317
    return {"op_output": op_output, "pre_side": pre_side, "stray": stray}
318
319
320
def torn_state(split: dict, ref_side: Optional[str]) -> bool:
321
    """Is the worktree out of step with where the ref says the operation is?
322
323
    ref_side "pre": any of the op's output (or its temp files) is present.
324
    ref_side "post": any changed path still on the pre side (or temp files).
325
    unknown ref: the worktree mixes both sides, or has temp files.
326
    """
327
    if split["stray"]:
328
        return True
329
    if ref_side == "pre":
330
        return bool(split["op_output"])
331
    if ref_side == "post":
332
        return bool(split["pre_side"])
333
    return bool(split["op_output"]) and bool(split["pre_side"])
334
335
336
def torn_verdict(trial: "Trial", repo: Path, pre: Optional[dict], post: Optional[dict],
337
                 pre_head: Optional[str], op_res: Optional[Result] = None) -> tuple[Optional[bool], dict]:
338
    """Invariant ``worktree_not_torn``.
339
340
    Fails when, after the crash, the worktree holds the interrupted operation's
341
    own output out of step with the ref (e.g. files already on the target side
342
    while HEAD still names the old side, or the op's temp files) and the
343
    subject presents it as ordinary uncommitted user changes: status lists
344
    changes, reports no operation in progress, and does not refuse. Committing
345
    that state would record the op's partial output as the user's own work.
346
    Passing: worktree in step with the ref, an in-progress flag (MERGE_HEAD /
347
    progress_state / a stale git lock - index.lock, HEAD.lock or the current
348
    branch's ref lock - that makes the next commit refuse), a status
349
    refusal (judged by ``worktree_consistent``), or the failing op itself naming
350
    a recovery command (e.g. git's "checkout failed ... retry with git restore").
351
    A killed op prints nothing, so crash rows rely on the persistent flags only.
352
    """
353
    from .readers import worktree_manifest
354
355
    wt = worktree_manifest(repo)
356
    split = torn_worktree(wt, pre, post)
357
    if split is None:
358
        return None, {"torn": "unmeasurable"}
359
    head = trial.adapter.head(repo)
360
    # The op either leaves HEAD where it was or moves it to its result (a merge /
361
    # re-parent commit's hash differs per run, so "moved" is the post side).
362
    ref_side = None if head is None else ("pre" if head == pre_head else "post")
363
    detail = {"ref_side": ref_side, "op_output_files": len(split["op_output"]), "stray_files": len(split["stray"]),
364
              "pre_side_files": len(split["pre_side"]), "sample": (split["stray"] + split["op_output"])[:5]}
365
    if not torn_state(split, ref_side):
366
        return True, {**detail, "torn": "no"}
367
    if op_res is not None and (op_res.returncode or 0) > 0 and RECOVERY_COMMAND_RE.search(op_res.text()):
368
        return True, {**detail, "torn": "failing op named a recovery command"}
369
    changes, in_progress, res = trial.adapter.status_state(repo)
370
    detail["status_rc"] = res.returncode
371
    if changes is None:
372
        return True, {**detail, "torn": "status refused (judged by worktree_consistent)"}
373
    if in_progress:
374
        return True, {**detail, "torn": "flagged in progress"}
375
    if changes:
376
        return False, {**detail, "torn": "PARTIAL_OP_OUTPUT_AS_USER_CHANGES"}
377
    return True, {**detail, "torn": "status clean (judged by worktree_consistent)"}
378
379
380
def classify(invariants: dict[str, Optional[bool]], harness_error: Optional[str] = None,
381
             fault_fired: Optional[bool] = None, reader_errors: Optional[list] = None) -> str:
382
    """held / violated / harness_error / fault_not_fired.
383
384
    ``fault_fired=False`` (the fault demonstrably never took effect) is never
385
    ``held``: the row is a clean run. ``None`` means the family has no single
386
    fault to fire (concurrency). Reader exceptions are harness errors unless a
387
    subject-level invariant already failed independently of the reader.
388
    """
389
    if harness_error:
390
        return HARNESS_ERROR
391
    if fault_fired is False:
392
        return FAULT_NOT_FIRED
393
    if any(v is False for v in invariants.values()):
394
        return VIOLATED
395
    if reader_errors:
396
        return HARNESS_ERROR
397
    return HELD
398
399
400
def converge_check(trial: Trial, calib: Calibration, repo: Path) -> tuple[Optional[bool], dict]:
401
    """After a successful next op, is the state exactly the calibrated outcome?"""
402
    from .readers import worktree_manifest
403
404
    op = trial.fixture.op
405
    detail: dict = {}
406
    if op == "push":
407
        local = trial.adapter.head(repo)
408
        remote = trial.server_head()
409
        detail.update(local_head=local, server_head=remote)
410
        if remote is None or local is None or remote != local:
411
            return False, detail
412
        tree, status = trial.server_tree(remote)
413
        detail["server_tree_reader"] = status
414
        if tree is None:
415
            return None, detail
416
        return tree == calib.post_tree, detail
417
    if op == "switch":
418
        branch = trial.adapter.current_branch(repo)
419
        detail["branch"] = branch
420
        return branch == trial.fixture.meta["target"] and worktree_manifest(repo) == calib.post_worktree, detail
421
    tree, status = trial.adapter.tree(repo)
422
    detail["head_tree_reader"] = status
423
    if tree is None:
424
        return (False if status.startswith("corrupt") else None), detail
425
    same_tree = tree == calib.post_tree
426
    same_wt = worktree_manifest(repo) == calib.post_worktree
427
    detail.update(head_tree_matches=same_tree, worktree_matches=same_wt)
428
    if op == "clone":
429
        detail["head"] = trial.adapter.head(repo)
430
    return same_tree and same_wt, detail