Viewing files at commit c44c91118a6cCommit detailsView latest

suites/robustness/robustness/engine.py 434 lines · 20.4 KB · python
1
"""Trial engine: copy a pristine fixture, inject one fault, judge invariants.
2
3
Invariants (each True / False / None=unmeasured, ADR-0002):
4
5
- ``op_no_hang``            the faulted operation ended (killed or exited) within its timeout
6
- ``repo_opens``            independent open: SQLite integrity_check (Oak) / git fsck (Git)
7
- ``acked_local_reachable`` every commit a local ref named before the op is still reachable
8
                            from a local ref (branch head / HEAD / tag), per the independent reader
9
- ``server_intact``         remote store opens, its pre-op heads stay reachable, and the branch
10
                            head is either the pre-op head or the operation's intended head
11
- ``worktree_consistent``   the worktree equals the pre- or post-op state, or a committed tree,
12
                            or the subject's status reports it dirty/in-progress (never "clean"
13
                            while differing from HEAD)
14
- ``next_op_ok_or_actionable`` rerunning the operation succeeds, or fails with a typed/actionable
15
                            message (not a hang, panic, or signal)
16
- ``no_false_success``      a reported success (exit 0) leaves exactly the calibrated outcome
17
- ``acked_after_next``      pre-op local commits are still reachable after the next operation
18
"""
19
20
from __future__ import annotations
21
22
import json
23
import re
24
import shutil
25
import time
26
from dataclasses import dataclass, field
27
from pathlib import Path
28
from typing import Any, Callable, Optional
29
30
from . import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, VIOLATED
31
from .adapters import GitAdapter, OakAdapter, Server
32
from .fixtures import Fixture
33
from .procs import Result, copy_tree
34
from .readers import OakGraph
35
36
# A failure is actionable only if it tells the user what to do next:
37
# a concrete recovery command (a real subcommand of the subject, or a recovery
38
# flag), an explicit manual step, a documented retryable condition, or a typed,
39
# specific JSON error code / recommended command list. Mentioning "oak" or
40
# "git" is not enough ("failed to open oak database", "not a git repository").
41
RECOVERY_SUBCOMMANDS = (
42
    "pull", "push", "commit", "fetch", "clone", "switch", "checkout", "merge", "rebase", "reset", "restore",
43
    "stash", "add", "rm", "status", "init", "close", "am", "cherry-pick", "revert", "gc", "fsck", "prune",
44
    "doctor", "sparse", "conflict", "login", "finish", "desc")
45
RECOVERY_COMMAND_RE = re.compile(
46
    r"\b(?:oak|git)\s+(?:" + "|".join(re.escape(c) for c in RECOVERY_SUBCOMMANDS) + r")\b"
47
    r"|(?:^|\s)--(?:continue|abort|skip|force)\b", re.I)
48
RECOVERY_STEP_RE = re.compile(
49
    r"remove the file manually|remove this directory|move or remove|delete (?:it|this directory|the directory)"
50
    r"|commit your changes|stash them|\bpull (?:changes )?first\b|\((?:fetch first|non-fast-forward)\)", re.I)
51
RETRYABLE_RE = re.compile(r"\btry again\b|\bretry\b|\bre-?run\b|repository (?:is )?locked|lock(?:ed)? by (?:another|pid)",
52
                          re.I)
53
# Git's normal outcome messages for a conflicted merge / refused fast-forward. They
54
# name the next step (resolve and commit; merge or rebase instead of --ff-only). They
55
# are accepted only from the git subject and only when EVERY output line is one of
56
# git's ordinary merge/pull lines (an allowlist, not a denylist of corruption words):
57
# any other line - "fatal: bad object HEAD", "unable to read sha1 file", a segfault,
58
# "unable to open database file" - means something else went wrong, so the hint
59
# cannot launder it.
60
GIT_OUTCOME_HINT_RE = re.compile(
61
    r"^(?:fatal: )?Not possible to fast-forward, aborting\.$"
62
    r"|^Automatic merge failed; fix conflicts and then commit the result\.$", re.M)
63
GIT_OUTCOME_LINE_RE = re.compile(
64
    r"Automatic merge failed; fix conflicts and then commit the result\."
65
    r"|(?:fatal: )?Not possible to fast-forward, aborting\."
66
    r"|hint:.*"
67
    r"|Auto-merging [^\n]+"
68
    r"|CONFLICT \([a-z/ -]+\): [^\n]+"
69
    r"|From [^\s]+"
70
    r"|\s*\* branch\s+\S+\s+-> FETCH_HEAD"
71
    r"|\s*(?:\* \[new branch\]|[0-9a-f]{7,40}\.\.[0-9a-f]{7,40}|\+ [0-9a-f]{7,40}\.\.\.[0-9a-f]{7,40})"
72
    r"\s+\S+\s+-> \S+(?: \(forced update\))?"
73
    r"|Updating [0-9a-f]{7,40}\.\.[0-9a-f]{7,40}")
74
75
76
def git_outcome_hint(res: Result) -> bool:
77
    """git's conflict / refused-fast-forward outcome with nothing else in the output."""
78
    if not res.argv or Path(res.argv[0]).name != "git":
79
        return False
80
    text = res.text()
81
    if not GIT_OUTCOME_HINT_RE.search(text):
82
        return False
83
    return all(GIT_OUTCOME_LINE_RE.fullmatch(line.rstrip()) for line in text.splitlines() if line.strip())
84
# Corruption / internal-failure wording: never actionable on retry wording or a generic code alone.
85
NON_ACTIONABLE_RE = re.compile(
86
    r"panicked at|RUST_BACKTRACE|not a database|malformed|corrupt|internal (?:server )?error|\binternal\b"
87
    r"|integrity check failed|unexpected (?:eof|end)|not a git repository|HTTP 500\b"
88
    r"|unable to open database file|\bbad object\b|\binvalid object\b|unable to read (?:sha1 file|tree)"
89
    r"|\bearly EOF\b|Segmentation fault|object file \S+ is empty", re.I)
90
GENERIC_ERROR_CODES = {"error", "internal", "internal_error", "unknown", "unknown_error", "io", "io_error", "other",
91
                       "panic", "database", "database_error", "sqlite", "unexpected", "server_error", "http_error"}
92
93
94
def _json_error(stdout: str) -> Optional[dict]:
95
    try:
96
        data = json.loads(stdout)
97
    except (ValueError, TypeError):
98
        return None
99
    if isinstance(data, dict) and isinstance(data.get("error"), dict):
100
        return data["error"]
101
    return None
102
103
104
def actionable(res: Result) -> bool:
105
    """True when a failed command tells the user a concrete next step."""
106
    text = res.text()
107
    if res.returncode is None or res.returncode <= 0:
108
        return False
109
    # Corruption / internal / opaque-store wording wins over any recovery mention:
110
    # "loose object is corrupt ... git merge --abort" names a command but the
111
    # repository is broken, so the advice cannot be followed as given.
112
    if NON_ACTIONABLE_RE.search(text):
113
        return False
114
    if RECOVERY_COMMAND_RE.search(text) or RECOVERY_STEP_RE.search(text):
115
        return True
116
    if git_outcome_hint(res):
117
        return True
118
    error = _json_error(res.stdout)
119
    if error is not None:
120
        code = str(error.get("code") or "").strip().lower()
121
        if error.get("recommended_next_commands"):
122
            return True
123
        if code and code not in GENERIC_ERROR_CODES and not code.startswith(("internal", "unknown", "io_")):
124
            return True
125
    return bool(RETRYABLE_RE.search(text))
126
127
128
@dataclass
129
class Calibration:
130
    elapsed_ms: float
131
    post_tree: Optional[dict]
132
    post_worktree: Optional[dict]
133
    post_head: Optional[str]
134
    post_server_head: Optional[str]
135
    boundaries: int = 0
136
    boundary_kinds: list = field(default_factory=list)
137
    requests: list = field(default_factory=list)
138
    ok: bool = True
139
    detail: str = ""
140
141
142
class Trial:
143
    """One copied fixture + (for Oak) its loopback server, ready for one faulted op."""
144
145
    def __init__(self, fixture: Fixture, adapter_bin: str, work: Path, name: str):
146
        self.fixture = fixture
147
        self.dir = work / name
148
        copy_tree(fixture.pristine, self.dir)
149
        self.home = self.dir / "home"
150
        self.kind = fixture.kind
151
        if self.kind == "oak":
152
            self.adapter = OakAdapter(adapter_bin, self.home)
153
            self.server: Optional[Server] = Server(adapter_bin, self.dir / "srv", fixture.port or 0,
154
                                                   self.dir / "serve.log")
155
        else:
156
            self.adapter = GitAdapter(adapter_bin, self.home)
157
            self.server = None
158
        self.client = self.dir / "client"
159
160
    # op plumbing ---------------------------------------------------------------
161
    def op_cwd(self) -> Path:
162
        return self.dir if self.fixture.op == "clone" else self.client
163
164
    def op_argv(self, bin_override: Optional[str] = None, dest: str = "cloned", url: Optional[str] = None) -> list[str]:
165
        kw: dict[str, Any] = {"bin": bin_override}
166
        if self.fixture.op == "clone":
167
            kw.update(dest=dest, branch=self.fixture.branch, url=url or self.fixture.meta["url"])
168
        if self.fixture.op == "switch":
169
            kw["target"] = self.fixture.meta["target"]
170
        return self.adapter.argv(self.fixture.op, **kw)
171
172
    def pre(self) -> None:
173
        for argv in self.adapter.pre_argv(self.fixture.op):
174
            self.adapter.must(argv, self.client)
175
176
    def start_server(self, port: Optional[int] = None) -> None:
177
        if self.server is not None:
178
            self.server.start(port)
179
180
    def stop_server(self) -> None:
181
        if self.server is not None:
182
            self.server.stop()
183
184
    def settle(self) -> None:
185
        """Let orphaned server-side children (Git local receive-pack) finish."""
186
        if self.kind == "git":
187
            remote = self.dir / "srv" / "remote.git"
188
            deadline = time.monotonic() + 10
189
            while time.monotonic() < deadline and any(remote.rglob("*.lock")):
190
                time.sleep(0.05)
191
            time.sleep(0.1)
192
193
    def target_repo(self, dest: str = "cloned") -> Path:
194
        return self.dir / dest if self.fixture.op == "clone" else self.client
195
196
    def cleanup(self) -> None:
197
        self.stop_server()
198
        shutil.rmtree(self.dir, ignore_errors=True)
199
200
    # independent observations ---------------------------------------------------
201
    def server_head(self) -> Optional[str]:
202
        if self.kind == "oak":
203
            db = self.server.db() if self.server else None
204
            if not db or not db.exists():
205
                return None
206
            with OakGraph(db) as g:
207
                return g.branch_heads().get(self.fixture.branch)
208
        refs = self.adapter.oracle.refs(self.dir / "srv" / "remote.git")
209
        return refs.get(f"refs/heads/{self.fixture.branch}")
210
211
    def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
212
        if self.kind == "oak":
213
            from .readers import oak_commit_tree
214
            return oak_commit_tree(self.server.db(), commit)
215
        return self.adapter.oracle.commit_tree(self.dir / "srv" / "remote.git", commit)
216
217
    def repo_opens(self, repo: Path) -> Optional[bool]:
218
        if self.kind == "oak":
219
            db = repo / ".oak" / "oak.db"
220
            if not db.exists():
221
                return False
222
            try:
223
                with OakGraph(db) as g:
224
                    return g.integrity() == "ok" and g.metadata("current_branch") is not None
225
            except Exception:  # noqa: BLE001 - any failure to open is the finding
226
                return False
227
        if not (repo / ".git").exists():
228
            return False
229
        return self.adapter.oracle.fsck(repo) == "ok"
230
231
    def acked_reachable(self, repo: Path, heads: list[str]) -> tuple[Optional[bool], list[str]]:
232
        if not heads:
233
            return None, []
234
        try:
235
            if self.kind == "oak":
236
                with OakGraph(repo / ".oak" / "oak.db") as g:
237
                    reach = g.reachable()
238
                lost = [h for h in heads if h not in reach]
239
            else:
240
                lost = [h for h in heads if not self.adapter.oracle.reachable_from_refs(repo, h)]
241
        except Exception as exc:  # noqa: BLE001 - reader failure is unmeasured, never a subject violation
242
            return None, [f"reader error: {type(exc).__name__}: {exc}"]
243
        return not lost, lost
244
245
    def server_intact(self, intended: Optional[str]) -> tuple[Optional[bool], dict]:
246
        pre = self.fixture.meta.get("pre_server_heads", {})
247
        detail: dict = {}
248
        try:
249
            if self.kind == "oak":
250
                with OakGraph(self.server.db()) as g:
251
                    detail["integrity"] = g.integrity()
252
                    heads = g.branch_heads()
253
                    reach = g.ancestors(heads.values())
254
                    missing = g.missing_parents([h for h in heads.values()])
255
            else:
256
                remote = self.dir / "srv" / "remote.git"
257
                detail["integrity"] = self.adapter.oracle.fsck(remote)
258
                refs = self.adapter.oracle.refs(remote)
259
                heads = {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
260
                reach = set()
261
                for v in pre.values():
262
                    if any(self.adapter.oracle.is_ancestor(remote, v, h) for h in heads.values()):
263
                        reach.add(v)
264
                missing = []
265
        except Exception as exc:  # noqa: BLE001 - reader failure is unmeasured, never a subject violation
266
            return None, {"reader_error": f"{type(exc).__name__}: {exc}"}
267
        head = heads.get(self.fixture.branch)
268
        detail.update(server_head=head, lost_pre_heads=[v for v in pre.values() if v not in reach],
269
                      torn_graph=missing[:5])
270
        allowed = {pre.get(self.fixture.branch), intended}
271
        detail["head_in_allowed"] = head in allowed
272
        ok = detail["integrity"] == "ok" and not detail["lost_pre_heads"] and not missing and head in allowed
273
        return ok, detail
274
275
276
def worktree_verdict(trial: Trial, repo: Path, allowed: list[Optional[dict]]) -> tuple[Optional[bool], dict]:
277
    from .readers import worktree_manifest
278
279
    wt = worktree_manifest(repo)
280
    if wt is None:
281
        return None, {"worktree": "unreadable"}
282
    if any(a is not None and wt == a for a in allowed):
283
        return True, {"worktree": "pre_or_post"}
284
    head_tree, status = trial.adapter.tree(repo)
285
    if head_tree is not None and wt == head_tree:
286
        return True, {"worktree": "matches_head_tree"}
287
    dirty, res = trial.adapter.status_dirty(repo)
288
    if dirty is None:
289
        # status refused with an error: acceptable only if it is actionable
290
        ok = actionable(res)
291
        return ok, {"worktree": "differs; status errored", "status_actionable": ok, "status": res.brief(300),
292
                    "head_tree_reader": status}
293
    if dirty:
294
        return True, {"worktree": "differs; reported dirty", "head_tree_reader": status}
295
    if head_tree is None:
296
        return None, {"worktree": "differs; status clean; head tree unreadable", "head_tree_reader": status}
297
    return False, {"worktree": "SILENT_TORN: differs from HEAD tree while status reports clean"}
298
299
300
def torn_worktree(wt: Optional[dict], pre: Optional[dict], post: Optional[dict]) -> Optional[dict]:
301
    """Split a crashed worktree against the interrupted operation's two sides.
302
303
    Returns None when unmeasurable, else lists of paths the op changes that
304
    already hold the post-op (target) side (``op_output``) or still hold the
305
    pre-op side (``pre_side``), plus op artifacts in neither side (``stray``,
306
    e.g. temp files).
307
    """
308
    if wt is None or pre is None or post is None:
309
        return None
310
    changed_by_op = {p for p in set(pre) | set(post) if pre.get(p) != post.get(p)}
311
    op_output, pre_side, stray = [], [], []
312
    for path in sorted(set(wt) | changed_by_op):
313
        have = wt.get(path)
314
        if path in changed_by_op:
315
            if have == post.get(path):
316
                op_output.append(path)
317
            elif have == pre.get(path):
318
                pre_side.append(path)
319
        elif path not in pre and path not in post and have is not None:
320
            stray.append(path)
321
    return {"op_output": op_output, "pre_side": pre_side, "stray": stray}
322
323
324
def torn_state(split: dict, ref_side: Optional[str]) -> bool:
325
    """Is the worktree out of step with where the ref says the operation is?
326
327
    ref_side "pre": any of the op's output (or its temp files) is present.
328
    ref_side "post": any changed path still on the pre side (or temp files).
329
    unknown ref: the worktree mixes both sides, or has temp files.
330
    """
331
    if split["stray"]:
332
        return True
333
    if ref_side == "pre":
334
        return bool(split["op_output"])
335
    if ref_side == "post":
336
        return bool(split["pre_side"])
337
    return bool(split["op_output"]) and bool(split["pre_side"])
338
339
340
def torn_verdict(trial: "Trial", repo: Path, pre: Optional[dict], post: Optional[dict],
341
                 pre_head: Optional[str], op_res: Optional[Result] = None) -> tuple[Optional[bool], dict]:
342
    """Invariant ``worktree_not_torn``.
343
344
    Fails when, after the crash, the worktree holds the interrupted operation's
345
    own output out of step with the ref (e.g. files already on the target side
346
    while HEAD still names the old side, or the op's temp files) and the
347
    subject presents it as ordinary uncommitted user changes: status lists
348
    changes, reports no operation in progress, and does not refuse. Committing
349
    that state would record the op's partial output as the user's own work.
350
    Passing: worktree in step with the ref, an in-progress flag (MERGE_HEAD /
351
    progress_state / a stale git lock - index.lock, HEAD.lock or the current
352
    branch's ref lock - that makes the next commit refuse), a status
353
    refusal (judged by ``worktree_consistent``), or the failing op itself naming
354
    a recovery command (e.g. git's "checkout failed ... retry with git restore").
355
    A killed op prints nothing, so crash rows rely on the persistent flags only.
356
    """
357
    from .readers import worktree_manifest
358
359
    wt = worktree_manifest(repo)
360
    split = torn_worktree(wt, pre, post)
361
    if split is None:
362
        return None, {"torn": "unmeasurable"}
363
    head = trial.adapter.head(repo)
364
    # The op either leaves HEAD where it was or moves it to its result (a merge /
365
    # re-parent commit's hash differs per run, so "moved" is the post side).
366
    ref_side = None if head is None else ("pre" if head == pre_head else "post")
367
    detail = {"ref_side": ref_side, "op_output_files": len(split["op_output"]), "stray_files": len(split["stray"]),
368
              "pre_side_files": len(split["pre_side"]), "sample": (split["stray"] + split["op_output"])[:5]}
369
    if not torn_state(split, ref_side):
370
        return True, {**detail, "torn": "no"}
371
    if op_res is not None and (op_res.returncode or 0) > 0 and RECOVERY_COMMAND_RE.search(op_res.text()):
372
        return True, {**detail, "torn": "failing op named a recovery command"}
373
    changes, in_progress, res = trial.adapter.status_state(repo)
374
    detail["status_rc"] = res.returncode
375
    if changes is None:
376
        return True, {**detail, "torn": "status refused (judged by worktree_consistent)"}
377
    if in_progress:
378
        return True, {**detail, "torn": "flagged in progress"}
379
    if changes:
380
        return False, {**detail, "torn": "PARTIAL_OP_OUTPUT_AS_USER_CHANGES"}
381
    return True, {**detail, "torn": "status clean (judged by worktree_consistent)"}
382
383
384
def classify(invariants: dict[str, Optional[bool]], harness_error: Optional[str] = None,
385
             fault_fired: Optional[bool] = None, reader_errors: Optional[list] = None) -> str:
386
    """held / violated / harness_error / fault_not_fired.
387
388
    ``fault_fired=False`` (the fault demonstrably never took effect) is never
389
    ``held``: the row is a clean run. ``None`` means the family has no single
390
    fault to fire (concurrency). Reader exceptions are harness errors unless a
391
    subject-level invariant already failed independently of the reader.
392
    """
393
    if harness_error:
394
        return HARNESS_ERROR
395
    if fault_fired is False:
396
        return FAULT_NOT_FIRED
397
    if any(v is False for v in invariants.values()):
398
        return VIOLATED
399
    if reader_errors:
400
        return HARNESS_ERROR
401
    return HELD
402
403
404
def converge_check(trial: Trial, calib: Calibration, repo: Path) -> tuple[Optional[bool], dict]:
405
    """After a successful next op, is the state exactly the calibrated outcome?"""
406
    from .readers import worktree_manifest
407
408
    op = trial.fixture.op
409
    detail: dict = {}
410
    if op == "push":
411
        local = trial.adapter.head(repo)
412
        remote = trial.server_head()
413
        detail.update(local_head=local, server_head=remote)
414
        if remote is None or local is None or remote != local:
415
            return False, detail
416
        tree, status = trial.server_tree(remote)
417
        detail["server_tree_reader"] = status
418
        if tree is None:
419
            return None, detail
420
        return tree == calib.post_tree, detail
421
    if op == "switch":
422
        branch = trial.adapter.current_branch(repo)
423
        detail["branch"] = branch
424
        return branch == trial.fixture.meta["target"] and worktree_manifest(repo) == calib.post_worktree, detail
425
    tree, status = trial.adapter.tree(repo)
426
    detail["head_tree_reader"] = status
427
    if tree is None:
428
        return (False if status.startswith("corrupt") else None), detail
429
    same_tree = tree == calib.post_tree
430
    same_wt = worktree_manifest(repo) == calib.post_worktree
431
    detail.update(head_tree_matches=same_tree, worktree_matches=same_wt)
432
    if op == "clone":
433
        detail["head"] = trial.adapter.head(repo)
434
    return same_tree and same_wt, detail