Viewing files at commit f2acb751aa96Commit detailsView latest

suites/robustness/robustness/engine.py 265 lines · 11.3 KB · python
1
"""Trial engine: copy a pristine fixture, inject one fault, judge invariants.
2
3
Invariants (each True / False / None=unmeasured, ADR-0002):
4
5
- ``op_no_hang``            the faulted operation ended (killed or exited) within its timeout
6
- ``repo_opens``            independent open: SQLite integrity_check (Oak) / git fsck (Git)
7
- ``acked_local_reachable`` every commit a local ref named before the op is still reachable
8
                            from a local ref (branch head / HEAD / tag), per the independent reader
9
- ``server_intact``         remote store opens, its pre-op heads stay reachable, and the branch
10
                            head is either the pre-op head or the operation's intended head
11
- ``worktree_consistent``   the worktree equals the pre- or post-op state, or a committed tree,
12
                            or the subject's status reports it dirty/in-progress (never "clean"
13
                            while differing from HEAD)
14
- ``next_op_ok_or_actionable`` rerunning the operation succeeds, or fails with a typed/actionable
15
                            message (not a hang, panic, or signal)
16
- ``no_false_success``      a reported success (exit 0) leaves exactly the calibrated outcome
17
- ``acked_after_next``      pre-op local commits are still reachable after the next operation
18
"""
19
20
from __future__ import annotations
21
22
import json
23
import re
24
import shutil
25
import time
26
from dataclasses import dataclass, field
27
from pathlib import Path
28
from typing import Any, Callable, Optional
29
30
from . import HARNESS_ERROR, HELD, VIOLATED
31
from .adapters import GitAdapter, OakAdapter, Server
32
from .fixtures import Fixture
33
from .procs import Result, copy_tree
34
from .readers import OakGraph
35
36
ACTIONABLE_RE = re.compile(
37
    r"\b(oak|git)\s+[a-z][\w-]*|manually|--continue|--abort|try again|retry|\bpull\b|re-?run|remove this|rejected|move or remove", re.I)
38
39
40
def actionable(res: Result) -> bool:
41
    text = res.text()
42
    if res.returncode is None or res.returncode < 0:
43
        return False
44
    if "panicked at" in text or "RUST_BACKTRACE" in text:
45
        return False
46
    try:
47
        data = json.loads(res.stdout)
48
        if isinstance(data, dict) and isinstance(data.get("error"), dict) and data["error"].get("code"):
49
            return True
50
    except (ValueError, TypeError):
51
        pass
52
    return bool(text.strip()) and bool(ACTIONABLE_RE.search(text))
53
54
55
@dataclass
56
class Calibration:
57
    elapsed_ms: float
58
    post_tree: Optional[dict]
59
    post_worktree: Optional[dict]
60
    post_head: Optional[str]
61
    post_server_head: Optional[str]
62
    boundaries: int = 0
63
    boundary_kinds: list = field(default_factory=list)
64
    requests: list = field(default_factory=list)
65
    ok: bool = True
66
    detail: str = ""
67
68
69
class Trial:
70
    """One copied fixture + (for Oak) its loopback server, ready for one faulted op."""
71
72
    def __init__(self, fixture: Fixture, adapter_bin: str, work: Path, name: str):
73
        self.fixture = fixture
74
        self.dir = work / name
75
        copy_tree(fixture.pristine, self.dir)
76
        self.home = self.dir / "home"
77
        self.kind = fixture.kind
78
        if self.kind == "oak":
79
            self.adapter = OakAdapter(adapter_bin, self.home)
80
            self.server: Optional[Server] = Server(adapter_bin, self.dir / "srv", fixture.port or 0,
81
                                                   self.dir / "serve.log")
82
        else:
83
            self.adapter = GitAdapter(adapter_bin, self.home)
84
            self.server = None
85
        self.client = self.dir / "client"
86
87
    # op plumbing ---------------------------------------------------------------
88
    def op_cwd(self) -> Path:
89
        return self.dir if self.fixture.op == "clone" else self.client
90
91
    def op_argv(self, bin_override: Optional[str] = None, dest: str = "cloned", url: Optional[str] = None) -> list[str]:
92
        kw: dict[str, Any] = {"bin": bin_override}
93
        if self.fixture.op == "clone":
94
            kw.update(dest=dest, branch=self.fixture.branch, url=url or self.fixture.meta["url"])
95
        if self.fixture.op == "switch":
96
            kw["target"] = self.fixture.meta["target"]
97
        return self.adapter.argv(self.fixture.op, **kw)
98
99
    def pre(self) -> None:
100
        for argv in self.adapter.pre_argv(self.fixture.op):
101
            self.adapter.must(argv, self.client)
102
103
    def start_server(self, port: Optional[int] = None) -> None:
104
        if self.server is not None:
105
            self.server.start(port)
106
107
    def stop_server(self) -> None:
108
        if self.server is not None:
109
            self.server.stop()
110
111
    def settle(self) -> None:
112
        """Let orphaned server-side children (Git local receive-pack) finish."""
113
        if self.kind == "git":
114
            remote = self.dir / "srv" / "remote.git"
115
            deadline = time.monotonic() + 10
116
            while time.monotonic() < deadline and any(remote.rglob("*.lock")):
117
                time.sleep(0.05)
118
            time.sleep(0.1)
119
120
    def target_repo(self, dest: str = "cloned") -> Path:
121
        return self.dir / dest if self.fixture.op == "clone" else self.client
122
123
    def cleanup(self) -> None:
124
        self.stop_server()
125
        shutil.rmtree(self.dir, ignore_errors=True)
126
127
    # independent observations ---------------------------------------------------
128
    def server_head(self) -> Optional[str]:
129
        if self.kind == "oak":
130
            db = self.server.db() if self.server else None
131
            if not db or not db.exists():
132
                return None
133
            with OakGraph(db) as g:
134
                return g.branch_heads().get(self.fixture.branch)
135
        refs = self.adapter.oracle.refs(self.dir / "srv" / "remote.git")
136
        return refs.get(f"refs/heads/{self.fixture.branch}")
137
138
    def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
139
        if self.kind == "oak":
140
            from .readers import oak_commit_tree
141
            return oak_commit_tree(self.server.db(), commit)
142
        return self.adapter.oracle.commit_tree(self.dir / "srv" / "remote.git", commit)
143
144
    def repo_opens(self, repo: Path) -> Optional[bool]:
145
        if self.kind == "oak":
146
            db = repo / ".oak" / "oak.db"
147
            if not db.exists():
148
                return False
149
            try:
150
                with OakGraph(db) as g:
151
                    return g.integrity() == "ok" and g.metadata("current_branch") is not None
152
            except Exception:  # noqa: BLE001 - any failure to open is the finding
153
                return False
154
        if not (repo / ".git").exists():
155
            return False
156
        return self.adapter.oracle.fsck(repo) == "ok"
157
158
    def acked_reachable(self, repo: Path, heads: list[str]) -> tuple[Optional[bool], list[str]]:
159
        if not heads:
160
            return None, []
161
        try:
162
            if self.kind == "oak":
163
                with OakGraph(repo / ".oak" / "oak.db") as g:
164
                    reach = g.reachable()
165
                lost = [h for h in heads if h not in reach]
166
            else:
167
                lost = [h for h in heads if not self.adapter.oracle.reachable_from_refs(repo, h)]
168
        except Exception as exc:  # noqa: BLE001
169
            return False, [f"reader error: {exc}"]
170
        return not lost, lost
171
172
    def server_intact(self, intended: Optional[str]) -> tuple[Optional[bool], dict]:
173
        pre = self.fixture.meta.get("pre_server_heads", {})
174
        detail: dict = {}
175
        try:
176
            if self.kind == "oak":
177
                with OakGraph(self.server.db()) as g:
178
                    detail["integrity"] = g.integrity()
179
                    heads = g.branch_heads()
180
                    reach = g.ancestors(heads.values())
181
                    missing = g.missing_parents([h for h in heads.values()])
182
            else:
183
                remote = self.dir / "srv" / "remote.git"
184
                detail["integrity"] = self.adapter.oracle.fsck(remote)
185
                refs = self.adapter.oracle.refs(remote)
186
                heads = {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
187
                reach = set()
188
                for v in pre.values():
189
                    if any(self.adapter.oracle.is_ancestor(remote, v, h) for h in heads.values()):
190
                        reach.add(v)
191
                missing = []
192
        except Exception as exc:  # noqa: BLE001
193
            return False, {"error": str(exc)}
194
        head = heads.get(self.fixture.branch)
195
        detail.update(server_head=head, lost_pre_heads=[v for v in pre.values() if v not in reach],
196
                      torn_graph=missing[:5])
197
        allowed = {pre.get(self.fixture.branch), intended}
198
        detail["head_in_allowed"] = head in allowed
199
        ok = detail["integrity"] == "ok" and not detail["lost_pre_heads"] and not missing and head in allowed
200
        return ok, detail
201
202
203
def worktree_verdict(trial: Trial, repo: Path, allowed: list[Optional[dict]]) -> tuple[Optional[bool], dict]:
204
    from .readers import worktree_manifest
205
206
    wt = worktree_manifest(repo)
207
    if wt is None:
208
        return None, {"worktree": "unreadable"}
209
    if any(a is not None and wt == a for a in allowed):
210
        return True, {"worktree": "pre_or_post"}
211
    head_tree, status = trial.adapter.tree(repo)
212
    if head_tree is not None and wt == head_tree:
213
        return True, {"worktree": "matches_head_tree"}
214
    dirty, res = trial.adapter.status_dirty(repo)
215
    if dirty is None:
216
        # status refused with an error: acceptable only if it is actionable
217
        ok = actionable(res)
218
        return ok, {"worktree": "differs; status errored", "status_actionable": ok, "status": res.brief(300),
219
                    "head_tree_reader": status}
220
    if dirty:
221
        return True, {"worktree": "differs; reported dirty", "head_tree_reader": status}
222
    if head_tree is None:
223
        return None, {"worktree": "differs; status clean; head tree unreadable", "head_tree_reader": status}
224
    return False, {"worktree": "SILENT_TORN: differs from HEAD tree while status reports clean"}
225
226
227
def classify(invariants: dict[str, Optional[bool]], harness_error: Optional[str] = None) -> str:
228
    if harness_error:
229
        return HARNESS_ERROR
230
    if any(v is False for v in invariants.values()):
231
        return VIOLATED
232
    return HELD
233
234
235
def converge_check(trial: Trial, calib: Calibration, repo: Path) -> tuple[Optional[bool], dict]:
236
    """After a successful next op, is the state exactly the calibrated outcome?"""
237
    from .readers import worktree_manifest
238
239
    op = trial.fixture.op
240
    detail: dict = {}
241
    if op == "push":
242
        local = trial.adapter.head(repo)
243
        remote = trial.server_head()
244
        detail.update(local_head=local, server_head=remote)
245
        if remote is None or local is None or remote != local:
246
            return False, detail
247
        tree, status = trial.server_tree(remote)
248
        detail["server_tree_reader"] = status
249
        if tree is None:
250
            return None, detail
251
        return tree == calib.post_tree, detail
252
    if op == "switch":
253
        branch = trial.adapter.current_branch(repo)
254
        detail["branch"] = branch
255
        return branch == trial.fixture.meta["target"] and worktree_manifest(repo) == calib.post_worktree, detail
256
    tree, status = trial.adapter.tree(repo)
257
    detail["head_tree_reader"] = status
258
    if tree is None:
259
        return (False if status.startswith("corrupt") else None), detail
260
    same_tree = tree == calib.post_tree
261
    same_wt = worktree_manifest(repo) == calib.post_worktree
262
    detail.update(head_tree_matches=same_tree, worktree_matches=same_wt)
263
    if op == "clone":
264
        detail["head"] = trial.adapter.head(repo)
265
    return same_tree and same_wt, detail