Viewing files at commit c44c91118a6cCommit detailsView latest

suites/robustness/robustness/crash.py 242 lines · 10.8 KB · python
1
"""Crash-point family: kill -9 the client at time offsets and at every durability boundary."""
2
3
from __future__ import annotations
4
5
import time
6
from pathlib import Path
7
from typing import Callable, Optional
8
9
from . import FAULT_NOT_FIRED, HARNESS_ERROR
10
from .crashpoints import CrashInstrument, boundary_kinds, count_boundaries
11
from .engine import Calibration, Trial, actionable, classify, converge_check, torn_verdict, worktree_verdict
12
from .fixtures import Fixture
13
from .procs import run, run_killed_after
14
from .readers import worktree_manifest
15
16
TIME_FRACTIONS = (0.02, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.97)
17
MAX_SYSCALL_POINTS = 16
18
19
20
def calibrate(fixture: Fixture, subject_bin: str, work: Path, instrument: Optional[CrashInstrument]) -> Calibration:
21
    trial = Trial(fixture, subject_bin, work, "calibrate")
22
    try:
23
        trial.start_server()
24
        trial.pre()
25
        res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
26
        trial.settle()
27
        if res.returncode != 0:
28
            return Calibration(res.elapsed_ms, None, None, None, None, ok=False,
29
                               detail=f"uncrashed op failed rc={res.returncode}: {res.stderr[-300:]}")
30
        repo = trial.target_repo()
31
        tree, status = trial.adapter.tree(repo)
32
        calib = Calibration(res.elapsed_ms, tree, worktree_manifest(repo), trial.adapter.head(repo),
33
                            trial.server_head() if fixture.op in ("push", "clone", "pull_ff", "pull_diverged") else None,
34
                            detail=f"tree_reader={status}")
35
    finally:
36
        trial.cleanup()
37
    if instrument is not None:
38
        bin_copy = instrument.oak_bin if fixture.kind == "oak" else instrument.git_bin
39
        if bin_copy is not None:
40
            trial = Trial(fixture, subject_bin, work, "calibrate-syscalls")
41
            try:
42
                trial.start_server()
43
                trial.pre()
44
                log = trial.dir / "boundaries.log"
45
                res = trial.adapter.run(trial.op_argv(bin_override=str(bin_copy)), trial.op_cwd(),
46
                                        instrument.env(0, log, fixture.kind))
47
                calib.boundaries = count_boundaries(log) if res.returncode == 0 else 0
48
                calib.boundary_kinds = boundary_kinds(log)
49
            finally:
50
                trial.cleanup()
51
    return calib
52
53
54
def evaluate(trial: Trial, calib: Calibration, op_res, fault_fired: bool) -> tuple[dict, dict]:
55
    fx = trial.fixture
56
    inv: dict[str, Optional[bool]] = {}
57
    obs: dict = {"op": op_res.brief(400), "fault_fired": fault_fired}
58
    reader_errors: list[str] = []
59
    inv["op_no_hang"] = not op_res.timed_out
60
    trial.settle()
61
    repo = trial.target_repo()
62
    if fx.op == "clone":
63
        exists = (repo / (".oak" if trial.kind == "oak" else ".git")).exists()
64
        obs["clone_dest"] = "absent" if not repo.exists() else ("repo" if exists else "partial_without_store")
65
        inv["repo_opens"] = None
66
        inv["worktree_consistent"] = None
67
        if exists and trial.repo_opens(repo):
68
            inv["repo_opens"] = True
69
            ok, detail = worktree_verdict(trial, repo, [calib.post_worktree])
70
            inv["worktree_consistent"] = ok
71
            obs["worktree"] = detail
72
            ok, detail = torn_verdict(trial, repo, {}, calib.post_worktree, None, op_res)
73
            inv["worktree_not_torn"] = ok
74
            obs["torn"] = detail
75
        elif repo.exists():
76
            # A torn clone is fine only if the subject recognises it: its status must
77
            # refuse with an actionable error (e.g. "remove this directory and re-clone").
78
            dirty, st = trial.adapter.status_dirty(repo)
79
            obs["partial_clone_status"] = st.brief(300)
80
            inv["partial_clone_detected"] = dirty is None and actionable(st)
81
    else:
82
        inv["repo_opens"] = trial.repo_opens(repo)
83
        ok, lost = trial.acked_reachable(repo, fx.meta.get("pre_local_heads", []))
84
        inv["acked_local_reachable"] = ok
85
        if ok is None and lost:
86
            reader_errors += lost
87
        elif lost:
88
            obs["lost_local_heads"] = lost
89
        pre_wt = worktree_manifest(trial.fixture.pristine / "client")
90
        ok, detail = worktree_verdict(trial, repo, [pre_wt, calib.post_worktree])
91
        inv["worktree_consistent"] = ok
92
        obs["worktree"] = detail
93
        ok, detail = torn_verdict(trial, repo, pre_wt, calib.post_worktree, fx.meta.get("pre_head"), op_res)
94
        inv["worktree_not_torn"] = ok
95
        obs["torn"] = detail
96
    if trial.server is not None or trial.kind == "git":
97
        intended = calib.post_server_head if fx.op == "push" else None
98
        ok, detail = trial.server_intact(intended)
99
        inv["server_intact"] = ok
100
        obs["server"] = detail
101
        if "reader_error" in detail:
102
            reader_errors.append("server: " + detail["reader_error"])
103
    # next operation (no fault)
104
    if fx.op == "clone":
105
        nxt = trial.adapter.run(trial.op_argv(dest="cloned2"), trial.op_cwd())
106
        next_repo = trial.dir / "cloned2"
107
        if (trial.dir / "cloned").exists() and inv.get("repo_opens"):
108
            st_dirty, st_res = trial.adapter.status_dirty(trial.dir / "cloned")
109
            obs["crashed_clone_status"] = {"dirty": st_dirty, "rc": st_res.returncode}
110
    else:
111
        nxt = None
112
        for argv in trial.adapter.pre_argv(fx.op):
113
            staged = trial.adapter.run(argv, trial.client)
114
            if staged.returncode != 0:
115
                nxt = staged  # e.g. a stale index.lock blocks staging: judged like the op itself
116
                break
117
        if nxt is None:
118
            nxt = trial.adapter.run(trial.op_argv(), trial.op_cwd())
119
        next_repo = repo
120
    obs["next_op"] = nxt.brief(400)
121
    if nxt.timed_out:
122
        inv["next_op_ok_or_actionable"] = False
123
    elif nxt.returncode == 0:
124
        inv["next_op_ok_or_actionable"] = True
125
        trial.settle()
126
        ok, detail = converge_check(trial, calib, next_repo)
127
        inv["no_false_success"] = ok
128
        obs["converge"] = detail
129
    else:
130
        inv["next_op_ok_or_actionable"] = actionable(nxt)
131
        obs["recovery_required"] = True
132
    if fx.op != "clone":
133
        ok, lost = trial.acked_reachable(repo, fx.meta.get("pre_local_heads", []))
134
        inv["acked_after_next"] = ok
135
        if ok is None and lost:
136
            reader_errors += lost
137
        elif lost:
138
            obs["lost_after_next"] = lost
139
    if reader_errors:
140
        obs["reader_errors"] = reader_errors[:5]
141
    return inv, obs
142
143
144
def run_trial(fixture: Fixture, subject_bin: str, work: Path, name: str, calib: Calibration,
145
              fault: Callable[[Trial], tuple], keep: Callable[[Trial, str], None]) -> dict:
146
    """fault(trial) -> (op_result, fault_fired, fault_point). Returns a partial row."""
147
    trial = Trial(fixture, subject_bin, work, name)
148
    started = time.perf_counter()
149
    try:
150
        trial.start_server()
151
        trial.pre()
152
        op_res, fired, point = fault(trial)
153
        inv, obs = evaluate(trial, calib, op_res, fired)
154
        classification = classify(inv, fault_fired=fired, reader_errors=obs.get("reader_errors"))
155
        error = None
156
    except Exception as exc:  # noqa: BLE001 - recorded as harness_error, never hidden
157
        inv, obs, classification, error, point, op_res = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}", "?", None
158
        fired = None
159
    finally:
160
        trial.stop_server()
161
    row = {
162
        "fault_point": point,
163
        "fault_fired": fired,
164
        "invariants": inv,
165
        "violations": sorted(k for k, v in inv.items() if v is False),
166
        "observations": obs,
167
        "classification": classification,
168
        "elapsed_ms": round(op_res.elapsed_ms if op_res else (time.perf_counter() - started) * 1000.0, 3),
169
        "command": op_res.argv if op_res else [],
170
    }
171
    if error:
172
        row["harness_error"] = error
173
    elif classification == HARNESS_ERROR and obs.get("reader_errors"):
174
        row["harness_error"] = "independent reader failed: " + "; ".join(obs["reader_errors"][:2])
175
    if classification == FAULT_NOT_FIRED:
176
        row["skip_reason"] = not_fired_reason(op_res, point)
177
    keep(trial, classification)
178
    return row
179
180
181
def not_fired_reason(op_res, point: str) -> str:
182
    if op_res is None:
183
        return f"fault at {point} did not fire"
184
    return (f"fault at {point} did not fire: the operation exited rc={op_res.returncode} after "
185
            f"{op_res.elapsed_ms:.1f}ms without being interrupted")
186
187
188
def time_fault(delay_s: float, argv_fn=None):
189
    def fault(trial: Trial):
190
        res = run_killed_after(trial.op_argv(), trial.op_cwd(), trial.adapter.env(), delay_s)
191
        # Fired only with evidence: the client died of our SIGKILL. A kill sent to an
192
        # already-exited (zombie) client leaves its real exit status.
193
        return res, res.returncode == -9 and not res.timed_out, f"t={delay_s * 1000:.1f}ms"
194
    return fault
195
196
197
TIME_RETRIES = 2
198
199
200
def run_time_point(fixture: Fixture, subject_bin: str, work: Path, name: str, calib: Calibration, frac: float,
201
                   keep: Callable[[Trial, str], None]) -> dict:
202
    """Kill at ``frac`` of the op's duration; if the op beat the kill, re-aim from its observed duration.
203
204
    Calibration and trial durations differ run to run, so a late fraction can
205
    land after the op exited. Each retry uses a fresh copy of the fixture and
206
    the duration the previous attempt actually took; if the kill still never
207
    lands, the row is ``fault_not_fired`` (never ``held``).
208
    """
209
    delay = max(0.002, calib.elapsed_ms / 1000.0 * frac)
210
    attempts = []
211
    row: dict = {}
212
    for attempt in range(TIME_RETRIES + 1):
213
        row = run_trial(fixture, subject_bin, work, f"{name}-a{attempt}", calib, time_fault(delay), keep)
214
        attempts.append({"delay_ms": round(delay * 1000, 1), "fired": row.get("fault_fired"),
215
                         "op_elapsed_ms": row.get("elapsed_ms")})
216
        if row.get("fault_fired") is not False:
217
            break
218
        observed = float(row.get("elapsed_ms") or 0.0) / 1000.0
219
        delay = max(0.002, min(delay, observed) * frac * (0.9 ** attempt))
220
    row.setdefault("observations", {})["time_attempts"] = attempts
221
    row["observations"]["target_fraction"] = frac
222
    return row
223
224
225
def syscall_fault(instrument: CrashInstrument, n: int, kinds: list[str]):
226
    def fault(trial: Trial):
227
        bin_copy = instrument.oak_bin if trial.kind == "oak" else instrument.git_bin
228
        log = trial.dir / "boundaries.log"
229
        res = trial.adapter.run(trial.op_argv(bin_override=str(bin_copy)), trial.op_cwd(),
230
                                instrument.env(n, log, trial.kind))
231
        fired = res.returncode == -9
232
        kind = kinds[n - 1] if 0 < n <= len(kinds) else "?"
233
        return res, fired, f"before {kind}#{n}"
234
    return fault
235
236
237
def syscall_points(boundaries: int) -> list[int]:
238
    if boundaries <= MAX_SYSCALL_POINTS:
239
        return list(range(1, boundaries + 1))
240
    step = boundaries / MAX_SYSCALL_POINTS
241
    points = sorted({max(1, int(round(i * step))) for i in range(1, MAX_SYSCALL_POINTS + 1)})
242
    return points