| 1 | """Crash-point family: kill -9 the client at time offsets and at every durability boundary."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import time
|
| 6 | from pathlib import Path
|
| 7 | from typing import Callable, Optional
|
| 8 |
|
| 9 | from . import FAULT_NOT_FIRED, HARNESS_ERROR
|
| 10 | from .crashpoints import CrashInstrument, boundary_kinds, count_boundaries
|
| 11 | from .engine import Calibration, Trial, actionable, classify, converge_check, torn_verdict, worktree_verdict
|
| 12 | from .fixtures import Fixture
|
| 13 | from .procs import run, run_killed_after
|
| 14 | from .readers import worktree_manifest
|
| 15 |
|
| 16 | TIME_FRACTIONS = (0.02, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.97)
|
| 17 | MAX_SYSCALL_POINTS = 16
|
| 18 |
|
| 19 |
|
| 20 | def calibrate(fixture: Fixture, subject_bin: str, work: Path, instrument: Optional[CrashInstrument]) -> Calibration:
|
| 21 | trial = Trial(fixture, subject_bin, work, "calibrate")
|
| 22 | try:
|
| 23 | trial.start_server()
|
| 24 | trial.pre()
|
| 25 | res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
|
| 26 | trial.settle()
|
| 27 | if res.returncode != 0:
|
| 28 | return Calibration(res.elapsed_ms, None, None, None, None, ok=False,
|
| 29 | detail=f"uncrashed op failed rc={res.returncode}: {res.stderr[-300:]}")
|
| 30 | repo = trial.target_repo()
|
| 31 | tree, status = trial.adapter.tree(repo)
|
| 32 | calib = Calibration(res.elapsed_ms, tree, worktree_manifest(repo), trial.adapter.head(repo),
|
| 33 | trial.server_head() if fixture.op in ("push", "clone", "pull_ff", "pull_diverged") else None,
|
| 34 | detail=f"tree_reader={status}")
|
| 35 | finally:
|
| 36 | trial.cleanup()
|
| 37 | if instrument is not None:
|
| 38 | bin_copy = instrument.oak_bin if fixture.kind == "oak" else instrument.git_bin
|
| 39 | if bin_copy is not None:
|
| 40 | trial = Trial(fixture, subject_bin, work, "calibrate-syscalls")
|
| 41 | try:
|
| 42 | trial.start_server()
|
| 43 | trial.pre()
|
| 44 | log = trial.dir / "boundaries.log"
|
| 45 | res = trial.adapter.run(trial.op_argv(bin_override=str(bin_copy)), trial.op_cwd(),
|
| 46 | instrument.env(0, log, fixture.kind))
|
| 47 | calib.boundaries = count_boundaries(log) if res.returncode == 0 else 0
|
| 48 | calib.boundary_kinds = boundary_kinds(log)
|
| 49 | finally:
|
| 50 | trial.cleanup()
|
| 51 | return calib
|
| 52 |
|
| 53 |
|
| 54 | def evaluate(trial: Trial, calib: Calibration, op_res, fault_fired: bool) -> tuple[dict, dict]:
|
| 55 | fx = trial.fixture
|
| 56 | inv: dict[str, Optional[bool]] = {}
|
| 57 | obs: dict = {"op": op_res.brief(400), "fault_fired": fault_fired}
|
| 58 | reader_errors: list[str] = []
|
| 59 | inv["op_no_hang"] = not op_res.timed_out
|
| 60 | trial.settle()
|
| 61 | repo = trial.target_repo()
|
| 62 | if fx.op == "clone":
|
| 63 | exists = (repo / (".oak" if trial.kind == "oak" else ".git")).exists()
|
| 64 | obs["clone_dest"] = "absent" if not repo.exists() else ("repo" if exists else "partial_without_store")
|
| 65 | inv["repo_opens"] = None
|
| 66 | inv["worktree_consistent"] = None
|
| 67 | if exists and trial.repo_opens(repo):
|
| 68 | inv["repo_opens"] = True
|
| 69 | ok, detail = worktree_verdict(trial, repo, [calib.post_worktree])
|
| 70 | inv["worktree_consistent"] = ok
|
| 71 | obs["worktree"] = detail
|
| 72 | ok, detail = torn_verdict(trial, repo, {}, calib.post_worktree, None, op_res)
|
| 73 | inv["worktree_not_torn"] = ok
|
| 74 | obs["torn"] = detail
|
| 75 | elif repo.exists():
|
| 76 | # A torn clone is fine only if the subject recognises it: its status must
|
| 77 | # refuse with an actionable error (e.g. "remove this directory and re-clone").
|
| 78 | dirty, st = trial.adapter.status_dirty(repo)
|
| 79 | obs["partial_clone_status"] = st.brief(300)
|
| 80 | inv["partial_clone_detected"] = dirty is None and actionable(st)
|
| 81 | else:
|
| 82 | inv["repo_opens"] = trial.repo_opens(repo)
|
| 83 | ok, lost = trial.acked_reachable(repo, fx.meta.get("pre_local_heads", []))
|
| 84 | inv["acked_local_reachable"] = ok
|
| 85 | if ok is None and lost:
|
| 86 | reader_errors += lost
|
| 87 | elif lost:
|
| 88 | obs["lost_local_heads"] = lost
|
| 89 | pre_wt = worktree_manifest(trial.fixture.pristine / "client")
|
| 90 | ok, detail = worktree_verdict(trial, repo, [pre_wt, calib.post_worktree])
|
| 91 | inv["worktree_consistent"] = ok
|
| 92 | obs["worktree"] = detail
|
| 93 | ok, detail = torn_verdict(trial, repo, pre_wt, calib.post_worktree, fx.meta.get("pre_head"), op_res)
|
| 94 | inv["worktree_not_torn"] = ok
|
| 95 | obs["torn"] = detail
|
| 96 | if trial.server is not None or trial.kind == "git":
|
| 97 | intended = calib.post_server_head if fx.op == "push" else None
|
| 98 | ok, detail = trial.server_intact(intended)
|
| 99 | inv["server_intact"] = ok
|
| 100 | obs["server"] = detail
|
| 101 | if "reader_error" in detail:
|
| 102 | reader_errors.append("server: " + detail["reader_error"])
|
| 103 | # next operation (no fault)
|
| 104 | if fx.op == "clone":
|
| 105 | nxt = trial.adapter.run(trial.op_argv(dest="cloned2"), trial.op_cwd())
|
| 106 | next_repo = trial.dir / "cloned2"
|
| 107 | if (trial.dir / "cloned").exists() and inv.get("repo_opens"):
|
| 108 | st_dirty, st_res = trial.adapter.status_dirty(trial.dir / "cloned")
|
| 109 | obs["crashed_clone_status"] = {"dirty": st_dirty, "rc": st_res.returncode}
|
| 110 | else:
|
| 111 | nxt = None
|
| 112 | for argv in trial.adapter.pre_argv(fx.op):
|
| 113 | staged = trial.adapter.run(argv, trial.client)
|
| 114 | if staged.returncode != 0:
|
| 115 | nxt = staged # e.g. a stale index.lock blocks staging: judged like the op itself
|
| 116 | break
|
| 117 | if nxt is None:
|
| 118 | nxt = trial.adapter.run(trial.op_argv(), trial.op_cwd())
|
| 119 | next_repo = repo
|
| 120 | obs["next_op"] = nxt.brief(400)
|
| 121 | if nxt.timed_out:
|
| 122 | inv["next_op_ok_or_actionable"] = False
|
| 123 | elif nxt.returncode == 0:
|
| 124 | inv["next_op_ok_or_actionable"] = True
|
| 125 | trial.settle()
|
| 126 | ok, detail = converge_check(trial, calib, next_repo)
|
| 127 | inv["no_false_success"] = ok
|
| 128 | obs["converge"] = detail
|
| 129 | else:
|
| 130 | inv["next_op_ok_or_actionable"] = actionable(nxt)
|
| 131 | obs["recovery_required"] = True
|
| 132 | if fx.op != "clone":
|
| 133 | ok, lost = trial.acked_reachable(repo, fx.meta.get("pre_local_heads", []))
|
| 134 | inv["acked_after_next"] = ok
|
| 135 | if ok is None and lost:
|
| 136 | reader_errors += lost
|
| 137 | elif lost:
|
| 138 | obs["lost_after_next"] = lost
|
| 139 | if reader_errors:
|
| 140 | obs["reader_errors"] = reader_errors[:5]
|
| 141 | return inv, obs
|
| 142 |
|
| 143 |
|
| 144 | def run_trial(fixture: Fixture, subject_bin: str, work: Path, name: str, calib: Calibration,
|
| 145 | fault: Callable[[Trial], tuple], keep: Callable[[Trial, str], None]) -> dict:
|
| 146 | """fault(trial) -> (op_result, fault_fired, fault_point). Returns a partial row."""
|
| 147 | trial = Trial(fixture, subject_bin, work, name)
|
| 148 | started = time.perf_counter()
|
| 149 | try:
|
| 150 | trial.start_server()
|
| 151 | trial.pre()
|
| 152 | op_res, fired, point = fault(trial)
|
| 153 | inv, obs = evaluate(trial, calib, op_res, fired)
|
| 154 | classification = classify(inv, fault_fired=fired, reader_errors=obs.get("reader_errors"))
|
| 155 | error = None
|
| 156 | except Exception as exc: # noqa: BLE001 - recorded as harness_error, never hidden
|
| 157 | inv, obs, classification, error, point, op_res = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}", "?", None
|
| 158 | fired = None
|
| 159 | finally:
|
| 160 | trial.stop_server()
|
| 161 | row = {
|
| 162 | "fault_point": point,
|
| 163 | "fault_fired": fired,
|
| 164 | "invariants": inv,
|
| 165 | "violations": sorted(k for k, v in inv.items() if v is False),
|
| 166 | "observations": obs,
|
| 167 | "classification": classification,
|
| 168 | "elapsed_ms": round(op_res.elapsed_ms if op_res else (time.perf_counter() - started) * 1000.0, 3),
|
| 169 | "command": op_res.argv if op_res else [],
|
| 170 | }
|
| 171 | if error:
|
| 172 | row["harness_error"] = error
|
| 173 | elif classification == HARNESS_ERROR and obs.get("reader_errors"):
|
| 174 | row["harness_error"] = "independent reader failed: " + "; ".join(obs["reader_errors"][:2])
|
| 175 | if classification == FAULT_NOT_FIRED:
|
| 176 | row["skip_reason"] = not_fired_reason(op_res, point)
|
| 177 | keep(trial, classification)
|
| 178 | return row
|
| 179 |
|
| 180 |
|
| 181 | def not_fired_reason(op_res, point: str) -> str:
|
| 182 | if op_res is None:
|
| 183 | return f"fault at {point} did not fire"
|
| 184 | return (f"fault at {point} did not fire: the operation exited rc={op_res.returncode} after "
|
| 185 | f"{op_res.elapsed_ms:.1f}ms without being interrupted")
|
| 186 |
|
| 187 |
|
| 188 | def time_fault(delay_s: float, argv_fn=None):
|
| 189 | def fault(trial: Trial):
|
| 190 | res = run_killed_after(trial.op_argv(), trial.op_cwd(), trial.adapter.env(), delay_s)
|
| 191 | # Fired only with evidence: the client died of our SIGKILL. A kill sent to an
|
| 192 | # already-exited (zombie) client leaves its real exit status.
|
| 193 | return res, res.returncode == -9 and not res.timed_out, f"t={delay_s * 1000:.1f}ms"
|
| 194 | return fault
|
| 195 |
|
| 196 |
|
| 197 | TIME_RETRIES = 2
|
| 198 |
|
| 199 |
|
| 200 | def run_time_point(fixture: Fixture, subject_bin: str, work: Path, name: str, calib: Calibration, frac: float,
|
| 201 | keep: Callable[[Trial, str], None]) -> dict:
|
| 202 | """Kill at ``frac`` of the op's duration; if the op beat the kill, re-aim from its observed duration.
|
| 203 |
|
| 204 | Calibration and trial durations differ run to run, so a late fraction can
|
| 205 | land after the op exited. Each retry uses a fresh copy of the fixture and
|
| 206 | the duration the previous attempt actually took; if the kill still never
|
| 207 | lands, the row is ``fault_not_fired`` (never ``held``).
|
| 208 | """
|
| 209 | delay = max(0.002, calib.elapsed_ms / 1000.0 * frac)
|
| 210 | attempts = []
|
| 211 | row: dict = {}
|
| 212 | for attempt in range(TIME_RETRIES + 1):
|
| 213 | row = run_trial(fixture, subject_bin, work, f"{name}-a{attempt}", calib, time_fault(delay), keep)
|
| 214 | attempts.append({"delay_ms": round(delay * 1000, 1), "fired": row.get("fault_fired"),
|
| 215 | "op_elapsed_ms": row.get("elapsed_ms")})
|
| 216 | if row.get("fault_fired") is not False:
|
| 217 | break
|
| 218 | observed = float(row.get("elapsed_ms") or 0.0) / 1000.0
|
| 219 | delay = max(0.002, min(delay, observed) * frac * (0.9 ** attempt))
|
| 220 | row.setdefault("observations", {})["time_attempts"] = attempts
|
| 221 | row["observations"]["target_fraction"] = frac
|
| 222 | return row
|
| 223 |
|
| 224 |
|
| 225 | def syscall_fault(instrument: CrashInstrument, n: int, kinds: list[str]):
|
| 226 | def fault(trial: Trial):
|
| 227 | bin_copy = instrument.oak_bin if trial.kind == "oak" else instrument.git_bin
|
| 228 | log = trial.dir / "boundaries.log"
|
| 229 | res = trial.adapter.run(trial.op_argv(bin_override=str(bin_copy)), trial.op_cwd(),
|
| 230 | instrument.env(n, log, trial.kind))
|
| 231 | fired = res.returncode == -9
|
| 232 | kind = kinds[n - 1] if 0 < n <= len(kinds) else "?"
|
| 233 | return res, fired, f"before {kind}#{n}"
|
| 234 | return fault
|
| 235 |
|
| 236 |
|
| 237 | def syscall_points(boundaries: int) -> list[int]:
|
| 238 | if boundaries <= MAX_SYSCALL_POINTS:
|
| 239 | return list(range(1, boundaries + 1))
|
| 240 | step = boundaries / MAX_SYSCALL_POINTS
|
| 241 | points = sorted({max(1, int(round(i * step))) for i in range(1, MAX_SYSCALL_POINTS + 1)})
|
| 242 | return points
|