Viewing files at commit f2acb751aa96Commit detailsView latest

suites/robustness/robustness/envfaults.py 216 lines · 8.5 KB · python
1
"""Environment-fault family: disk full, read-only directories, fsync load.
2
3
- disk_full   the trial (client + server data) lives on a small RAM disk
4
              (macOS: ``hdiutil attach -nomount ram://`` + ``newfs_hfs`` mounted
5
              inside the work dir; no sudo) filled to leave ``free_kib`` free.
6
              The operation must fail with an actionable error or succeed with
7
              the exact outcome; the store must open; pre-op commits survive;
8
              after the filler is removed the next operation must converge.
9
- readonly    the store directory (``.oak`` / ``.git``) or one worktree subdir is
10
              made read-only for the operation, then restored.
11
- fsync_load  (see runner) the same_branch concurrency scenario under a
12
              background fsync storm (4 writers x 256 KiB + F_FULLFSYNC).
13
"""
14
15
from __future__ import annotations
16
17
import os
18
import platform
19
import re
20
import shutil
21
import stat
22
import subprocess
23
import time
24
from pathlib import Path
25
from typing import Optional
26
27
from . import HARNESS_ERROR
28
from .crash import evaluate
29
from .engine import Calibration, Trial, actionable, classify, converge_check
30
31
FREE_KIB = (0, 256, 4096)
32
# An environment fault is actionable when the error names the OS cause the user must fix.
33
CAUSE_RE = re.compile(r"no space left|os error 28|ENOSPC|permission denied|os error 13|read-?only|disk I/O error|"
34
                      r"disk (is )?full|quota", re.I)
35
36
37
def env_actionable(res) -> bool:
38
    return actionable(res) or (res.returncode not in (None, 0) and res.returncode > 0
39
                               and "panicked at" not in res.text() and bool(CAUSE_RE.search(res.text())))
40
41
42
class RamDisk:
43
    def __init__(self, mountpoint: Path, mib: int = 96):
44
        self.mountpoint = mountpoint
45
        self.mib = mib
46
        self.device: Optional[str] = None
47
48
    def __enter__(self) -> "RamDisk":
49
        if platform.system() != "Darwin":
50
            raise NotImplementedError("disk-full fixture implemented with macOS hdiutil RAM disks (Linux: small tmpfs pending)")
51
        sectors = self.mib * 2048
52
        out = subprocess.run(["hdiutil", "attach", "-nomount", f"ram://{sectors}"], capture_output=True, text=True,
53
                             timeout=60)
54
        if out.returncode != 0:
55
            raise NotImplementedError("hdiutil ram disk unavailable: " + out.stderr[-200:])
56
        self.device = out.stdout.split()[0]
57
        subprocess.run(["newfs_hfs", "-v", "p4full", self.device], capture_output=True, check=True, timeout=60)
58
        self.mountpoint.mkdir(parents=True, exist_ok=True)
59
        subprocess.run(["diskutil", "mount", "-mountPoint", str(self.mountpoint), self.device], capture_output=True,
60
                       check=True, timeout=60)
61
        return self
62
63
    def __exit__(self, *exc) -> None:
64
        if self.device:
65
            subprocess.run(["diskutil", "unmount", "force", str(self.mountpoint)], capture_output=True, timeout=60)
66
            subprocess.run(["hdiutil", "detach", self.device, "-force"], capture_output=True, timeout=60)
67
            self.device = None
68
69
70
def fill(directory: Path, leave_kib: int) -> Path:
71
    filler = directory / "p4-filler.bin"
72
    chunk = b"\0" * (1 << 20)
73
    with filler.open("wb") as fh:
74
        try:
75
            while True:
76
                fh.write(chunk)
77
                fh.flush()
78
        except OSError:
79
            pass
80
    size = filler.stat().st_size
81
    with filler.open("r+b") as fh:
82
        fh.truncate(max(0, size - leave_kib * 1024))
83
    return filler
84
85
86
def _judge_env(trial: Trial, calib: Calibration, op_res, restore) -> tuple[dict, dict]:
87
    """Common judgement: op either converged or failed actionably; then restore and re-run."""
88
    inv: dict = {"op_no_hang": not op_res.timed_out}
89
    obs: dict = {"op": op_res.brief(300)}
90
    if op_res.returncode == 0:
91
        ok, detail = converge_check(trial, calib, trial.target_repo())
92
        inv["op_success_is_true"] = ok
93
        obs["op_converge"] = detail
94
    else:
95
        inv["op_failure_actionable"] = env_actionable(op_res)
96
    restore()
97
    more_inv, more_obs = evaluate(trial, calib, op_res, True)
98
    more_inv.pop("op_no_hang", None)
99
    inv.update(more_inv)
100
    obs.update({k: v for k, v in more_obs.items() if k != "op"})
101
    return inv, obs
102
103
104
def disk_full_trial(fixture, subject_bin: str, work: Path, name: str, calib: Calibration, leave_kib: int) -> dict:
105
    started = time.perf_counter()
106
    mount = work / f"{name}-mnt"
107
    op_res = None
108
    try:
109
        with RamDisk(mount):
110
            trial = Trial(fixture, subject_bin, mount, "t")
111
            try:
112
                trial.start_server()
113
                trial.pre()
114
                filler = fill(mount, leave_kib)
115
                op_res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
116
                inv, obs = _judge_env(trial, calib, op_res, lambda: filler.unlink(missing_ok=True))
117
                obs["free_kib_before_op"] = leave_kib
118
                classification, error = classify(inv), None
119
            finally:
120
                trial.stop_server()
121
    except NotImplementedError as exc:
122
        return {"unsupported": str(exc)}
123
    except Exception as exc:  # noqa: BLE001
124
        inv, obs, classification, error = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
125
    finally:
126
        shutil.rmtree(mount, ignore_errors=True)
127
    return _row(f"free={leave_kib}KiB", inv, obs, classification, error, op_res, started)
128
129
130
def readonly_trial(fixture, subject_bin: str, work: Path, name: str, calib: Calibration, target: str) -> dict:
131
    """target: 'store' (.oak/.git) or 'subdir' (worktree d3/)."""
132
    started = time.perf_counter()
133
    trial = Trial(fixture, subject_bin, work, name)
134
    op_res = None
135
    changed: list[tuple[Path, int]] = []
136
137
    def lock_down() -> None:
138
        repo = trial.client
139
        base = repo / (".oak" if trial.kind == "oak" else ".git") if target == "store" else repo / "d3"
140
        for path in [base, *base.rglob("*")] if target == "store" else [base]:
141
            mode = path.stat().st_mode
142
            changed.append((path, mode))
143
            os.chmod(path, mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
144
145
    def restore() -> None:
146
        for path, mode in reversed(changed):
147
            try:
148
                os.chmod(path, mode)
149
            except OSError:
150
                pass
151
        changed.clear()
152
153
    try:
154
        trial.start_server()
155
        trial.pre()
156
        lock_down()
157
        op_res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
158
        inv, obs = _judge_env(trial, calib, op_res, restore)
159
        classification, error = classify(inv), None
160
    except Exception as exc:  # noqa: BLE001
161
        restore()
162
        inv, obs, classification, error = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
163
    finally:
164
        restore()
165
        trial.cleanup()
166
    return _row(f"readonly={target}", inv, obs, classification, error, op_res, started)
167
168
169
def _row(point, inv, obs, classification, error, op_res, started) -> dict:
170
    row = {"fault_point": point, "invariants": inv, "violations": sorted(k for k, v in inv.items() if v is False),
171
           "observations": obs, "classification": classification,
172
           "elapsed_ms": round(op_res.elapsed_ms if op_res else (time.perf_counter() - started) * 1000.0, 3),
173
           "command": op_res.argv if op_res else []}
174
    if error:
175
        row["harness_error"] = error
176
    return row
177
178
179
class FsyncLoad:
180
    """Background fsync storm (idea from the L8 fsyncload.py harness)."""
181
182
    SCRIPT = r"""
183
import os, sys, fcntl, time
184
d, stop = sys.argv[1], sys.argv[2]
185
buf = os.urandom(256 * 1024)
186
p = os.path.join(d, f"load-{os.getpid()}")
187
while not os.path.exists(stop):
188
    with open(p, "wb") as f:
189
        f.write(buf); f.flush()
190
        try: fcntl.fcntl(f.fileno(), 51)
191
        except OSError: os.fsync(f.fileno())
192
os.unlink(p)
193
"""
194
195
    def __init__(self, directory: Path, workers: int = 4):
196
        self.dir = directory
197
        self.workers = workers
198
        self.procs: list[subprocess.Popen] = []
199
200
    def __enter__(self) -> "FsyncLoad":
201
        self.dir.mkdir(parents=True, exist_ok=True)
202
        self.stop_file = self.dir / "stop"
203
        self.stop_file.unlink(missing_ok=True)
204
        script = self.dir / "fsyncload.py"
205
        script.write_text(self.SCRIPT)
206
        for _ in range(self.workers):
207
            self.procs.append(subprocess.Popen(["python3", str(script), str(self.dir), str(self.stop_file)]))
208
        return self
209
210
    def __exit__(self, *exc) -> None:
211
        self.stop_file.write_text("stop")
212
        for proc in self.procs:
213
            try:
214
                proc.wait(timeout=20)
215
            except subprocess.TimeoutExpired:
216
                proc.kill()