Viewing files at commit e60fa60d3a02Commit detailsView latest

suites/robustness/robustness/envfaults.py 301 lines · 12.1 KB · python
1
"""Environment-fault family: disk full, read-only directories, fsync load.
2
3
- disk_full   the trial (client + server data) lives on a small RAM disk
4
              (macOS: ``hdiutil attach -nomount ram://`` + ``newfs_hfs`` mounted
5
              inside the work dir; no sudo) filled to leave ``free_kib`` free.
6
              The operation must fail with an actionable error or succeed with
7
              the exact outcome; the store must open; pre-op commits survive;
8
              after the filler is removed the next operation must converge.
9
- readonly    the store directory (``.oak`` / ``.git``) or one worktree subdir is
10
              made read-only for the operation, then restored.
11
- fsync_load  (see runner) the same_branch concurrency scenario under a
12
              background fsync storm (4 writers x 256 KiB + F_FULLFSYNC).
13
"""
14
15
from __future__ import annotations
16
17
import errno
18
import os
19
import platform
20
import re
21
import shutil
22
import stat
23
import subprocess
24
import time
25
from pathlib import Path
26
from typing import Optional
27
28
from . import FAULT_NOT_FIRED, HARNESS_ERROR
29
from .crash import evaluate
30
from .engine import Calibration, Trial, actionable, classify, converge_check
31
32
FREE_KIB = (0, 256, 4096)
33
# An environment fault is actionable when the error names the OS cause the user must fix.
34
CAUSE_RE = re.compile(r"no space left|os error 28|ENOSPC|permission denied|os error 13|read-?only|disk I/O error|"
35
                      r"disk (is )?full|out of disk ?space|quota", re.I)
36
37
38
def env_actionable(res) -> bool:
39
    return actionable(res) or (res.returncode not in (None, 0) and res.returncode > 0
40
                               and "panicked at" not in res.text() and bool(CAUSE_RE.search(res.text())))
41
42
43
class RamDisk:
44
    def __init__(self, mountpoint: Path, mib: int = 96):
45
        self.mountpoint = mountpoint
46
        self.mib = mib
47
        self.device: Optional[str] = None
48
49
    def __enter__(self) -> "RamDisk":
50
        if platform.system() != "Darwin":
51
            raise NotImplementedError("disk-full fixture implemented with macOS hdiutil RAM disks (Linux: small tmpfs pending)")
52
        sectors = self.mib * 2048
53
        out = subprocess.run(["hdiutil", "attach", "-nomount", f"ram://{sectors}"], capture_output=True, text=True,
54
                             timeout=60)
55
        if out.returncode != 0:
56
            raise NotImplementedError("hdiutil ram disk unavailable: " + out.stderr[-200:])
57
        self.device = out.stdout.split()[0]
58
        subprocess.run(["newfs_hfs", "-v", "p4full", self.device], capture_output=True, check=True, timeout=60)
59
        self.mountpoint.mkdir(parents=True, exist_ok=True)
60
        subprocess.run(["diskutil", "mount", "-mountPoint", str(self.mountpoint), self.device], capture_output=True,
61
                       check=True, timeout=60)
62
        return self
63
64
    def __exit__(self, *exc) -> None:
65
        if self.device:
66
            subprocess.run(["diskutil", "unmount", "force", str(self.mountpoint)], capture_output=True, timeout=60)
67
            subprocess.run(["hdiutil", "detach", self.device, "-force"], capture_output=True, timeout=60)
68
            self.device = None
69
70
71
FILL_CHUNKS = (1 << 20, 64 << 10, 4 << 10, 512, 1)
72
73
74
def free_bytes(directory: Path) -> int:
75
    """Bytes an unprivileged writer can still allocate on directory's filesystem."""
76
    st = os.statvfs(directory)
77
    return st.f_bavail * st.f_frsize
78
79
80
def fill(directory: Path, leave_kib: int) -> tuple[Path, int]:
81
    """Fill the filesystem until writes fail at 1-byte granularity, then free ``leave_kib``.
82
83
    Writes shrink from 1 MiB to 64 KiB, 4 KiB, 512 B and 1 B, each until ENOSPC,
84
    with an fsync between rounds so delayed allocation cannot hide free space.
85
    Returns (filler, measured free bytes after the optional release).
86
    """
87
    filler = directory / "p4-filler.bin"
88
    fd = os.open(filler, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644)
89
    try:
90
        for _round in range(3):
91
            for size in FILL_CHUNKS:
92
                buf = b"\0" * size
93
                while True:
94
                    try:
95
                        written = os.write(fd, buf)
96
                    except OSError as exc:
97
                        if exc.errno in (errno.ENOSPC, errno.EDQUOT):
98
                            break
99
                        raise
100
                    if written < size:
101
                        break
102
            try:
103
                os.fsync(fd)
104
            except OSError:
105
                pass
106
            if free_bytes(directory) == 0:
107
                break
108
        if leave_kib:
109
            size = os.fstat(fd).st_size
110
            os.ftruncate(fd, max(0, size - leave_kib * 1024))
111
            os.fsync(fd)
112
    finally:
113
        os.close(fd)
114
    return filler, free_bytes(directory)
115
116
117
def allocated_bytes(root: Path) -> int:
118
    total = 0
119
    for path in [root, *root.rglob("*")]:
120
        try:
121
            total += path.lstat().st_blocks * 512
122
        except OSError:
123
            pass
124
    return total
125
126
127
def op_need_bytes(fixture, subject_bin: str, work: Path) -> int:
128
    """Net bytes the clean op allocates (client + server data) on an ordinary disk."""
129
    trial = Trial(fixture, subject_bin, work, "need")
130
    try:
131
        trial.start_server()
132
        trial.pre()
133
        before = allocated_bytes(trial.dir)
134
        res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
135
        trial.settle()
136
        if res.returncode != 0:
137
            return 0
138
        return max(0, allocated_bytes(trial.dir) - before)
139
    finally:
140
        trial.cleanup()
141
142
143
ENOSPC_RE = re.compile(r"no space left|os error 28|ENOSPC|disk (is )?full|database or disk is full", re.I)
144
145
146
def _judge_env(trial: Trial, calib: Calibration, op_res, restore) -> tuple[dict, dict]:
147
    """Common judgement: op either converged or failed actionably; then restore and re-run."""
148
    inv: dict = {"op_no_hang": not op_res.timed_out}
149
    obs: dict = {"op": op_res.brief(300)}
150
    if op_res.returncode == 0:
151
        ok, detail = converge_check(trial, calib, trial.target_repo())
152
        inv["op_success_is_true"] = ok
153
        obs["op_converge"] = detail
154
    else:
155
        inv["op_failure_actionable"] = env_actionable(op_res)
156
    restore()
157
    more_inv, more_obs = evaluate(trial, calib, op_res, True)
158
    more_inv.pop("op_no_hang", None)
159
    inv.update(more_inv)
160
    obs.update({k: v for k, v in more_obs.items() if k != "op"})
161
    return inv, obs
162
163
164
def disk_full_trial(fixture, subject_bin: str, work: Path, name: str, calib: Calibration, leave_kib: int,
165
                    need_bytes: int = 0) -> dict:
166
    """Run the op with the trial's filesystem filled to ``leave_kib`` free (measured).
167
168
    The fault fired only if the measured free space was below the op's need
169
    (net allocation of a clean run), or the op failed / reported ENOSPC.
170
    Otherwise the row is ``fault_not_fired``: the disk was not actually full
171
    for this op, so a success proves nothing.
172
    """
173
    started = time.perf_counter()
174
    mount = work / f"{name}-mnt"
175
    op_res = None
176
    fired = None
177
    try:
178
        with RamDisk(mount):
179
            trial = Trial(fixture, subject_bin, mount, "t")
180
            try:
181
                trial.start_server()
182
                trial.pre()
183
                trial.adapter.env()  # create the subject HOME before the disk is full (harness setup)
184
                filler, free = fill(mount, leave_kib)
185
                op_res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
186
                enospc = bool(ENOSPC_RE.search(op_res.text()))
187
                fired = free < need_bytes or op_res.returncode != 0 or enospc
188
                inv, obs = _judge_env(trial, calib, op_res, lambda: filler.unlink(missing_ok=True))
189
                obs.update(free_bytes_measured_before_op=free, op_need_bytes=need_bytes, enospc_reported=enospc,
190
                           fill_granularity_bytes=FILL_CHUNKS[-1], requested_free_kib=leave_kib)
191
                classification = classify(inv, fault_fired=fired, reader_errors=obs.get("reader_errors"))
192
                error = None
193
            finally:
194
                trial.stop_server()
195
    except NotImplementedError as exc:
196
        return {"unsupported": str(exc)}
197
    except Exception as exc:  # noqa: BLE001
198
        inv, obs, classification, error = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
199
    finally:
200
        shutil.rmtree(mount, ignore_errors=True)
201
    row = _row(f"free={leave_kib}KiB", inv, obs, classification, error, op_res, started)
202
    row["fault_fired"] = fired
203
    if "free_bytes_measured_before_op" in obs:
204
        row["free_bytes_measured"] = obs["free_bytes_measured_before_op"]
205
    if classification == FAULT_NOT_FIRED:
206
        row["skip_reason"] = (f"disk not full for this op: {obs.get('free_bytes_measured_before_op')} B free >= "
207
                              f"{need_bytes} B needed and the op succeeded without ENOSPC")
208
    return row
209
210
211
def readonly_trial(fixture, subject_bin: str, work: Path, name: str, calib: Calibration, target: str) -> dict:
212
    """target: 'store' (.oak/.git) or 'subdir' (worktree d3/)."""
213
    started = time.perf_counter()
214
    trial = Trial(fixture, subject_bin, work, name)
215
    op_res = None
216
    changed: list[tuple[Path, int]] = []
217
218
    def lock_down() -> None:
219
        repo = trial.client
220
        base = repo / (".oak" if trial.kind == "oak" else ".git") if target == "store" else repo / "d3"
221
        for path in [base, *base.rglob("*")] if target == "store" else [base]:
222
            mode = path.stat().st_mode
223
            changed.append((path, mode))
224
            os.chmod(path, mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
225
226
    def restore() -> None:
227
        for path, mode in reversed(changed):
228
            try:
229
                os.chmod(path, mode)
230
            except OSError:
231
                pass
232
        changed.clear()
233
234
    try:
235
        trial.start_server()
236
        trial.pre()
237
        lock_down()
238
        op_res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
239
        inv, obs = _judge_env(trial, calib, op_res, restore)
240
        classification, error = classify(inv, fault_fired=True, reader_errors=obs.get("reader_errors")), None
241
    except Exception as exc:  # noqa: BLE001
242
        restore()
243
        inv, obs, classification, error = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
244
    finally:
245
        restore()
246
        trial.cleanup()
247
    row = _row(f"readonly={target}", inv, obs, classification, error, op_res, started)
248
    row["fault_fired"] = True  # the path the op writes was read-only for the whole op
249
    return row
250
251
252
def _row(point, inv, obs, classification, error, op_res, started) -> dict:
253
    row = {"fault_point": point, "invariants": inv, "violations": sorted(k for k, v in inv.items() if v is False),
254
           "observations": obs, "classification": classification,
255
           "elapsed_ms": round(op_res.elapsed_ms if op_res else (time.perf_counter() - started) * 1000.0, 3),
256
           "command": op_res.argv if op_res else []}
257
    if error:
258
        row["harness_error"] = error
259
    elif classification == HARNESS_ERROR and obs.get("reader_errors"):
260
        row["harness_error"] = "independent reader failed: " + "; ".join(obs["reader_errors"][:2])
261
    return row
262
263
264
class FsyncLoad:
265
    """Background fsync storm (idea from the L8 fsyncload.py harness)."""
266
267
    SCRIPT = r"""
268
import os, sys, fcntl, time
269
d, stop = sys.argv[1], sys.argv[2]
270
buf = os.urandom(256 * 1024)
271
p = os.path.join(d, f"load-{os.getpid()}")
272
while not os.path.exists(stop):
273
    with open(p, "wb") as f:
274
        f.write(buf); f.flush()
275
        try: fcntl.fcntl(f.fileno(), 51)
276
        except OSError: os.fsync(f.fileno())
277
os.unlink(p)
278
"""
279
280
    def __init__(self, directory: Path, workers: int = 4):
281
        self.dir = directory
282
        self.workers = workers
283
        self.procs: list[subprocess.Popen] = []
284
285
    def __enter__(self) -> "FsyncLoad":
286
        self.dir.mkdir(parents=True, exist_ok=True)
287
        self.stop_file = self.dir / "stop"
288
        self.stop_file.unlink(missing_ok=True)
289
        script = self.dir / "fsyncload.py"
290
        script.write_text(self.SCRIPT)
291
        for _ in range(self.workers):
292
            self.procs.append(subprocess.Popen(["python3", str(script), str(self.dir), str(self.stop_file)]))
293
        return self
294
295
    def __exit__(self, *exc) -> None:
296
        self.stop_file.write_text("stop")
297
        for proc in self.procs:
298
            try:
299
                proc.wait(timeout=20)
300
            except subprocess.TimeoutExpired:
301
                proc.kill()