| 1 | """Environment-fault family: disk full, read-only directories, fsync load.
|
| 2 |
|
| 3 | - disk_full the trial (client + server data) lives on a small RAM disk
|
| 4 | (macOS: ``hdiutil attach -nomount ram://`` + ``newfs_hfs`` mounted
|
| 5 | inside the work dir; no sudo) filled to leave ``free_kib`` free.
|
| 6 | The operation must fail with an actionable error or succeed with
|
| 7 | the exact outcome; the store must open; pre-op commits survive;
|
| 8 | after the filler is removed the next operation must converge.
|
| 9 | - readonly the store directory (``.oak`` / ``.git``) or one worktree subdir is
|
| 10 | made read-only for the operation, then restored.
|
| 11 | - fsync_load (see runner) the same_branch concurrency scenario under a
|
| 12 | background fsync storm (4 writers x 256 KiB + F_FULLFSYNC).
|
| 13 | """
|
| 14 |
|
| 15 | from __future__ import annotations
|
| 16 |
|
| 17 | import os
|
| 18 | import platform
|
| 19 | import re
|
| 20 | import shutil
|
| 21 | import stat
|
| 22 | import subprocess
|
| 23 | import time
|
| 24 | from pathlib import Path
|
| 25 | from typing import Optional
|
| 26 |
|
| 27 | from . import HARNESS_ERROR
|
| 28 | from .crash import evaluate
|
| 29 | from .engine import Calibration, Trial, actionable, classify, converge_check
|
| 30 |
|
| 31 | FREE_KIB = (0, 256, 4096)
|
| 32 | # An environment fault is actionable when the error names the OS cause the user must fix.
|
| 33 | CAUSE_RE = re.compile(r"no space left|os error 28|ENOSPC|permission denied|os error 13|read-?only|disk I/O error|"
|
| 34 | r"disk (is )?full|quota", re.I)
|
| 35 |
|
| 36 |
|
| 37 | def env_actionable(res) -> bool:
|
| 38 | return actionable(res) or (res.returncode not in (None, 0) and res.returncode > 0
|
| 39 | and "panicked at" not in res.text() and bool(CAUSE_RE.search(res.text())))
|
| 40 |
|
| 41 |
|
| 42 | class RamDisk:
|
| 43 | def __init__(self, mountpoint: Path, mib: int = 96):
|
| 44 | self.mountpoint = mountpoint
|
| 45 | self.mib = mib
|
| 46 | self.device: Optional[str] = None
|
| 47 |
|
| 48 | def __enter__(self) -> "RamDisk":
|
| 49 | if platform.system() != "Darwin":
|
| 50 | raise NotImplementedError("disk-full fixture implemented with macOS hdiutil RAM disks (Linux: small tmpfs pending)")
|
| 51 | sectors = self.mib * 2048
|
| 52 | out = subprocess.run(["hdiutil", "attach", "-nomount", f"ram://{sectors}"], capture_output=True, text=True,
|
| 53 | timeout=60)
|
| 54 | if out.returncode != 0:
|
| 55 | raise NotImplementedError("hdiutil ram disk unavailable: " + out.stderr[-200:])
|
| 56 | self.device = out.stdout.split()[0]
|
| 57 | subprocess.run(["newfs_hfs", "-v", "p4full", self.device], capture_output=True, check=True, timeout=60)
|
| 58 | self.mountpoint.mkdir(parents=True, exist_ok=True)
|
| 59 | subprocess.run(["diskutil", "mount", "-mountPoint", str(self.mountpoint), self.device], capture_output=True,
|
| 60 | check=True, timeout=60)
|
| 61 | return self
|
| 62 |
|
| 63 | def __exit__(self, *exc) -> None:
|
| 64 | if self.device:
|
| 65 | subprocess.run(["diskutil", "unmount", "force", str(self.mountpoint)], capture_output=True, timeout=60)
|
| 66 | subprocess.run(["hdiutil", "detach", self.device, "-force"], capture_output=True, timeout=60)
|
| 67 | self.device = None
|
| 68 |
|
| 69 |
|
| 70 | def fill(directory: Path, leave_kib: int) -> Path:
|
| 71 | filler = directory / "p4-filler.bin"
|
| 72 | chunk = b"\0" * (1 << 20)
|
| 73 | with filler.open("wb") as fh:
|
| 74 | try:
|
| 75 | while True:
|
| 76 | fh.write(chunk)
|
| 77 | fh.flush()
|
| 78 | except OSError:
|
| 79 | pass
|
| 80 | size = filler.stat().st_size
|
| 81 | with filler.open("r+b") as fh:
|
| 82 | fh.truncate(max(0, size - leave_kib * 1024))
|
| 83 | return filler
|
| 84 |
|
| 85 |
|
| 86 | def _judge_env(trial: Trial, calib: Calibration, op_res, restore) -> tuple[dict, dict]:
|
| 87 | """Common judgement: op either converged or failed actionably; then restore and re-run."""
|
| 88 | inv: dict = {"op_no_hang": not op_res.timed_out}
|
| 89 | obs: dict = {"op": op_res.brief(300)}
|
| 90 | if op_res.returncode == 0:
|
| 91 | ok, detail = converge_check(trial, calib, trial.target_repo())
|
| 92 | inv["op_success_is_true"] = ok
|
| 93 | obs["op_converge"] = detail
|
| 94 | else:
|
| 95 | inv["op_failure_actionable"] = env_actionable(op_res)
|
| 96 | restore()
|
| 97 | more_inv, more_obs = evaluate(trial, calib, op_res, True)
|
| 98 | more_inv.pop("op_no_hang", None)
|
| 99 | inv.update(more_inv)
|
| 100 | obs.update({k: v for k, v in more_obs.items() if k != "op"})
|
| 101 | return inv, obs
|
| 102 |
|
| 103 |
|
| 104 | def disk_full_trial(fixture, subject_bin: str, work: Path, name: str, calib: Calibration, leave_kib: int) -> dict:
|
| 105 | started = time.perf_counter()
|
| 106 | mount = work / f"{name}-mnt"
|
| 107 | op_res = None
|
| 108 | try:
|
| 109 | with RamDisk(mount):
|
| 110 | trial = Trial(fixture, subject_bin, mount, "t")
|
| 111 | try:
|
| 112 | trial.start_server()
|
| 113 | trial.pre()
|
| 114 | filler = fill(mount, leave_kib)
|
| 115 | op_res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
|
| 116 | inv, obs = _judge_env(trial, calib, op_res, lambda: filler.unlink(missing_ok=True))
|
| 117 | obs["free_kib_before_op"] = leave_kib
|
| 118 | classification, error = classify(inv), None
|
| 119 | finally:
|
| 120 | trial.stop_server()
|
| 121 | except NotImplementedError as exc:
|
| 122 | return {"unsupported": str(exc)}
|
| 123 | except Exception as exc: # noqa: BLE001
|
| 124 | inv, obs, classification, error = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
|
| 125 | finally:
|
| 126 | shutil.rmtree(mount, ignore_errors=True)
|
| 127 | return _row(f"free={leave_kib}KiB", inv, obs, classification, error, op_res, started)
|
| 128 |
|
| 129 |
|
| 130 | def readonly_trial(fixture, subject_bin: str, work: Path, name: str, calib: Calibration, target: str) -> dict:
|
| 131 | """target: 'store' (.oak/.git) or 'subdir' (worktree d3/)."""
|
| 132 | started = time.perf_counter()
|
| 133 | trial = Trial(fixture, subject_bin, work, name)
|
| 134 | op_res = None
|
| 135 | changed: list[tuple[Path, int]] = []
|
| 136 |
|
| 137 | def lock_down() -> None:
|
| 138 | repo = trial.client
|
| 139 | base = repo / (".oak" if trial.kind == "oak" else ".git") if target == "store" else repo / "d3"
|
| 140 | for path in [base, *base.rglob("*")] if target == "store" else [base]:
|
| 141 | mode = path.stat().st_mode
|
| 142 | changed.append((path, mode))
|
| 143 | os.chmod(path, mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
|
| 144 |
|
| 145 | def restore() -> None:
|
| 146 | for path, mode in reversed(changed):
|
| 147 | try:
|
| 148 | os.chmod(path, mode)
|
| 149 | except OSError:
|
| 150 | pass
|
| 151 | changed.clear()
|
| 152 |
|
| 153 | try:
|
| 154 | trial.start_server()
|
| 155 | trial.pre()
|
| 156 | lock_down()
|
| 157 | op_res = trial.adapter.run(trial.op_argv(), trial.op_cwd())
|
| 158 | inv, obs = _judge_env(trial, calib, op_res, restore)
|
| 159 | classification, error = classify(inv), None
|
| 160 | except Exception as exc: # noqa: BLE001
|
| 161 | restore()
|
| 162 | inv, obs, classification, error = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
|
| 163 | finally:
|
| 164 | restore()
|
| 165 | trial.cleanup()
|
| 166 | return _row(f"readonly={target}", inv, obs, classification, error, op_res, started)
|
| 167 |
|
| 168 |
|
| 169 | def _row(point, inv, obs, classification, error, op_res, started) -> dict:
|
| 170 | row = {"fault_point": point, "invariants": inv, "violations": sorted(k for k, v in inv.items() if v is False),
|
| 171 | "observations": obs, "classification": classification,
|
| 172 | "elapsed_ms": round(op_res.elapsed_ms if op_res else (time.perf_counter() - started) * 1000.0, 3),
|
| 173 | "command": op_res.argv if op_res else []}
|
| 174 | if error:
|
| 175 | row["harness_error"] = error
|
| 176 | return row
|
| 177 |
|
| 178 |
|
| 179 | class FsyncLoad:
|
| 180 | """Background fsync storm (idea from the L8 fsyncload.py harness)."""
|
| 181 |
|
| 182 | SCRIPT = r"""
|
| 183 | import os, sys, fcntl, time
|
| 184 | d, stop = sys.argv[1], sys.argv[2]
|
| 185 | buf = os.urandom(256 * 1024)
|
| 186 | p = os.path.join(d, f"load-{os.getpid()}")
|
| 187 | while not os.path.exists(stop):
|
| 188 | with open(p, "wb") as f:
|
| 189 | f.write(buf); f.flush()
|
| 190 | try: fcntl.fcntl(f.fileno(), 51)
|
| 191 | except OSError: os.fsync(f.fileno())
|
| 192 | os.unlink(p)
|
| 193 | """
|
| 194 |
|
| 195 | def __init__(self, directory: Path, workers: int = 4):
|
| 196 | self.dir = directory
|
| 197 | self.workers = workers
|
| 198 | self.procs: list[subprocess.Popen] = []
|
| 199 |
|
| 200 | def __enter__(self) -> "FsyncLoad":
|
| 201 | self.dir.mkdir(parents=True, exist_ok=True)
|
| 202 | self.stop_file = self.dir / "stop"
|
| 203 | self.stop_file.unlink(missing_ok=True)
|
| 204 | script = self.dir / "fsyncload.py"
|
| 205 | script.write_text(self.SCRIPT)
|
| 206 | for _ in range(self.workers):
|
| 207 | self.procs.append(subprocess.Popen(["python3", str(script), str(self.dir), str(self.stop_file)]))
|
| 208 | return self
|
| 209 |
|
| 210 | def __exit__(self, *exc) -> None:
|
| 211 | self.stop_file.write_text("stop")
|
| 212 | for proc in self.procs:
|
| 213 | try:
|
| 214 | proc.wait(timeout=20)
|
| 215 | except subprocess.TimeoutExpired:
|
| 216 | proc.kill()
|