| 1 | """stress-ng load generation for noisy-neighbor benchmarks.
|
| 2 |
|
| 3 | stress-ng is a perturbation instrument, never a benchmark: it injects a known,
|
| 4 | machine-independent load tier while a lane measures VCS commands under
|
| 5 | contention, and it powers falsification CI (inject load, assert
|
| 6 | ``envwatch.environment_suspect`` fires).
|
| 7 |
|
| 8 | Discipline (every command this module builds):
|
| 9 |
|
| 10 | - ``--timeout`` is ALWAYS present — stress-ng's default run length is 24 hours.
|
| 11 | - ``--temp-path`` is ALWAYS present and must live on the same filesystem as
|
| 12 | the benchmarked repos, otherwise the io tier is not actually contending.
|
| 13 | - ``--seed`` is ALWAYS present so a tier is deterministic across runs.
|
| 14 | - Load tiers are machine-independent shapes (half the cores at 50%%, two hdd
|
| 15 | workers), not absolute op counts.
|
| 16 | - bogo-ops output is parsed ONLY to verify the load ran (``verify_ran``),
|
| 17 | never reported as a benchmark number.
|
| 18 |
|
| 19 | stress-ng is an OPTIONAL external binary (the only non-stdlib dependency in
|
| 20 | the harness, and it stays optional): when it is absent, ``LoadGenerator.start``
|
| 21 | reports ``{"started": False, "reason": "stress_ng_missing"}`` and lanes emit
|
| 22 | skip rows — a row may never claim load that was not applied (ADR-0002).
|
| 23 | """
|
| 24 |
|
| 25 | from __future__ import annotations
|
| 26 |
|
| 27 | import os
|
| 28 | import re
|
| 29 | import shutil
|
| 30 | import subprocess
|
| 31 | import tempfile
|
| 32 | from pathlib import Path
|
| 33 | from typing import Any
|
| 34 |
|
| 35 | STRESS_NG = "stress-ng"
|
| 36 |
|
| 37 | # Deterministic default for --seed; any fixed value works, it only has to be
|
| 38 | # recorded and stable so two runs of the same tier perturb identically.
|
| 39 | DEFAULT_SEED = 20260612
|
| 40 |
|
| 41 | # Safety-net run cap. stop() is the normal end of a load window; the timeout
|
| 42 | # only bounds orphaned generators (stress-ng's own default is 24 hours).
|
| 43 | DEFAULT_TIMEOUT_S = 1800
|
| 44 |
|
| 45 | # Machine-independent load tiers. "none" is the explicit no-load tier (rows
|
| 46 | # still record the string "none" so the field is never ambiguous).
|
| 47 | LOAD_TIERS: dict[str, dict[str, Any] | None] = {
|
| 48 | "none": None,
|
| 49 | "cpu50": {"cpu_load_pct": 50},
|
| 50 | "io": {"hdd_workers": 2, "hdd_bytes": "256M"},
|
| 51 | "mixed": {"cpu_load_pct": 50, "hdd_workers": 2, "hdd_bytes": "256M"},
|
| 52 | }
|
| 53 |
|
| 54 |
|
| 55 | def cpu_worker_count(cpu_count: int | None = None) -> int:
|
| 56 | """Half the cores, at least one — the machine-independent cpu tier width."""
|
| 57 | cores = cpu_count if cpu_count is not None else (os.cpu_count() or 1)
|
| 58 | return max(1, cores // 2)
|
| 59 |
|
| 60 |
|
| 61 | def stress_ng_command(
|
| 62 | tier: str,
|
| 63 | *,
|
| 64 | timeout_s: int | float,
|
| 65 | temp_path: str | Path,
|
| 66 | seed: int = DEFAULT_SEED,
|
| 67 | cpu_count: int | None = None,
|
| 68 | ) -> list[str] | None:
|
| 69 | """argv for one load tier, or None for the "none" tier.
|
| 70 |
|
| 71 | Pure: no filesystem or process access, so the exact argument shape is
|
| 72 | unit-testable. argv[0] is the bare binary name; ``LoadGenerator`` resolves
|
| 73 | the real path (or reports the binary missing).
|
| 74 | """
|
| 75 | if tier not in LOAD_TIERS:
|
| 76 | raise ValueError(f"unknown load tier {tier!r}; known: {', '.join(sorted(LOAD_TIERS))}")
|
| 77 | spec = LOAD_TIERS[tier]
|
| 78 | if spec is None:
|
| 79 | return None
|
| 80 | if timeout_s <= 0:
|
| 81 | raise ValueError("timeout_s must be positive: stress-ng's default run length is 24h")
|
| 82 | command = [
|
| 83 | STRESS_NG,
|
| 84 | "--timeout",
|
| 85 | f"{int(timeout_s)}s",
|
| 86 | "--temp-path",
|
| 87 | str(temp_path),
|
| 88 | "--seed",
|
| 89 | str(int(seed)),
|
| 90 | "--metrics-brief",
|
| 91 | ]
|
| 92 | if "cpu_load_pct" in spec:
|
| 93 | command += ["--cpu", str(cpu_worker_count(cpu_count)), "--cpu-load", str(spec["cpu_load_pct"])]
|
| 94 | if "hdd_workers" in spec:
|
| 95 | command += ["--hdd", str(spec["hdd_workers"]), "--hdd-bytes", str(spec["hdd_bytes"])]
|
| 96 | return command
|
| 97 |
|
| 98 |
|
| 99 | # A stress-ng metrics row: "stress-ng: metrc: [pid] cpu 20704 10.00 ..."
|
| 100 | # (older releases emit the same row under "info:"). The stressor name is
|
| 101 | # followed by integer bogo ops then the real-time column. Header lines
|
| 102 | # ("stressor bogo ops real time ...") and chatter ("dispatching hogs: 4 cpu")
|
| 103 | # do not match because no integer directly follows the first word.
|
| 104 | _BOGO_OPS_ROW = re.compile(
|
| 105 | r"^stress-ng:\s+(?:metrc|info):\s+\[\d+\]\s+([a-z][a-z0-9_-]*)\s+(\d+)\s+[0-9]+(?:\.[0-9]+)?\s",
|
| 106 | re.MULTILINE,
|
| 107 | )
|
| 108 |
|
| 109 |
|
| 110 | def verify_ran(stderr_text: str) -> bool:
|
| 111 | """Did the injected load actually run?
|
| 112 |
|
| 113 | True when at least one stressor reports positive bogo ops. The numbers are
|
| 114 | used ONLY as a did-it-run check; they are never recorded as a benchmark
|
| 115 | measurement.
|
| 116 | """
|
| 117 | return any(int(ops) > 0 for _name, ops in _BOGO_OPS_ROW.findall(stderr_text or ""))
|
| 118 |
|
| 119 |
|
| 120 | def load_tier_unavailable_reason(detail: str) -> str:
|
| 121 | """Canonical skip reason for a tier the host cannot apply."""
|
| 122 | return f"load_tier_unavailable:{detail}"
|
| 123 |
|
| 124 |
|
| 125 | def environment_boundary_fields(
|
| 126 | load_tier: str,
|
| 127 | before: dict[str, Any] | None,
|
| 128 | after: dict[str, Any] | None,
|
| 129 | *,
|
| 130 | cpu_count: int | None = None,
|
| 131 | ) -> dict[str, Any]:
|
| 132 | """Row fields for one measurement window's environment boundary.
|
| 133 |
|
| 134 | ``environment_suspect`` keeps the ADR-0002 distinction: None when the
|
| 135 | suspicion machinery did not run (a boundary sample is missing), [] when it
|
| 136 | ran and the window is clean, [reasons] when suspect. ``load_tier`` is
|
| 137 | always the explicit string ("none" included) so it is never ambiguous.
|
| 138 | """
|
| 139 | from oakbench.envwatch import environment_suspect
|
| 140 |
|
| 141 | suspect: list[str] | None = None
|
| 142 | if before is not None and after is not None:
|
| 143 | suspect = environment_suspect(before, after, cpu_count=cpu_count)
|
| 144 | return {
|
| 145 | "load_tier": load_tier,
|
| 146 | "environment_before": before,
|
| 147 | "environment_after": after,
|
| 148 | "environment_suspect": suspect,
|
| 149 | }
|
| 150 |
|
| 151 |
|
| 152 | class LoadGenerator:
|
| 153 | """Owns one stress-ng subprocess for a measurement window.
|
| 154 |
|
| 155 | start() before the timed measurements, stop() after (both untimed).
|
| 156 | Honesty contract: start() never raises for an absent binary — it returns
|
| 157 | ``{"started": False, "reason": "stress_ng_missing"}`` so the lane can emit
|
| 158 | skip rows instead of claiming load that was not applied.
|
| 159 | """
|
| 160 |
|
| 161 | def __init__(
|
| 162 | self,
|
| 163 | tier: str,
|
| 164 | temp_path: str | Path,
|
| 165 | *,
|
| 166 | timeout_s: int | float = DEFAULT_TIMEOUT_S,
|
| 167 | seed: int = DEFAULT_SEED,
|
| 168 | cpu_count: int | None = None,
|
| 169 | stress_ng_path: str | None = None,
|
| 170 | ) -> None:
|
| 171 | if tier not in LOAD_TIERS:
|
| 172 | raise ValueError(f"unknown load tier {tier!r}; known: {', '.join(sorted(LOAD_TIERS))}")
|
| 173 | self.tier = tier
|
| 174 | self.temp_path = Path(temp_path)
|
| 175 | self.timeout_s = timeout_s
|
| 176 | self.seed = seed
|
| 177 | self.cpu_count = cpu_count
|
| 178 | self.stress_ng_path = stress_ng_path
|
| 179 | self._proc: subprocess.Popen[bytes] | None = None
|
| 180 | self._stderr_handle: Any = None
|
| 181 |
|
| 182 | def start(self) -> dict[str, Any]:
|
| 183 | command = stress_ng_command(
|
| 184 | self.tier,
|
| 185 | timeout_s=self.timeout_s,
|
| 186 | temp_path=self.temp_path,
|
| 187 | seed=self.seed,
|
| 188 | cpu_count=self.cpu_count,
|
| 189 | )
|
| 190 | if command is None:
|
| 191 | return {"started": False, "reason": "tier_none"}
|
| 192 | binary = self.stress_ng_path or shutil.which(STRESS_NG)
|
| 193 | if binary is None:
|
| 194 | return {"started": False, "reason": "stress_ng_missing"}
|
| 195 | self.temp_path.mkdir(parents=True, exist_ok=True)
|
| 196 | self._stderr_handle = tempfile.TemporaryFile()
|
| 197 | try:
|
| 198 | self._proc = subprocess.Popen(
|
| 199 | [binary, *command[1:]],
|
| 200 | cwd=self.temp_path,
|
| 201 | stdout=subprocess.DEVNULL,
|
| 202 | stderr=self._stderr_handle,
|
| 203 | )
|
| 204 | except OSError as error:
|
| 205 | self._close_stderr()
|
| 206 | return {"started": False, "reason": f"spawn_failed: {type(error).__name__}: {error}"}
|
| 207 | return {"started": True, "reason": None}
|
| 208 |
|
| 209 | def stop(self) -> dict[str, Any]:
|
| 210 | """Terminate the generator (untimed) and verify the load ran.
|
| 211 |
|
| 212 | ``verified`` is True only when stress-ng's own bogo-ops metrics confirm
|
| 213 | work happened; None when no process ever started (unmeasured, ADR-0002).
|
| 214 | """
|
| 215 | if self._proc is None:
|
| 216 | self._close_stderr()
|
| 217 | return {"stopped": False, "returncode": None, "verified": None, "stderr_tail": None}
|
| 218 | proc = self._proc
|
| 219 | self._proc = None
|
| 220 | proc.terminate()
|
| 221 | try:
|
| 222 | returncode = proc.wait(timeout=15)
|
| 223 | except subprocess.TimeoutExpired:
|
| 224 | proc.kill()
|
| 225 | returncode = proc.wait()
|
| 226 | stderr_text = self._read_stderr()
|
| 227 | self._close_stderr()
|
| 228 | return {
|
| 229 | "stopped": True,
|
| 230 | "returncode": returncode,
|
| 231 | "verified": verify_ran(stderr_text),
|
| 232 | "stderr_tail": stderr_text[-2000:],
|
| 233 | }
|
| 234 |
|
| 235 | def _read_stderr(self) -> str:
|
| 236 | if self._stderr_handle is None:
|
| 237 | return ""
|
| 238 | try:
|
| 239 | self._stderr_handle.flush()
|
| 240 | self._stderr_handle.seek(0)
|
| 241 | return self._stderr_handle.read().decode("utf-8", "replace")
|
| 242 | except (OSError, ValueError):
|
| 243 | return ""
|
| 244 |
|
| 245 | def _close_stderr(self) -> None:
|
| 246 | if self._stderr_handle is not None:
|
| 247 | try:
|
| 248 | self._stderr_handle.close()
|
| 249 | except OSError:
|
| 250 | pass
|
| 251 | self._stderr_handle = None
|
| 252 |
|
| 253 | def __enter__(self) -> "LoadGenerator":
|
| 254 | return self
|
| 255 |
|
| 256 | def __exit__(self, *exc_info: Any) -> None:
|
| 257 | self.stop()
|