Viewing files at commit c8b43cc46596Commit detailsView latest

scripts/oakbench/loadgen.py 257 lines · 9.2 KB · python
1
"""stress-ng load generation for noisy-neighbor benchmarks.
2
3
stress-ng is a perturbation instrument, never a benchmark: it injects a known,
4
machine-independent load tier while a lane measures VCS commands under
5
contention, and it powers falsification CI (inject load, assert
6
``envwatch.environment_suspect`` fires).
7
8
Discipline (every command this module builds):
9
10
- ``--timeout`` is ALWAYS present — stress-ng's default run length is 24 hours.
11
- ``--temp-path`` is ALWAYS present and must live on the same filesystem as
12
  the benchmarked repos, otherwise the io tier is not actually contending.
13
- ``--seed`` is ALWAYS present so a tier is deterministic across runs.
14
- Load tiers are machine-independent shapes (half the cores at 50%%, two hdd
15
  workers), not absolute op counts.
16
- bogo-ops output is parsed ONLY to verify the load ran (``verify_ran``),
17
  never reported as a benchmark number.
18
19
stress-ng is an OPTIONAL external binary (the only non-stdlib dependency in
20
the harness, and it stays optional): when it is absent, ``LoadGenerator.start``
21
reports ``{"started": False, "reason": "stress_ng_missing"}`` and lanes emit
22
skip rows — a row may never claim load that was not applied (ADR-0002).
23
"""
24
25
from __future__ import annotations
26
27
import os
28
import re
29
import shutil
30
import subprocess
31
import tempfile
32
from pathlib import Path
33
from typing import Any
34
35
STRESS_NG = "stress-ng"
36
37
# Deterministic default for --seed; any fixed value works, it only has to be
38
# recorded and stable so two runs of the same tier perturb identically.
39
DEFAULT_SEED = 20260612
40
41
# Safety-net run cap. stop() is the normal end of a load window; the timeout
42
# only bounds orphaned generators (stress-ng's own default is 24 hours).
43
DEFAULT_TIMEOUT_S = 1800
44
45
# Machine-independent load tiers. "none" is the explicit no-load tier (rows
46
# still record the string "none" so the field is never ambiguous).
47
LOAD_TIERS: dict[str, dict[str, Any] | None] = {
48
    "none": None,
49
    "cpu50": {"cpu_load_pct": 50},
50
    "io": {"hdd_workers": 2, "hdd_bytes": "256M"},
51
    "mixed": {"cpu_load_pct": 50, "hdd_workers": 2, "hdd_bytes": "256M"},
52
}
53
54
55
def cpu_worker_count(cpu_count: int | None = None) -> int:
56
    """Half the cores, at least one — the machine-independent cpu tier width."""
57
    cores = cpu_count if cpu_count is not None else (os.cpu_count() or 1)
58
    return max(1, cores // 2)
59
60
61
def stress_ng_command(
62
    tier: str,
63
    *,
64
    timeout_s: int | float,
65
    temp_path: str | Path,
66
    seed: int = DEFAULT_SEED,
67
    cpu_count: int | None = None,
68
) -> list[str] | None:
69
    """argv for one load tier, or None for the "none" tier.
70
71
    Pure: no filesystem or process access, so the exact argument shape is
72
    unit-testable. argv[0] is the bare binary name; ``LoadGenerator`` resolves
73
    the real path (or reports the binary missing).
74
    """
75
    if tier not in LOAD_TIERS:
76
        raise ValueError(f"unknown load tier {tier!r}; known: {', '.join(sorted(LOAD_TIERS))}")
77
    spec = LOAD_TIERS[tier]
78
    if spec is None:
79
        return None
80
    if timeout_s <= 0:
81
        raise ValueError("timeout_s must be positive: stress-ng's default run length is 24h")
82
    command = [
83
        STRESS_NG,
84
        "--timeout",
85
        f"{int(timeout_s)}s",
86
        "--temp-path",
87
        str(temp_path),
88
        "--seed",
89
        str(int(seed)),
90
        "--metrics-brief",
91
    ]
92
    if "cpu_load_pct" in spec:
93
        command += ["--cpu", str(cpu_worker_count(cpu_count)), "--cpu-load", str(spec["cpu_load_pct"])]
94
    if "hdd_workers" in spec:
95
        command += ["--hdd", str(spec["hdd_workers"]), "--hdd-bytes", str(spec["hdd_bytes"])]
96
    return command
97
98
99
# A stress-ng metrics row: "stress-ng: metrc: [pid] cpu  20704  10.00 ..."
100
# (older releases emit the same row under "info:"). The stressor name is
101
# followed by integer bogo ops then the real-time column. Header lines
102
# ("stressor   bogo ops real time ...") and chatter ("dispatching hogs: 4 cpu")
103
# do not match because no integer directly follows the first word.
104
_BOGO_OPS_ROW = re.compile(
105
    r"^stress-ng:\s+(?:metrc|info):\s+\[\d+\]\s+([a-z][a-z0-9_-]*)\s+(\d+)\s+[0-9]+(?:\.[0-9]+)?\s",
106
    re.MULTILINE,
107
)
108
109
110
def verify_ran(stderr_text: str) -> bool:
111
    """Did the injected load actually run?
112
113
    True when at least one stressor reports positive bogo ops. The numbers are
114
    used ONLY as a did-it-run check; they are never recorded as a benchmark
115
    measurement.
116
    """
117
    return any(int(ops) > 0 for _name, ops in _BOGO_OPS_ROW.findall(stderr_text or ""))
118
119
120
def load_tier_unavailable_reason(detail: str) -> str:
121
    """Canonical skip reason for a tier the host cannot apply."""
122
    return f"load_tier_unavailable:{detail}"
123
124
125
def environment_boundary_fields(
126
    load_tier: str,
127
    before: dict[str, Any] | None,
128
    after: dict[str, Any] | None,
129
    *,
130
    cpu_count: int | None = None,
131
) -> dict[str, Any]:
132
    """Row fields for one measurement window's environment boundary.
133
134
    ``environment_suspect`` keeps the ADR-0002 distinction: None when the
135
    suspicion machinery did not run (a boundary sample is missing), [] when it
136
    ran and the window is clean, [reasons] when suspect. ``load_tier`` is
137
    always the explicit string ("none" included) so it is never ambiguous.
138
    """
139
    from oakbench.envwatch import environment_suspect
140
141
    suspect: list[str] | None = None
142
    if before is not None and after is not None:
143
        suspect = environment_suspect(before, after, cpu_count=cpu_count)
144
    return {
145
        "load_tier": load_tier,
146
        "environment_before": before,
147
        "environment_after": after,
148
        "environment_suspect": suspect,
149
    }
150
151
152
class LoadGenerator:
153
    """Owns one stress-ng subprocess for a measurement window.
154
155
    start() before the timed measurements, stop() after (both untimed).
156
    Honesty contract: start() never raises for an absent binary — it returns
157
    ``{"started": False, "reason": "stress_ng_missing"}`` so the lane can emit
158
    skip rows instead of claiming load that was not applied.
159
    """
160
161
    def __init__(
162
        self,
163
        tier: str,
164
        temp_path: str | Path,
165
        *,
166
        timeout_s: int | float = DEFAULT_TIMEOUT_S,
167
        seed: int = DEFAULT_SEED,
168
        cpu_count: int | None = None,
169
        stress_ng_path: str | None = None,
170
    ) -> None:
171
        if tier not in LOAD_TIERS:
172
            raise ValueError(f"unknown load tier {tier!r}; known: {', '.join(sorted(LOAD_TIERS))}")
173
        self.tier = tier
174
        self.temp_path = Path(temp_path)
175
        self.timeout_s = timeout_s
176
        self.seed = seed
177
        self.cpu_count = cpu_count
178
        self.stress_ng_path = stress_ng_path
179
        self._proc: subprocess.Popen[bytes] | None = None
180
        self._stderr_handle: Any = None
181
182
    def start(self) -> dict[str, Any]:
183
        command = stress_ng_command(
184
            self.tier,
185
            timeout_s=self.timeout_s,
186
            temp_path=self.temp_path,
187
            seed=self.seed,
188
            cpu_count=self.cpu_count,
189
        )
190
        if command is None:
191
            return {"started": False, "reason": "tier_none"}
192
        binary = self.stress_ng_path or shutil.which(STRESS_NG)
193
        if binary is None:
194
            return {"started": False, "reason": "stress_ng_missing"}
195
        self.temp_path.mkdir(parents=True, exist_ok=True)
196
        self._stderr_handle = tempfile.TemporaryFile()
197
        try:
198
            self._proc = subprocess.Popen(
199
                [binary, *command[1:]],
200
                cwd=self.temp_path,
201
                stdout=subprocess.DEVNULL,
202
                stderr=self._stderr_handle,
203
            )
204
        except OSError as error:
205
            self._close_stderr()
206
            return {"started": False, "reason": f"spawn_failed: {type(error).__name__}: {error}"}
207
        return {"started": True, "reason": None}
208
209
    def stop(self) -> dict[str, Any]:
210
        """Terminate the generator (untimed) and verify the load ran.
211
212
        ``verified`` is True only when stress-ng's own bogo-ops metrics confirm
213
        work happened; None when no process ever started (unmeasured, ADR-0002).
214
        """
215
        if self._proc is None:
216
            self._close_stderr()
217
            return {"stopped": False, "returncode": None, "verified": None, "stderr_tail": None}
218
        proc = self._proc
219
        self._proc = None
220
        proc.terminate()
221
        try:
222
            returncode = proc.wait(timeout=15)
223
        except subprocess.TimeoutExpired:
224
            proc.kill()
225
            returncode = proc.wait()
226
        stderr_text = self._read_stderr()
227
        self._close_stderr()
228
        return {
229
            "stopped": True,
230
            "returncode": returncode,
231
            "verified": verify_ran(stderr_text),
232
            "stderr_tail": stderr_text[-2000:],
233
        }
234
235
    def _read_stderr(self) -> str:
236
        if self._stderr_handle is None:
237
            return ""
238
        try:
239
            self._stderr_handle.flush()
240
            self._stderr_handle.seek(0)
241
            return self._stderr_handle.read().decode("utf-8", "replace")
242
        except (OSError, ValueError):
243
            return ""
244
245
    def _close_stderr(self) -> None:
246
        if self._stderr_handle is not None:
247
            try:
248
                self._stderr_handle.close()
249
            except OSError:
250
                pass
251
            self._stderr_handle = None
252
253
    def __enter__(self) -> "LoadGenerator":
254
        return self
255
256
    def __exit__(self, *exc_info: Any) -> None:
257
        self.stop()