Viewing files at commit 186ae84b74bdCommit detailsView latest

scripts/oakbench/stats.py 322 lines · 13.0 KB · python
1
"""Shared statistical helpers for benchmark reports and scorecards."""
2
3
from __future__ import annotations
4
5
import math
6
import random
7
import statistics
8
from typing import Callable, Iterable
9
10
TAIL_MIN_SAMPLES = {"p95": 20, "p99": 100}
11
12
13
def validate_interval_settings(samples: int, confidence: float) -> None:
14
    if type(samples) is not int or samples <= 0:
15
        raise ValueError("bootstrap samples must be a positive integer")
16
    if (isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or
17
            not math.isfinite(confidence) or not 0 < confidence < 1):
18
        raise ValueError("confidence must be finite and between zero and one")
19
20
21
def percentile_nearest(values: Iterable[float], percentile: float) -> float | None:
22
    ordered = sorted(float(value) for value in values)
23
    if not ordered:
24
        return None
25
    rank = math.ceil((percentile / 100.0) * len(ordered))
26
    index = min(max(rank, 1), len(ordered)) - 1
27
    return ordered[index]
28
29
30
def bootstrap_ci(
31
    values: list[float],
32
    statistic: Callable[[list[float]], float] | str = "median",
33
    *,
34
    samples: int = 2000,
35
    confidence: float = 0.95,
36
    seed: int = 20260609,
37
) -> tuple[float, float]:
38
    if not values:
39
        return float("nan"), float("nan")
40
    if len(values) == 1:
41
        return float(values[0]), float(values[0])
42
43
    if isinstance(statistic, str):
44
        stat = named_statistic(statistic)
45
    else:
46
        stat = statistic
47
48
    rng = random.Random(seed)
49
    n = len(values)
50
    boot = []
51
    for _ in range(samples):
52
        sample = [values[rng.randrange(n)] for _ in range(n)]
53
        boot.append(float(stat(sample)))
54
    boot.sort()
55
    alpha = 1.0 - confidence
56
    low_index = max(0, min(len(boot) - 1, math.floor((alpha / 2.0) * len(boot))))
57
    high_index = max(0, min(len(boot) - 1, math.ceil((1.0 - alpha / 2.0) * len(boot)) - 1))
58
    return boot[low_index], boot[high_index]
59
60
61
def named_statistic(name: str) -> Callable[[list[float]], float]:
62
    if name == "median":
63
        return lambda sample: float(statistics.median(sample))
64
    if name == "mean":
65
        return lambda sample: float(statistics.mean(sample))
66
    if name == "p90":
67
        return lambda sample: _percentile_or_nan(sample, 90.0)
68
    if name == "p95":
69
        return lambda sample: _percentile_or_nan(sample, 95.0)
70
    if name == "p99":
71
        return lambda sample: _percentile_or_nan(sample, 99.0)
72
    raise ValueError(name)
73
74
75
def _percentile_or_nan(sample: list[float], percentile: float) -> float:
76
    value = percentile_nearest(sample, percentile)
77
    return float("nan") if value is None else float(value)
78
79
80
def ratio_ci(
81
    numerator: list[float],
82
    denominator: list[float],
83
    *,
84
    statistic: str = "median",
85
    samples: int = 2000,
86
    confidence: float = 0.95,
87
    seed: int = 20260609,
88
) -> tuple[float, float]:
89
    if not numerator or not denominator:
90
        return float("nan"), float("nan")
91
    stat = named_statistic(statistic)
92
    rng = random.Random(seed)
93
    ratios = []
94
    for _ in range(samples):
95
        n_sample = [numerator[rng.randrange(len(numerator))] for _ in range(len(numerator))]
96
        d_sample = [denominator[rng.randrange(len(denominator))] for _ in range(len(denominator))]
97
        d_value = stat(d_sample)
98
        ratios.append(float("inf") if d_value == 0 else stat(n_sample) / d_value)
99
    ratios.sort()
100
    alpha = 1.0 - confidence
101
    low_index = max(0, min(len(ratios) - 1, math.floor((alpha / 2.0) * len(ratios))))
102
    high_index = max(0, min(len(ratios) - 1, math.ceil((1.0 - alpha / 2.0) * len(ratios)) - 1))
103
    return ratios[low_index], ratios[high_index]
104
105
106
def benjamini_hochberg(p_values: list[float], alpha: float = 0.05) -> list[bool]:
107
    indexed = sorted((p, i) for i, p in enumerate(p_values))
108
    cutoff_rank = 0
109
    m = len(indexed)
110
    for rank, (p, _) in enumerate(indexed, start=1):
111
        if p <= (rank / m) * alpha:
112
            cutoff_rank = rank
113
    rejected = [False] * m
114
    for _, original_index in indexed[:cutoff_rank]:
115
        rejected[original_index] = True
116
    return rejected
117
118
119
def required_n_for_relative_delta(noise_floor_pct: float, target_delta_pct: float, current_n: int) -> int | None:
120
    if target_delta_pct <= 0:
121
        return None
122
    if noise_floor_pct <= target_delta_pct:
123
        return max(1, current_n)
124
    ratio = noise_floor_pct / target_delta_pct
125
    return max(current_n + 1, math.ceil(max(1, current_n) * ratio * ratio))
126
127
128
# --- Statistical upgrades (additive; everything above is untouched) ---------
129
130
# Consistency constant: for normally distributed data, sigma_hat = 1.4826 * MAD,
131
# i.e. the MAD scaled by 1.4826 is an unbiased estimator of the standard
132
# deviation. Its reciprocal is the 0.6745 that appears in the classic
133
# Iglewicz-Hoaglin modified z-score (0.6745 = 1 / 1.4826).
134
MAD_CONSISTENCY = 1.4826
135
136
137
def mad_outliers(values: list[float], *, threshold: float = 14.826) -> dict:
138
    """Flag outliers via MAD-based modified z-scores. Never deletes anything.
139
140
    Algebra (so the threshold semantics are explicit):
141
142
        med  = median(values)
143
        MAD  = median(|x_i - med|)
144
        z_i  = (x_i - med) / (1.4826 * MAD)        # modified z-score
145
             = 0.6745 * (x_i - med) / MAD          # identical: 0.6745 = 1/1.4826
146
147
    Because ``1.4826 * MAD`` estimates one standard deviation on normal data,
148
    ``z_i`` is a robust z-score in sigma units. The hyperfine-style default
149
    threshold ``14.826`` therefore means "more than ~14.8 robust standard
150
    deviations from the median", equivalently a threshold of
151
    ``14.826 * 1.4826 ~= 21.98`` on the raw ratio ``|x_i - med| / MAD``. That
152
    is deliberately far more conservative than the conventional Iglewicz-
153
    Hoaglin cut of 3.5: only blatant escapees (stalls, paging storms) are
154
    flagged, not ordinary timing jitter.
155
156
    Degenerate scale: when ``MAD == 0`` (constant data, or more than half the
157
    samples tied at the median) the estimator is blind — every off-median
158
    point would divide by zero and be flagged at infinity. We document the
159
    opposite policy: MAD == 0 flags NOTHING and reports all modified z-scores
160
    as 0.0. A scale-collapsed sample is a job for eyeballs, not a formula.
161
162
    First-run special case: if index 0 is the SOLE outlier and it is on the
163
    slow side (``values[0] > median``), that is the classic warmup / cold-
164
    cache signature, reported via ``first_run_warmup_hint``. It is a hint to
165
    add warmups or run a cold-cache lane, not a reason to discard the sample.
166
167
    Returns ``{"outlier_indices", "modified_z", "first_run_warmup_hint",
168
    "policy": "flag_never_delete"}``. Flagged rows are metadata for humans
169
    and gates; no caller may delete them (see docs/runner-discipline.md).
170
    """
171
    vals = [float(v) for v in values]
172
    result = {
173
        "outlier_indices": [],
174
        "modified_z": [0.0] * len(vals),
175
        "first_run_warmup_hint": False,
176
        "policy": "flag_never_delete",
177
    }
178
    if not vals:
179
        return result
180
    med = float(statistics.median(vals))
181
    mad = float(statistics.median(abs(v - med) for v in vals))
182
    if mad == 0:
183
        return result  # scale collapsed: flag nothing (documented above)
184
    z_scores = [(v - med) / (MAD_CONSISTENCY * mad) for v in vals]
185
    outlier_indices = [i for i, z in enumerate(z_scores) if abs(z) > threshold]
186
    result["modified_z"] = z_scores
187
    result["outlier_indices"] = outlier_indices
188
    result["first_run_warmup_hint"] = outlier_indices == [0] and vals[0] > med
189
    return result
190
191
192
def adaptive_rep_plan(
193
    samples: list[float],
194
    *,
195
    min_runs: int = 10,
196
    time_budget_s: float,
197
    per_run_s_estimate: float,
198
    stddev_pct_threshold: float = 5.0,
199
    sigma_escapee: float = 3.0,
200
    cap_multiplier: int = 10,
201
) -> dict:
202
    """Pure planning math for adaptive repetitions: floor + top-up + cap.
203
204
    Policy (hyperfine floor, PTS-style variance top-up):
205
206
    - Floor: every cell gets at least ``min_runs`` repetitions.
207
    - Budget: ``time_budget_s / per_run_s_estimate`` is the total run count
208
      the wall-time budget pays for. The cap on total runs is
209
      ``min(min_runs * cap_multiplier, budget_runs)`` — long trials shrink
210
      the cap, short trials let the variance loop breathe.
211
    - Top-up triggers (checked only once the floor is met), in order:
212
        1. ``stddev_above_threshold``: sample %stddev (stdev/|mean| * 100)
213
           is at or above ``stddev_pct_threshold``;
214
        2. ``sigma_escapee``: any sample sits more than ``sigma_escapee``
215
           sample standard deviations from the mean.
216
      Either trigger grants one more floor-sized batch (``min_runs`` extra
217
      runs), clipped to the cap. Callers loop: measure, re-plan, repeat.
218
    - ``budget_exhausted``: a trigger (or the floor itself) wants more runs
219
      but the cap grants none. ``satisfied`` is False — the cell is reported
220
      with the variance it has, never silently blessed.
221
    - ``satisfied``: floor met and no trigger fires; ``additional_runs`` is 0.
222
223
    Returns ``{"satisfied", "additional_runs", "reason", "cap"}`` where
224
    ``reason`` is one of ``below_min_runs | stddev_above_threshold |
225
    sigma_escapee | satisfied | budget_exhausted``.
226
    """
227
    if per_run_s_estimate <= 0:
228
        raise ValueError("per_run_s_estimate must be positive")
229
    n = len(samples)
230
    budget_runs = int(time_budget_s // per_run_s_estimate)
231
    cap = max(0, min(min_runs * cap_multiplier, budget_runs))
232
233
    def plan(additional: int, reason: str) -> dict:
234
        return {
235
            "satisfied": reason == "satisfied",
236
            "additional_runs": additional,
237
            "reason": reason,
238
            "cap": cap,
239
        }
240
241
    if n < min_runs:
242
        grant = min(min_runs, cap) - n
243
        if grant <= 0:
244
            return plan(0, "budget_exhausted")
245
        return plan(grant, "below_min_runs")
246
247
    vals = [float(s) for s in samples]
248
    mean = statistics.fmean(vals)
249
    stdev = statistics.stdev(vals) if n >= 2 else 0.0
250
    if mean != 0:
251
        stddev_pct = abs(stdev / mean) * 100.0
252
    else:
253
        stddev_pct = 0.0 if stdev == 0 else float("inf")
254
255
    trigger = None
256
    if stddev_pct >= stddev_pct_threshold:
257
        trigger = "stddev_above_threshold"
258
    elif stdev > 0 and any(abs(v - mean) > sigma_escapee * stdev for v in vals):
259
        trigger = "sigma_escapee"
260
    if trigger is None:
261
        return plan(0, "satisfied")
262
    grant = min(n + min_runs, cap) - n
263
    if grant <= 0:
264
        return plan(0, "budget_exhausted")
265
    return plan(grant, trigger)
266
267
268
def benjamini_hochberg_detail(p_values: list[float], *, fdr: float = 0.05) -> dict:
269
    """Benjamini-Hochberg step-up with full detail, for full-matrix claims.
270
271
    Additive companion to the legacy ``benjamini_hochberg`` above (which
272
    returns a bare ``list[bool]`` and is imported elsewhere, so its signature
273
    is frozen). New code should prefer this one.
274
275
    Standard step-up procedure: sort the m p-values ascending (a stable sort,
276
    so ties keep input order); find the largest rank k (1-based) such that
277
    ``p_(k) <= (k / m) * fdr``; reject every hypothesis whose p-value is
278
    ``<= p_(k)``. Rejecting by value rather than by sorted position makes the
279
    result stable for ties: p-values equal to the critical one are all
280
    rejected, regardless of how the sort ordered them.
281
282
    Returns ``{"rejected": [bool per input, input order], "critical_index":
283
    k-1 as a 0-based rank in the sorted order (None if nothing rejects),
284
    "threshold": the critical p-value p_(k) (None if nothing rejects)}``.
285
    Empty input rejects nothing.
286
    """
287
    m = len(p_values)
288
    if m == 0:
289
        return {"rejected": [], "critical_index": None, "threshold": None}
290
    order = sorted(range(m), key=lambda i: float(p_values[i]))
291
    critical_index = None
292
    for rank, original in enumerate(order, start=1):
293
        if float(p_values[original]) <= (rank / m) * fdr:
294
            critical_index = rank - 1
295
    if critical_index is None:
296
        return {"rejected": [False] * m, "critical_index": None, "threshold": None}
297
    threshold = float(p_values[order[critical_index]])
298
    rejected = [float(p) <= threshold for p in p_values]
299
    return {"rejected": rejected, "critical_index": critical_index, "threshold": threshold}
300
301
302
def two_stage_confirmation(first_pct: float, second_pct: float, *, threshold_pct: float) -> dict:
303
    """Two-stage confirmation for borderline regression verdicts (pure math).
304
305
    Stage one is the normal devloop comparison; stage two is an independent
306
    re-measurement. A delta is ``confirmed`` only if BOTH stages exceed the
307
    threshold — a single borderline excursion is never a verdict. The first
308
    stage is ``borderline`` when it lands within +/-25% of the threshold
309
    (e.g. threshold 10% -> the 7.5%..12.5% band), which is the band where a
310
    second stage is worth running. Wiring into devloop is future work; this
311
    helper only encodes the decision rule.
312
    """
313
    first_exceeds = first_pct > threshold_pct
314
    second_exceeds = second_pct > threshold_pct
315
    borderline = abs(first_pct - threshold_pct) <= 0.25 * abs(threshold_pct)
316
    return {
317
        "confirmed": first_exceeds and second_exceeds,
318
        "borderline": borderline,
319
        "first_exceeds": first_exceeds,
320
        "second_exceeds": second_exceeds,
321
        "threshold_pct": float(threshold_pct),
322
    }