Viewing files at commit 25881039a2bbCommit detailsView latest

scripts/oakbench/stats.py 314 lines · 12.6 KB · python
1
"""Shared statistical helpers for benchmark reports and scorecards."""
2
3
from __future__ import annotations
4
5
import math
6
import random
7
import statistics
8
from typing import Callable, Iterable
9
10
TAIL_MIN_SAMPLES = {"p95": 20, "p99": 100}
11
12
13
def percentile_nearest(values: Iterable[float], percentile: float) -> float | None:
14
    ordered = sorted(float(value) for value in values)
15
    if not ordered:
16
        return None
17
    rank = math.ceil((percentile / 100.0) * len(ordered))
18
    index = min(max(rank, 1), len(ordered)) - 1
19
    return ordered[index]
20
21
22
def bootstrap_ci(
23
    values: list[float],
24
    statistic: Callable[[list[float]], float] | str = "median",
25
    *,
26
    samples: int = 2000,
27
    confidence: float = 0.95,
28
    seed: int = 20260609,
29
) -> tuple[float, float]:
30
    if not values:
31
        return float("nan"), float("nan")
32
    if len(values) == 1:
33
        return float(values[0]), float(values[0])
34
35
    if isinstance(statistic, str):
36
        stat = named_statistic(statistic)
37
    else:
38
        stat = statistic
39
40
    rng = random.Random(seed)
41
    n = len(values)
42
    boot = []
43
    for _ in range(samples):
44
        sample = [values[rng.randrange(n)] for _ in range(n)]
45
        boot.append(float(stat(sample)))
46
    boot.sort()
47
    alpha = 1.0 - confidence
48
    low_index = max(0, min(len(boot) - 1, math.floor((alpha / 2.0) * len(boot))))
49
    high_index = max(0, min(len(boot) - 1, math.ceil((1.0 - alpha / 2.0) * len(boot)) - 1))
50
    return boot[low_index], boot[high_index]
51
52
53
def named_statistic(name: str) -> Callable[[list[float]], float]:
54
    if name == "median":
55
        return lambda sample: float(statistics.median(sample))
56
    if name == "mean":
57
        return lambda sample: float(statistics.mean(sample))
58
    if name == "p90":
59
        return lambda sample: _percentile_or_nan(sample, 90.0)
60
    if name == "p95":
61
        return lambda sample: _percentile_or_nan(sample, 95.0)
62
    if name == "p99":
63
        return lambda sample: _percentile_or_nan(sample, 99.0)
64
    raise ValueError(name)
65
66
67
def _percentile_or_nan(sample: list[float], percentile: float) -> float:
68
    value = percentile_nearest(sample, percentile)
69
    return float("nan") if value is None else float(value)
70
71
72
def ratio_ci(
73
    numerator: list[float],
74
    denominator: list[float],
75
    *,
76
    statistic: str = "median",
77
    samples: int = 2000,
78
    confidence: float = 0.95,
79
    seed: int = 20260609,
80
) -> tuple[float, float]:
81
    if not numerator or not denominator:
82
        return float("nan"), float("nan")
83
    stat = named_statistic(statistic)
84
    rng = random.Random(seed)
85
    ratios = []
86
    for _ in range(samples):
87
        n_sample = [numerator[rng.randrange(len(numerator))] for _ in range(len(numerator))]
88
        d_sample = [denominator[rng.randrange(len(denominator))] for _ in range(len(denominator))]
89
        d_value = stat(d_sample)
90
        ratios.append(float("inf") if d_value == 0 else stat(n_sample) / d_value)
91
    ratios.sort()
92
    alpha = 1.0 - confidence
93
    low_index = max(0, min(len(ratios) - 1, math.floor((alpha / 2.0) * len(ratios))))
94
    high_index = max(0, min(len(ratios) - 1, math.ceil((1.0 - alpha / 2.0) * len(ratios)) - 1))
95
    return ratios[low_index], ratios[high_index]
96
97
98
def benjamini_hochberg(p_values: list[float], alpha: float = 0.05) -> list[bool]:
99
    indexed = sorted((p, i) for i, p in enumerate(p_values))
100
    cutoff_rank = 0
101
    m = len(indexed)
102
    for rank, (p, _) in enumerate(indexed, start=1):
103
        if p <= (rank / m) * alpha:
104
            cutoff_rank = rank
105
    rejected = [False] * m
106
    for _, original_index in indexed[:cutoff_rank]:
107
        rejected[original_index] = True
108
    return rejected
109
110
111
def required_n_for_relative_delta(noise_floor_pct: float, target_delta_pct: float, current_n: int) -> int | None:
112
    if target_delta_pct <= 0:
113
        return None
114
    if noise_floor_pct <= target_delta_pct:
115
        return max(1, current_n)
116
    ratio = noise_floor_pct / target_delta_pct
117
    return max(current_n + 1, math.ceil(max(1, current_n) * ratio * ratio))
118
119
120
# --- Statistical upgrades (additive; everything above is untouched) ---------
121
122
# Consistency constant: for normally distributed data, sigma_hat = 1.4826 * MAD,
123
# i.e. the MAD scaled by 1.4826 is an unbiased estimator of the standard
124
# deviation. Its reciprocal is the 0.6745 that appears in the classic
125
# Iglewicz-Hoaglin modified z-score (0.6745 = 1 / 1.4826).
126
MAD_CONSISTENCY = 1.4826
127
128
129
def mad_outliers(values: list[float], *, threshold: float = 14.826) -> dict:
130
    """Flag outliers via MAD-based modified z-scores. Never deletes anything.
131
132
    Algebra (so the threshold semantics are explicit):
133
134
        med  = median(values)
135
        MAD  = median(|x_i - med|)
136
        z_i  = (x_i - med) / (1.4826 * MAD)        # modified z-score
137
             = 0.6745 * (x_i - med) / MAD          # identical: 0.6745 = 1/1.4826
138
139
    Because ``1.4826 * MAD`` estimates one standard deviation on normal data,
140
    ``z_i`` is a robust z-score in sigma units. The hyperfine-style default
141
    threshold ``14.826`` therefore means "more than ~14.8 robust standard
142
    deviations from the median", equivalently a threshold of
143
    ``14.826 * 1.4826 ~= 21.98`` on the raw ratio ``|x_i - med| / MAD``. That
144
    is deliberately far more conservative than the conventional Iglewicz-
145
    Hoaglin cut of 3.5: only blatant escapees (stalls, paging storms) are
146
    flagged, not ordinary timing jitter.
147
148
    Degenerate scale: when ``MAD == 0`` (constant data, or more than half the
149
    samples tied at the median) the estimator is blind — every off-median
150
    point would divide by zero and be flagged at infinity. We document the
151
    opposite policy: MAD == 0 flags NOTHING and reports all modified z-scores
152
    as 0.0. A scale-collapsed sample is a job for eyeballs, not a formula.
153
154
    First-run special case: if index 0 is the SOLE outlier and it is on the
155
    slow side (``values[0] > median``), that is the classic warmup / cold-
156
    cache signature, reported via ``first_run_warmup_hint``. It is a hint to
157
    add warmups or run a cold-cache lane, not a reason to discard the sample.
158
159
    Returns ``{"outlier_indices", "modified_z", "first_run_warmup_hint",
160
    "policy": "flag_never_delete"}``. Flagged rows are metadata for humans
161
    and gates; no caller may delete them (see docs/runner-discipline.md).
162
    """
163
    vals = [float(v) for v in values]
164
    result = {
165
        "outlier_indices": [],
166
        "modified_z": [0.0] * len(vals),
167
        "first_run_warmup_hint": False,
168
        "policy": "flag_never_delete",
169
    }
170
    if not vals:
171
        return result
172
    med = float(statistics.median(vals))
173
    mad = float(statistics.median(abs(v - med) for v in vals))
174
    if mad == 0:
175
        return result  # scale collapsed: flag nothing (documented above)
176
    z_scores = [(v - med) / (MAD_CONSISTENCY * mad) for v in vals]
177
    outlier_indices = [i for i, z in enumerate(z_scores) if abs(z) > threshold]
178
    result["modified_z"] = z_scores
179
    result["outlier_indices"] = outlier_indices
180
    result["first_run_warmup_hint"] = outlier_indices == [0] and vals[0] > med
181
    return result
182
183
184
def adaptive_rep_plan(
185
    samples: list[float],
186
    *,
187
    min_runs: int = 10,
188
    time_budget_s: float,
189
    per_run_s_estimate: float,
190
    stddev_pct_threshold: float = 5.0,
191
    sigma_escapee: float = 3.0,
192
    cap_multiplier: int = 10,
193
) -> dict:
194
    """Pure planning math for adaptive repetitions: floor + top-up + cap.
195
196
    Policy (hyperfine floor, PTS-style variance top-up):
197
198
    - Floor: every cell gets at least ``min_runs`` repetitions.
199
    - Budget: ``time_budget_s / per_run_s_estimate`` is the total run count
200
      the wall-time budget pays for. The cap on total runs is
201
      ``min(min_runs * cap_multiplier, budget_runs)`` — long trials shrink
202
      the cap, short trials let the variance loop breathe.
203
    - Top-up triggers (checked only once the floor is met), in order:
204
        1. ``stddev_above_threshold``: sample %stddev (stdev/|mean| * 100)
205
           is at or above ``stddev_pct_threshold``;
206
        2. ``sigma_escapee``: any sample sits more than ``sigma_escapee``
207
           sample standard deviations from the mean.
208
      Either trigger grants one more floor-sized batch (``min_runs`` extra
209
      runs), clipped to the cap. Callers loop: measure, re-plan, repeat.
210
    - ``budget_exhausted``: a trigger (or the floor itself) wants more runs
211
      but the cap grants none. ``satisfied`` is False — the cell is reported
212
      with the variance it has, never silently blessed.
213
    - ``satisfied``: floor met and no trigger fires; ``additional_runs`` is 0.
214
215
    Returns ``{"satisfied", "additional_runs", "reason", "cap"}`` where
216
    ``reason`` is one of ``below_min_runs | stddev_above_threshold |
217
    sigma_escapee | satisfied | budget_exhausted``.
218
    """
219
    if per_run_s_estimate <= 0:
220
        raise ValueError("per_run_s_estimate must be positive")
221
    n = len(samples)
222
    budget_runs = int(time_budget_s // per_run_s_estimate)
223
    cap = max(0, min(min_runs * cap_multiplier, budget_runs))
224
225
    def plan(additional: int, reason: str) -> dict:
226
        return {
227
            "satisfied": reason == "satisfied",
228
            "additional_runs": additional,
229
            "reason": reason,
230
            "cap": cap,
231
        }
232
233
    if n < min_runs:
234
        grant = min(min_runs, cap) - n
235
        if grant <= 0:
236
            return plan(0, "budget_exhausted")
237
        return plan(grant, "below_min_runs")
238
239
    vals = [float(s) for s in samples]
240
    mean = statistics.fmean(vals)
241
    stdev = statistics.stdev(vals) if n >= 2 else 0.0
242
    if mean != 0:
243
        stddev_pct = abs(stdev / mean) * 100.0
244
    else:
245
        stddev_pct = 0.0 if stdev == 0 else float("inf")
246
247
    trigger = None
248
    if stddev_pct >= stddev_pct_threshold:
249
        trigger = "stddev_above_threshold"
250
    elif stdev > 0 and any(abs(v - mean) > sigma_escapee * stdev for v in vals):
251
        trigger = "sigma_escapee"
252
    if trigger is None:
253
        return plan(0, "satisfied")
254
    grant = min(n + min_runs, cap) - n
255
    if grant <= 0:
256
        return plan(0, "budget_exhausted")
257
    return plan(grant, trigger)
258
259
260
def benjamini_hochberg_detail(p_values: list[float], *, fdr: float = 0.05) -> dict:
261
    """Benjamini-Hochberg step-up with full detail, for full-matrix claims.
262
263
    Additive companion to the legacy ``benjamini_hochberg`` above (which
264
    returns a bare ``list[bool]`` and is imported elsewhere, so its signature
265
    is frozen). New code should prefer this one.
266
267
    Standard step-up procedure: sort the m p-values ascending (a stable sort,
268
    so ties keep input order); find the largest rank k (1-based) such that
269
    ``p_(k) <= (k / m) * fdr``; reject every hypothesis whose p-value is
270
    ``<= p_(k)``. Rejecting by value rather than by sorted position makes the
271
    result stable for ties: p-values equal to the critical one are all
272
    rejected, regardless of how the sort ordered them.
273
274
    Returns ``{"rejected": [bool per input, input order], "critical_index":
275
    k-1 as a 0-based rank in the sorted order (None if nothing rejects),
276
    "threshold": the critical p-value p_(k) (None if nothing rejects)}``.
277
    Empty input rejects nothing.
278
    """
279
    m = len(p_values)
280
    if m == 0:
281
        return {"rejected": [], "critical_index": None, "threshold": None}
282
    order = sorted(range(m), key=lambda i: float(p_values[i]))
283
    critical_index = None
284
    for rank, original in enumerate(order, start=1):
285
        if float(p_values[original]) <= (rank / m) * fdr:
286
            critical_index = rank - 1
287
    if critical_index is None:
288
        return {"rejected": [False] * m, "critical_index": None, "threshold": None}
289
    threshold = float(p_values[order[critical_index]])
290
    rejected = [float(p) <= threshold for p in p_values]
291
    return {"rejected": rejected, "critical_index": critical_index, "threshold": threshold}
292
293
294
def two_stage_confirmation(first_pct: float, second_pct: float, *, threshold_pct: float) -> dict:
295
    """Two-stage confirmation for borderline regression verdicts (pure math).
296
297
    Stage one is the normal devloop comparison; stage two is an independent
298
    re-measurement. A delta is ``confirmed`` only if BOTH stages exceed the
299
    threshold — a single borderline excursion is never a verdict. The first
300
    stage is ``borderline`` when it lands within +/-25% of the threshold
301
    (e.g. threshold 10% -> the 7.5%..12.5% band), which is the band where a
302
    second stage is worth running. Wiring into devloop is future work; this
303
    helper only encodes the decision rule.
304
    """
305
    first_exceeds = first_pct > threshold_pct
306
    second_exceeds = second_pct > threshold_pct
307
    borderline = abs(first_pct - threshold_pct) <= 0.25 * abs(threshold_pct)
308
    return {
309
        "confirmed": first_exceeds and second_exceeds,
310
        "borderline": borderline,
311
        "first_exceeds": first_exceeds,
312
        "second_exceeds": second_exceeds,
313
        "threshold_pct": float(threshold_pct),
314
    }