| 1 | """Shared statistical helpers for benchmark reports and scorecards."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import math
|
| 6 | import random
|
| 7 | import statistics
|
| 8 | from typing import Callable, Iterable
|
| 9 |
|
| 10 | TAIL_MIN_SAMPLES = {"p95": 20, "p99": 100}
|
| 11 |
|
| 12 |
|
| 13 | def percentile_nearest(values: Iterable[float], percentile: float) -> float | None:
|
| 14 | ordered = sorted(float(value) for value in values)
|
| 15 | if not ordered:
|
| 16 | return None
|
| 17 | rank = math.ceil((percentile / 100.0) * len(ordered))
|
| 18 | index = min(max(rank, 1), len(ordered)) - 1
|
| 19 | return ordered[index]
|
| 20 |
|
| 21 |
|
| 22 | def bootstrap_ci(
|
| 23 | values: list[float],
|
| 24 | statistic: Callable[[list[float]], float] | str = "median",
|
| 25 | *,
|
| 26 | samples: int = 2000,
|
| 27 | confidence: float = 0.95,
|
| 28 | seed: int = 20260609,
|
| 29 | ) -> tuple[float, float]:
|
| 30 | if not values:
|
| 31 | return float("nan"), float("nan")
|
| 32 | if len(values) == 1:
|
| 33 | return float(values[0]), float(values[0])
|
| 34 |
|
| 35 | if isinstance(statistic, str):
|
| 36 | stat = named_statistic(statistic)
|
| 37 | else:
|
| 38 | stat = statistic
|
| 39 |
|
| 40 | rng = random.Random(seed)
|
| 41 | n = len(values)
|
| 42 | boot = []
|
| 43 | for _ in range(samples):
|
| 44 | sample = [values[rng.randrange(n)] for _ in range(n)]
|
| 45 | boot.append(float(stat(sample)))
|
| 46 | boot.sort()
|
| 47 | alpha = 1.0 - confidence
|
| 48 | low_index = max(0, min(len(boot) - 1, math.floor((alpha / 2.0) * len(boot))))
|
| 49 | high_index = max(0, min(len(boot) - 1, math.ceil((1.0 - alpha / 2.0) * len(boot)) - 1))
|
| 50 | return boot[low_index], boot[high_index]
|
| 51 |
|
| 52 |
|
| 53 | def named_statistic(name: str) -> Callable[[list[float]], float]:
|
| 54 | if name == "median":
|
| 55 | return lambda sample: float(statistics.median(sample))
|
| 56 | if name == "mean":
|
| 57 | return lambda sample: float(statistics.mean(sample))
|
| 58 | if name == "p90":
|
| 59 | return lambda sample: _percentile_or_nan(sample, 90.0)
|
| 60 | if name == "p95":
|
| 61 | return lambda sample: _percentile_or_nan(sample, 95.0)
|
| 62 | if name == "p99":
|
| 63 | return lambda sample: _percentile_or_nan(sample, 99.0)
|
| 64 | raise ValueError(name)
|
| 65 |
|
| 66 |
|
| 67 | def _percentile_or_nan(sample: list[float], percentile: float) -> float:
|
| 68 | value = percentile_nearest(sample, percentile)
|
| 69 | return float("nan") if value is None else float(value)
|
| 70 |
|
| 71 |
|
| 72 | def ratio_ci(
|
| 73 | numerator: list[float],
|
| 74 | denominator: list[float],
|
| 75 | *,
|
| 76 | statistic: str = "median",
|
| 77 | samples: int = 2000,
|
| 78 | confidence: float = 0.95,
|
| 79 | seed: int = 20260609,
|
| 80 | ) -> tuple[float, float]:
|
| 81 | if not numerator or not denominator:
|
| 82 | return float("nan"), float("nan")
|
| 83 | stat = named_statistic(statistic)
|
| 84 | rng = random.Random(seed)
|
| 85 | ratios = []
|
| 86 | for _ in range(samples):
|
| 87 | n_sample = [numerator[rng.randrange(len(numerator))] for _ in range(len(numerator))]
|
| 88 | d_sample = [denominator[rng.randrange(len(denominator))] for _ in range(len(denominator))]
|
| 89 | d_value = stat(d_sample)
|
| 90 | ratios.append(float("inf") if d_value == 0 else stat(n_sample) / d_value)
|
| 91 | ratios.sort()
|
| 92 | alpha = 1.0 - confidence
|
| 93 | low_index = max(0, min(len(ratios) - 1, math.floor((alpha / 2.0) * len(ratios))))
|
| 94 | high_index = max(0, min(len(ratios) - 1, math.ceil((1.0 - alpha / 2.0) * len(ratios)) - 1))
|
| 95 | return ratios[low_index], ratios[high_index]
|
| 96 |
|
| 97 |
|
| 98 | def benjamini_hochberg(p_values: list[float], alpha: float = 0.05) -> list[bool]:
|
| 99 | indexed = sorted((p, i) for i, p in enumerate(p_values))
|
| 100 | cutoff_rank = 0
|
| 101 | m = len(indexed)
|
| 102 | for rank, (p, _) in enumerate(indexed, start=1):
|
| 103 | if p <= (rank / m) * alpha:
|
| 104 | cutoff_rank = rank
|
| 105 | rejected = [False] * m
|
| 106 | for _, original_index in indexed[:cutoff_rank]:
|
| 107 | rejected[original_index] = True
|
| 108 | return rejected
|
| 109 |
|
| 110 |
|
| 111 | def required_n_for_relative_delta(noise_floor_pct: float, target_delta_pct: float, current_n: int) -> int | None:
|
| 112 | if target_delta_pct <= 0:
|
| 113 | return None
|
| 114 | if noise_floor_pct <= target_delta_pct:
|
| 115 | return max(1, current_n)
|
| 116 | ratio = noise_floor_pct / target_delta_pct
|
| 117 | return max(current_n + 1, math.ceil(max(1, current_n) * ratio * ratio))
|
| 118 |
|
| 119 |
|
| 120 | # --- Statistical upgrades (additive; everything above is untouched) ---------
|
| 121 |
|
| 122 | # Consistency constant: for normally distributed data, sigma_hat = 1.4826 * MAD,
|
| 123 | # i.e. the MAD scaled by 1.4826 is an unbiased estimator of the standard
|
| 124 | # deviation. Its reciprocal is the 0.6745 that appears in the classic
|
| 125 | # Iglewicz-Hoaglin modified z-score (0.6745 = 1 / 1.4826).
|
| 126 | MAD_CONSISTENCY = 1.4826
|
| 127 |
|
| 128 |
|
| 129 | def mad_outliers(values: list[float], *, threshold: float = 14.826) -> dict:
|
| 130 | """Flag outliers via MAD-based modified z-scores. Never deletes anything.
|
| 131 |
|
| 132 | Algebra (so the threshold semantics are explicit):
|
| 133 |
|
| 134 | med = median(values)
|
| 135 | MAD = median(|x_i - med|)
|
| 136 | z_i = (x_i - med) / (1.4826 * MAD) # modified z-score
|
| 137 | = 0.6745 * (x_i - med) / MAD # identical: 0.6745 = 1/1.4826
|
| 138 |
|
| 139 | Because ``1.4826 * MAD`` estimates one standard deviation on normal data,
|
| 140 | ``z_i`` is a robust z-score in sigma units. The hyperfine-style default
|
| 141 | threshold ``14.826`` therefore means "more than ~14.8 robust standard
|
| 142 | deviations from the median", equivalently a threshold of
|
| 143 | ``14.826 * 1.4826 ~= 21.98`` on the raw ratio ``|x_i - med| / MAD``. That
|
| 144 | is deliberately far more conservative than the conventional Iglewicz-
|
| 145 | Hoaglin cut of 3.5: only blatant escapees (stalls, paging storms) are
|
| 146 | flagged, not ordinary timing jitter.
|
| 147 |
|
| 148 | Degenerate scale: when ``MAD == 0`` (constant data, or more than half the
|
| 149 | samples tied at the median) the estimator is blind — every off-median
|
| 150 | point would divide by zero and be flagged at infinity. We document the
|
| 151 | opposite policy: MAD == 0 flags NOTHING and reports all modified z-scores
|
| 152 | as 0.0. A scale-collapsed sample is a job for eyeballs, not a formula.
|
| 153 |
|
| 154 | First-run special case: if index 0 is the SOLE outlier and it is on the
|
| 155 | slow side (``values[0] > median``), that is the classic warmup / cold-
|
| 156 | cache signature, reported via ``first_run_warmup_hint``. It is a hint to
|
| 157 | add warmups or run a cold-cache lane, not a reason to discard the sample.
|
| 158 |
|
| 159 | Returns ``{"outlier_indices", "modified_z", "first_run_warmup_hint",
|
| 160 | "policy": "flag_never_delete"}``. Flagged rows are metadata for humans
|
| 161 | and gates; no caller may delete them (see docs/runner-discipline.md).
|
| 162 | """
|
| 163 | vals = [float(v) for v in values]
|
| 164 | result = {
|
| 165 | "outlier_indices": [],
|
| 166 | "modified_z": [0.0] * len(vals),
|
| 167 | "first_run_warmup_hint": False,
|
| 168 | "policy": "flag_never_delete",
|
| 169 | }
|
| 170 | if not vals:
|
| 171 | return result
|
| 172 | med = float(statistics.median(vals))
|
| 173 | mad = float(statistics.median(abs(v - med) for v in vals))
|
| 174 | if mad == 0:
|
| 175 | return result # scale collapsed: flag nothing (documented above)
|
| 176 | z_scores = [(v - med) / (MAD_CONSISTENCY * mad) for v in vals]
|
| 177 | outlier_indices = [i for i, z in enumerate(z_scores) if abs(z) > threshold]
|
| 178 | result["modified_z"] = z_scores
|
| 179 | result["outlier_indices"] = outlier_indices
|
| 180 | result["first_run_warmup_hint"] = outlier_indices == [0] and vals[0] > med
|
| 181 | return result
|
| 182 |
|
| 183 |
|
| 184 | def adaptive_rep_plan(
|
| 185 | samples: list[float],
|
| 186 | *,
|
| 187 | min_runs: int = 10,
|
| 188 | time_budget_s: float,
|
| 189 | per_run_s_estimate: float,
|
| 190 | stddev_pct_threshold: float = 5.0,
|
| 191 | sigma_escapee: float = 3.0,
|
| 192 | cap_multiplier: int = 10,
|
| 193 | ) -> dict:
|
| 194 | """Pure planning math for adaptive repetitions: floor + top-up + cap.
|
| 195 |
|
| 196 | Policy (hyperfine floor, PTS-style variance top-up):
|
| 197 |
|
| 198 | - Floor: every cell gets at least ``min_runs`` repetitions.
|
| 199 | - Budget: ``time_budget_s / per_run_s_estimate`` is the total run count
|
| 200 | the wall-time budget pays for. The cap on total runs is
|
| 201 | ``min(min_runs * cap_multiplier, budget_runs)`` — long trials shrink
|
| 202 | the cap, short trials let the variance loop breathe.
|
| 203 | - Top-up triggers (checked only once the floor is met), in order:
|
| 204 | 1. ``stddev_above_threshold``: sample %stddev (stdev/|mean| * 100)
|
| 205 | is at or above ``stddev_pct_threshold``;
|
| 206 | 2. ``sigma_escapee``: any sample sits more than ``sigma_escapee``
|
| 207 | sample standard deviations from the mean.
|
| 208 | Either trigger grants one more floor-sized batch (``min_runs`` extra
|
| 209 | runs), clipped to the cap. Callers loop: measure, re-plan, repeat.
|
| 210 | - ``budget_exhausted``: a trigger (or the floor itself) wants more runs
|
| 211 | but the cap grants none. ``satisfied`` is False — the cell is reported
|
| 212 | with the variance it has, never silently blessed.
|
| 213 | - ``satisfied``: floor met and no trigger fires; ``additional_runs`` is 0.
|
| 214 |
|
| 215 | Returns ``{"satisfied", "additional_runs", "reason", "cap"}`` where
|
| 216 | ``reason`` is one of ``below_min_runs | stddev_above_threshold |
|
| 217 | sigma_escapee | satisfied | budget_exhausted``.
|
| 218 | """
|
| 219 | if per_run_s_estimate <= 0:
|
| 220 | raise ValueError("per_run_s_estimate must be positive")
|
| 221 | n = len(samples)
|
| 222 | budget_runs = int(time_budget_s // per_run_s_estimate)
|
| 223 | cap = max(0, min(min_runs * cap_multiplier, budget_runs))
|
| 224 |
|
| 225 | def plan(additional: int, reason: str) -> dict:
|
| 226 | return {
|
| 227 | "satisfied": reason == "satisfied",
|
| 228 | "additional_runs": additional,
|
| 229 | "reason": reason,
|
| 230 | "cap": cap,
|
| 231 | }
|
| 232 |
|
| 233 | if n < min_runs:
|
| 234 | grant = min(min_runs, cap) - n
|
| 235 | if grant <= 0:
|
| 236 | return plan(0, "budget_exhausted")
|
| 237 | return plan(grant, "below_min_runs")
|
| 238 |
|
| 239 | vals = [float(s) for s in samples]
|
| 240 | mean = statistics.fmean(vals)
|
| 241 | stdev = statistics.stdev(vals) if n >= 2 else 0.0
|
| 242 | if mean != 0:
|
| 243 | stddev_pct = abs(stdev / mean) * 100.0
|
| 244 | else:
|
| 245 | stddev_pct = 0.0 if stdev == 0 else float("inf")
|
| 246 |
|
| 247 | trigger = None
|
| 248 | if stddev_pct >= stddev_pct_threshold:
|
| 249 | trigger = "stddev_above_threshold"
|
| 250 | elif stdev > 0 and any(abs(v - mean) > sigma_escapee * stdev for v in vals):
|
| 251 | trigger = "sigma_escapee"
|
| 252 | if trigger is None:
|
| 253 | return plan(0, "satisfied")
|
| 254 | grant = min(n + min_runs, cap) - n
|
| 255 | if grant <= 0:
|
| 256 | return plan(0, "budget_exhausted")
|
| 257 | return plan(grant, trigger)
|
| 258 |
|
| 259 |
|
| 260 | def benjamini_hochberg_detail(p_values: list[float], *, fdr: float = 0.05) -> dict:
|
| 261 | """Benjamini-Hochberg step-up with full detail, for full-matrix claims.
|
| 262 |
|
| 263 | Additive companion to the legacy ``benjamini_hochberg`` above (which
|
| 264 | returns a bare ``list[bool]`` and is imported elsewhere, so its signature
|
| 265 | is frozen). New code should prefer this one.
|
| 266 |
|
| 267 | Standard step-up procedure: sort the m p-values ascending (a stable sort,
|
| 268 | so ties keep input order); find the largest rank k (1-based) such that
|
| 269 | ``p_(k) <= (k / m) * fdr``; reject every hypothesis whose p-value is
|
| 270 | ``<= p_(k)``. Rejecting by value rather than by sorted position makes the
|
| 271 | result stable for ties: p-values equal to the critical one are all
|
| 272 | rejected, regardless of how the sort ordered them.
|
| 273 |
|
| 274 | Returns ``{"rejected": [bool per input, input order], "critical_index":
|
| 275 | k-1 as a 0-based rank in the sorted order (None if nothing rejects),
|
| 276 | "threshold": the critical p-value p_(k) (None if nothing rejects)}``.
|
| 277 | Empty input rejects nothing.
|
| 278 | """
|
| 279 | m = len(p_values)
|
| 280 | if m == 0:
|
| 281 | return {"rejected": [], "critical_index": None, "threshold": None}
|
| 282 | order = sorted(range(m), key=lambda i: float(p_values[i]))
|
| 283 | critical_index = None
|
| 284 | for rank, original in enumerate(order, start=1):
|
| 285 | if float(p_values[original]) <= (rank / m) * fdr:
|
| 286 | critical_index = rank - 1
|
| 287 | if critical_index is None:
|
| 288 | return {"rejected": [False] * m, "critical_index": None, "threshold": None}
|
| 289 | threshold = float(p_values[order[critical_index]])
|
| 290 | rejected = [float(p) <= threshold for p in p_values]
|
| 291 | return {"rejected": rejected, "critical_index": critical_index, "threshold": threshold}
|
| 292 |
|
| 293 |
|
| 294 | def two_stage_confirmation(first_pct: float, second_pct: float, *, threshold_pct: float) -> dict:
|
| 295 | """Two-stage confirmation for borderline regression verdicts (pure math).
|
| 296 |
|
| 297 | Stage one is the normal devloop comparison; stage two is an independent
|
| 298 | re-measurement. A delta is ``confirmed`` only if BOTH stages exceed the
|
| 299 | threshold — a single borderline excursion is never a verdict. The first
|
| 300 | stage is ``borderline`` when it lands within +/-25% of the threshold
|
| 301 | (e.g. threshold 10% -> the 7.5%..12.5% band), which is the band where a
|
| 302 | second stage is worth running. Wiring into devloop is future work; this
|
| 303 | helper only encodes the decision rule.
|
| 304 | """
|
| 305 | first_exceeds = first_pct > threshold_pct
|
| 306 | second_exceeds = second_pct > threshold_pct
|
| 307 | borderline = abs(first_pct - threshold_pct) <= 0.25 * abs(threshold_pct)
|
| 308 | return {
|
| 309 | "confirmed": first_exceeds and second_exceeds,
|
| 310 | "borderline": borderline,
|
| 311 | "first_exceeds": first_exceeds,
|
| 312 | "second_exceeds": second_exceeds,
|
| 313 | "threshold_pct": float(threshold_pct),
|
| 314 | }
|