| 1 | """Shared statistical helpers for benchmark reports and scorecards."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import math
|
| 6 | import random
|
| 7 | import statistics
|
| 8 | from typing import Callable, Iterable
|
| 9 |
|
| 10 | TAIL_MIN_SAMPLES = {"p95": 20, "p99": 100}
|
| 11 |
|
| 12 |
|
| 13 | def validate_interval_settings(samples: int, confidence: float) -> None:
|
| 14 | if type(samples) is not int or samples <= 0:
|
| 15 | raise ValueError("bootstrap samples must be a positive integer")
|
| 16 | if (isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or
|
| 17 | not math.isfinite(confidence) or not 0 < confidence < 1):
|
| 18 | raise ValueError("confidence must be finite and between zero and one")
|
| 19 |
|
| 20 |
|
| 21 | def percentile_nearest(values: Iterable[float], percentile: float) -> float | None:
|
| 22 | ordered = sorted(float(value) for value in values)
|
| 23 | if not ordered:
|
| 24 | return None
|
| 25 | rank = math.ceil((percentile / 100.0) * len(ordered))
|
| 26 | index = min(max(rank, 1), len(ordered)) - 1
|
| 27 | return ordered[index]
|
| 28 |
|
| 29 |
|
| 30 | def bootstrap_ci(
|
| 31 | values: list[float],
|
| 32 | statistic: Callable[[list[float]], float] | str = "median",
|
| 33 | *,
|
| 34 | samples: int = 2000,
|
| 35 | confidence: float = 0.95,
|
| 36 | seed: int = 20260609,
|
| 37 | ) -> tuple[float, float]:
|
| 38 | if not values:
|
| 39 | return float("nan"), float("nan")
|
| 40 | if len(values) == 1:
|
| 41 | return float(values[0]), float(values[0])
|
| 42 |
|
| 43 | if isinstance(statistic, str):
|
| 44 | stat = named_statistic(statistic)
|
| 45 | else:
|
| 46 | stat = statistic
|
| 47 |
|
| 48 | rng = random.Random(seed)
|
| 49 | n = len(values)
|
| 50 | boot = []
|
| 51 | for _ in range(samples):
|
| 52 | sample = [values[rng.randrange(n)] for _ in range(n)]
|
| 53 | boot.append(float(stat(sample)))
|
| 54 | boot.sort()
|
| 55 | alpha = 1.0 - confidence
|
| 56 | low_index = max(0, min(len(boot) - 1, math.floor((alpha / 2.0) * len(boot))))
|
| 57 | high_index = max(0, min(len(boot) - 1, math.ceil((1.0 - alpha / 2.0) * len(boot)) - 1))
|
| 58 | return boot[low_index], boot[high_index]
|
| 59 |
|
| 60 |
|
| 61 | def named_statistic(name: str) -> Callable[[list[float]], float]:
|
| 62 | if name == "median":
|
| 63 | return lambda sample: float(statistics.median(sample))
|
| 64 | if name == "mean":
|
| 65 | return lambda sample: float(statistics.mean(sample))
|
| 66 | if name == "p90":
|
| 67 | return lambda sample: _percentile_or_nan(sample, 90.0)
|
| 68 | if name == "p95":
|
| 69 | return lambda sample: _percentile_or_nan(sample, 95.0)
|
| 70 | if name == "p99":
|
| 71 | return lambda sample: _percentile_or_nan(sample, 99.0)
|
| 72 | raise ValueError(name)
|
| 73 |
|
| 74 |
|
| 75 | def _percentile_or_nan(sample: list[float], percentile: float) -> float:
|
| 76 | value = percentile_nearest(sample, percentile)
|
| 77 | return float("nan") if value is None else float(value)
|
| 78 |
|
| 79 |
|
| 80 | def ratio_ci(
|
| 81 | numerator: list[float],
|
| 82 | denominator: list[float],
|
| 83 | *,
|
| 84 | statistic: str = "median",
|
| 85 | samples: int = 2000,
|
| 86 | confidence: float = 0.95,
|
| 87 | seed: int = 20260609,
|
| 88 | ) -> tuple[float, float]:
|
| 89 | if not numerator or not denominator:
|
| 90 | return float("nan"), float("nan")
|
| 91 | stat = named_statistic(statistic)
|
| 92 | rng = random.Random(seed)
|
| 93 | ratios = []
|
| 94 | for _ in range(samples):
|
| 95 | n_sample = [numerator[rng.randrange(len(numerator))] for _ in range(len(numerator))]
|
| 96 | d_sample = [denominator[rng.randrange(len(denominator))] for _ in range(len(denominator))]
|
| 97 | d_value = stat(d_sample)
|
| 98 | ratios.append(float("inf") if d_value == 0 else stat(n_sample) / d_value)
|
| 99 | ratios.sort()
|
| 100 | alpha = 1.0 - confidence
|
| 101 | low_index = max(0, min(len(ratios) - 1, math.floor((alpha / 2.0) * len(ratios))))
|
| 102 | high_index = max(0, min(len(ratios) - 1, math.ceil((1.0 - alpha / 2.0) * len(ratios)) - 1))
|
| 103 | return ratios[low_index], ratios[high_index]
|
| 104 |
|
| 105 |
|
| 106 | def benjamini_hochberg(p_values: list[float], alpha: float = 0.05) -> list[bool]:
|
| 107 | indexed = sorted((p, i) for i, p in enumerate(p_values))
|
| 108 | cutoff_rank = 0
|
| 109 | m = len(indexed)
|
| 110 | for rank, (p, _) in enumerate(indexed, start=1):
|
| 111 | if p <= (rank / m) * alpha:
|
| 112 | cutoff_rank = rank
|
| 113 | rejected = [False] * m
|
| 114 | for _, original_index in indexed[:cutoff_rank]:
|
| 115 | rejected[original_index] = True
|
| 116 | return rejected
|
| 117 |
|
| 118 |
|
| 119 | def required_n_for_relative_delta(noise_floor_pct: float, target_delta_pct: float, current_n: int) -> int | None:
|
| 120 | if target_delta_pct <= 0:
|
| 121 | return None
|
| 122 | if noise_floor_pct <= target_delta_pct:
|
| 123 | return max(1, current_n)
|
| 124 | ratio = noise_floor_pct / target_delta_pct
|
| 125 | return max(current_n + 1, math.ceil(max(1, current_n) * ratio * ratio))
|
| 126 |
|
| 127 |
|
| 128 | # --- Statistical upgrades (additive; everything above is untouched) ---------
|
| 129 |
|
| 130 | # Consistency constant: for normally distributed data, sigma_hat = 1.4826 * MAD,
|
| 131 | # i.e. the MAD scaled by 1.4826 is an unbiased estimator of the standard
|
| 132 | # deviation. Its reciprocal is the 0.6745 that appears in the classic
|
| 133 | # Iglewicz-Hoaglin modified z-score (0.6745 = 1 / 1.4826).
|
| 134 | MAD_CONSISTENCY = 1.4826
|
| 135 |
|
| 136 |
|
| 137 | def mad_outliers(values: list[float], *, threshold: float = 14.826) -> dict:
|
| 138 | """Flag outliers via MAD-based modified z-scores. Never deletes anything.
|
| 139 |
|
| 140 | Algebra (so the threshold semantics are explicit):
|
| 141 |
|
| 142 | med = median(values)
|
| 143 | MAD = median(|x_i - med|)
|
| 144 | z_i = (x_i - med) / (1.4826 * MAD) # modified z-score
|
| 145 | = 0.6745 * (x_i - med) / MAD # identical: 0.6745 = 1/1.4826
|
| 146 |
|
| 147 | Because ``1.4826 * MAD`` estimates one standard deviation on normal data,
|
| 148 | ``z_i`` is a robust z-score in sigma units. The hyperfine-style default
|
| 149 | threshold ``14.826`` therefore means "more than ~14.8 robust standard
|
| 150 | deviations from the median", equivalently a threshold of
|
| 151 | ``14.826 * 1.4826 ~= 21.98`` on the raw ratio ``|x_i - med| / MAD``. That
|
| 152 | is deliberately far more conservative than the conventional Iglewicz-
|
| 153 | Hoaglin cut of 3.5: only blatant escapees (stalls, paging storms) are
|
| 154 | flagged, not ordinary timing jitter.
|
| 155 |
|
| 156 | Degenerate scale: when ``MAD == 0`` (constant data, or more than half the
|
| 157 | samples tied at the median) the estimator is blind — every off-median
|
| 158 | point would divide by zero and be flagged at infinity. We document the
|
| 159 | opposite policy: MAD == 0 flags NOTHING and reports all modified z-scores
|
| 160 | as 0.0. A scale-collapsed sample is a job for eyeballs, not a formula.
|
| 161 |
|
| 162 | First-run special case: if index 0 is the SOLE outlier and it is on the
|
| 163 | slow side (``values[0] > median``), that is the classic warmup / cold-
|
| 164 | cache signature, reported via ``first_run_warmup_hint``. It is a hint to
|
| 165 | add warmups or run a cold-cache lane, not a reason to discard the sample.
|
| 166 |
|
| 167 | Returns ``{"outlier_indices", "modified_z", "first_run_warmup_hint",
|
| 168 | "policy": "flag_never_delete"}``. Flagged rows are metadata for humans
|
| 169 | and gates; no caller may delete them (see docs/runner-discipline.md).
|
| 170 | """
|
| 171 | vals = [float(v) for v in values]
|
| 172 | result = {
|
| 173 | "outlier_indices": [],
|
| 174 | "modified_z": [0.0] * len(vals),
|
| 175 | "first_run_warmup_hint": False,
|
| 176 | "policy": "flag_never_delete",
|
| 177 | }
|
| 178 | if not vals:
|
| 179 | return result
|
| 180 | med = float(statistics.median(vals))
|
| 181 | mad = float(statistics.median(abs(v - med) for v in vals))
|
| 182 | if mad == 0:
|
| 183 | return result # scale collapsed: flag nothing (documented above)
|
| 184 | z_scores = [(v - med) / (MAD_CONSISTENCY * mad) for v in vals]
|
| 185 | outlier_indices = [i for i, z in enumerate(z_scores) if abs(z) > threshold]
|
| 186 | result["modified_z"] = z_scores
|
| 187 | result["outlier_indices"] = outlier_indices
|
| 188 | result["first_run_warmup_hint"] = outlier_indices == [0] and vals[0] > med
|
| 189 | return result
|
| 190 |
|
| 191 |
|
| 192 | def adaptive_rep_plan(
|
| 193 | samples: list[float],
|
| 194 | *,
|
| 195 | min_runs: int = 10,
|
| 196 | time_budget_s: float,
|
| 197 | per_run_s_estimate: float,
|
| 198 | stddev_pct_threshold: float = 5.0,
|
| 199 | sigma_escapee: float = 3.0,
|
| 200 | cap_multiplier: int = 10,
|
| 201 | ) -> dict:
|
| 202 | """Pure planning math for adaptive repetitions: floor + top-up + cap.
|
| 203 |
|
| 204 | Policy (hyperfine floor, PTS-style variance top-up):
|
| 205 |
|
| 206 | - Floor: every cell gets at least ``min_runs`` repetitions.
|
| 207 | - Budget: ``time_budget_s / per_run_s_estimate`` is the total run count
|
| 208 | the wall-time budget pays for. The cap on total runs is
|
| 209 | ``min(min_runs * cap_multiplier, budget_runs)`` — long trials shrink
|
| 210 | the cap, short trials let the variance loop breathe.
|
| 211 | - Top-up triggers (checked only once the floor is met), in order:
|
| 212 | 1. ``stddev_above_threshold``: sample %stddev (stdev/|mean| * 100)
|
| 213 | is at or above ``stddev_pct_threshold``;
|
| 214 | 2. ``sigma_escapee``: any sample sits more than ``sigma_escapee``
|
| 215 | sample standard deviations from the mean.
|
| 216 | Either trigger grants one more floor-sized batch (``min_runs`` extra
|
| 217 | runs), clipped to the cap. Callers loop: measure, re-plan, repeat.
|
| 218 | - ``budget_exhausted``: a trigger (or the floor itself) wants more runs
|
| 219 | but the cap grants none. ``satisfied`` is False — the cell is reported
|
| 220 | with the variance it has, never silently blessed.
|
| 221 | - ``satisfied``: floor met and no trigger fires; ``additional_runs`` is 0.
|
| 222 |
|
| 223 | Returns ``{"satisfied", "additional_runs", "reason", "cap"}`` where
|
| 224 | ``reason`` is one of ``below_min_runs | stddev_above_threshold |
|
| 225 | sigma_escapee | satisfied | budget_exhausted``.
|
| 226 | """
|
| 227 | if per_run_s_estimate <= 0:
|
| 228 | raise ValueError("per_run_s_estimate must be positive")
|
| 229 | n = len(samples)
|
| 230 | budget_runs = int(time_budget_s // per_run_s_estimate)
|
| 231 | cap = max(0, min(min_runs * cap_multiplier, budget_runs))
|
| 232 |
|
| 233 | def plan(additional: int, reason: str) -> dict:
|
| 234 | return {
|
| 235 | "satisfied": reason == "satisfied",
|
| 236 | "additional_runs": additional,
|
| 237 | "reason": reason,
|
| 238 | "cap": cap,
|
| 239 | }
|
| 240 |
|
| 241 | if n < min_runs:
|
| 242 | grant = min(min_runs, cap) - n
|
| 243 | if grant <= 0:
|
| 244 | return plan(0, "budget_exhausted")
|
| 245 | return plan(grant, "below_min_runs")
|
| 246 |
|
| 247 | vals = [float(s) for s in samples]
|
| 248 | mean = statistics.fmean(vals)
|
| 249 | stdev = statistics.stdev(vals) if n >= 2 else 0.0
|
| 250 | if mean != 0:
|
| 251 | stddev_pct = abs(stdev / mean) * 100.0
|
| 252 | else:
|
| 253 | stddev_pct = 0.0 if stdev == 0 else float("inf")
|
| 254 |
|
| 255 | trigger = None
|
| 256 | if stddev_pct >= stddev_pct_threshold:
|
| 257 | trigger = "stddev_above_threshold"
|
| 258 | elif stdev > 0 and any(abs(v - mean) > sigma_escapee * stdev for v in vals):
|
| 259 | trigger = "sigma_escapee"
|
| 260 | if trigger is None:
|
| 261 | return plan(0, "satisfied")
|
| 262 | grant = min(n + min_runs, cap) - n
|
| 263 | if grant <= 0:
|
| 264 | return plan(0, "budget_exhausted")
|
| 265 | return plan(grant, trigger)
|
| 266 |
|
| 267 |
|
| 268 | def benjamini_hochberg_detail(p_values: list[float], *, fdr: float = 0.05) -> dict:
|
| 269 | """Benjamini-Hochberg step-up with full detail, for full-matrix claims.
|
| 270 |
|
| 271 | Additive companion to the legacy ``benjamini_hochberg`` above (which
|
| 272 | returns a bare ``list[bool]`` and is imported elsewhere, so its signature
|
| 273 | is frozen). New code should prefer this one.
|
| 274 |
|
| 275 | Standard step-up procedure: sort the m p-values ascending (a stable sort,
|
| 276 | so ties keep input order); find the largest rank k (1-based) such that
|
| 277 | ``p_(k) <= (k / m) * fdr``; reject every hypothesis whose p-value is
|
| 278 | ``<= p_(k)``. Rejecting by value rather than by sorted position makes the
|
| 279 | result stable for ties: p-values equal to the critical one are all
|
| 280 | rejected, regardless of how the sort ordered them.
|
| 281 |
|
| 282 | Returns ``{"rejected": [bool per input, input order], "critical_index":
|
| 283 | k-1 as a 0-based rank in the sorted order (None if nothing rejects),
|
| 284 | "threshold": the critical p-value p_(k) (None if nothing rejects)}``.
|
| 285 | Empty input rejects nothing.
|
| 286 | """
|
| 287 | m = len(p_values)
|
| 288 | if m == 0:
|
| 289 | return {"rejected": [], "critical_index": None, "threshold": None}
|
| 290 | order = sorted(range(m), key=lambda i: float(p_values[i]))
|
| 291 | critical_index = None
|
| 292 | for rank, original in enumerate(order, start=1):
|
| 293 | if float(p_values[original]) <= (rank / m) * fdr:
|
| 294 | critical_index = rank - 1
|
| 295 | if critical_index is None:
|
| 296 | return {"rejected": [False] * m, "critical_index": None, "threshold": None}
|
| 297 | threshold = float(p_values[order[critical_index]])
|
| 298 | rejected = [float(p) <= threshold for p in p_values]
|
| 299 | return {"rejected": rejected, "critical_index": critical_index, "threshold": threshold}
|
| 300 |
|
| 301 |
|
| 302 | def two_stage_confirmation(first_pct: float, second_pct: float, *, threshold_pct: float) -> dict:
|
| 303 | """Two-stage confirmation for borderline regression verdicts (pure math).
|
| 304 |
|
| 305 | Stage one is the normal devloop comparison; stage two is an independent
|
| 306 | re-measurement. A delta is ``confirmed`` only if BOTH stages exceed the
|
| 307 | threshold — a single borderline excursion is never a verdict. The first
|
| 308 | stage is ``borderline`` when it lands within +/-25% of the threshold
|
| 309 | (e.g. threshold 10% -> the 7.5%..12.5% band), which is the band where a
|
| 310 | second stage is worth running. Wiring into devloop is future work; this
|
| 311 | helper only encodes the decision rule.
|
| 312 | """
|
| 313 | first_exceeds = first_pct > threshold_pct
|
| 314 | second_exceeds = second_pct > threshold_pct
|
| 315 | borderline = abs(first_pct - threshold_pct) <= 0.25 * abs(threshold_pct)
|
| 316 | return {
|
| 317 | "confirmed": first_exceeds and second_exceeds,
|
| 318 | "borderline": borderline,
|
| 319 | "first_exceeds": first_exceeds,
|
| 320 | "second_exceeds": second_exceeds,
|
| 321 | "threshold_pct": float(threshold_pct),
|
| 322 | }
|