| 1 | """Shared reporting math and formatting.
|
| 2 |
|
| 3 | One definition of every delta so summaries, regression reports, and the
|
| 4 | dashboard cannot disagree about what "vs baseline" means:
|
| 5 |
|
| 6 | - delta_pct: POSITIVE means the value is LOWER/BETTER than the baseline
|
| 7 | (used by summary tables labelled "positive deltas mean better").
|
| 8 | - slower_pct / lower_is_better_pct: POSITIVE means the value is HIGHER/WORSE
|
| 9 | than the baseline (used by regression flagging).
|
| 10 | """
|
| 11 |
|
| 12 | from __future__ import annotations
|
| 13 |
|
| 14 | import math
|
| 15 | import statistics
|
| 16 | from typing import Any, Optional
|
| 17 |
|
| 18 | from oakbench.stats import percentile_nearest
|
| 19 |
|
| 20 |
|
| 21 | def delta_pct(base: float, value: float) -> float:
|
| 22 | if base == 0:
|
| 23 | return 0.0
|
| 24 | return ((base - value) / base) * 100.0
|
| 25 |
|
| 26 |
|
| 27 | def slower_pct(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
|
| 28 | if baseline is None or value is None or baseline <= 0:
|
| 29 | return None
|
| 30 | return ((value - baseline) / baseline) * 100.0
|
| 31 |
|
| 32 |
|
| 33 | def symmetric_ratio_spread_pct(left: Optional[float], right: Optional[float]) -> Optional[float]:
|
| 34 | """Null-control spread with no privileged subject or denominator."""
|
| 35 | if left is None or right is None or not all(math.isfinite(value) and value > 0 for value in (left, right)):
|
| 36 | return None
|
| 37 | return slower_pct(min(left, right), max(left, right))
|
| 38 |
|
| 39 |
|
| 40 | def lower_is_better_pct(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
|
| 41 | if baseline is None or value is None or baseline <= 0:
|
| 42 | return None
|
| 43 | return ((value - baseline) / baseline) * 100.0
|
| 44 |
|
| 45 |
|
| 46 | def delta_ms(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
|
| 47 | if baseline is None or value is None:
|
| 48 | return None
|
| 49 | return value - baseline
|
| 50 |
|
| 51 |
|
| 52 | # Minimum sample counts for a nearest-rank tail percentile to be more than a
|
| 53 | # relabelled max: below 1/(1-p) samples the percentile IS the max. Reporting
|
| 54 | # null instead (ADR-0002) keeps tail claims honest at small n.
|
| 55 | TAIL_PERCENTILE_MIN_SAMPLES = {"p95": 20, "p99": 100}
|
| 56 |
|
| 57 |
|
| 58 | def tail_latency_summary(values: list[float]) -> dict[str, Any]:
|
| 59 | """p50/p95/p99/max with sample-count honesty.
|
| 60 |
|
| 61 | Agents pay tail latency as stuck tool calls and timeouts; medians hide it.
|
| 62 | Unsupportable percentiles are null, and `tail_supported` says which levels
|
| 63 | this sample count can claim.
|
| 64 | """
|
| 65 | n = len(values)
|
| 66 | summary: dict[str, Any] = {
|
| 67 | "n": n,
|
| 68 | "p50_ms": median(values),
|
| 69 | "max_ms": max(values) if values else None,
|
| 70 | }
|
| 71 | supported: list[str] = []
|
| 72 | for label, pct in (("p95", 95.0), ("p99", 99.0)):
|
| 73 | if n >= TAIL_PERCENTILE_MIN_SAMPLES[label]:
|
| 74 | summary[f"{label}_ms"] = percentile_nearest(values, pct)
|
| 75 | supported.append(label)
|
| 76 | else:
|
| 77 | summary[f"{label}_ms"] = None
|
| 78 | summary["tail_supported"] = supported
|
| 79 | return summary
|
| 80 |
|
| 81 |
|
| 82 | def median(values: list[float]) -> Optional[float]:
|
| 83 | if not values:
|
| 84 | return None
|
| 85 | return float(statistics.median(values))
|
| 86 |
|
| 87 |
|
| 88 | def fmt_ms(value: Optional[float]) -> str:
|
| 89 | if value is None:
|
| 90 | return ""
|
| 91 | return f"{value:.1f}"
|
| 92 |
|
| 93 |
|
| 94 | def fmt_num(value: Optional[float], precision: int = 1) -> str:
|
| 95 | if value is None:
|
| 96 | return ""
|
| 97 | return f"{value:.{precision}f}"
|
| 98 |
|
| 99 |
|
| 100 | def fmt_pct(value: Optional[float]) -> str:
|
| 101 | if value is None:
|
| 102 | return ""
|
| 103 | return f"{value:+.1f}%"
|
| 104 |
|
| 105 |
|
| 106 | def fmt_rate(value: float) -> str:
|
| 107 | return f"{value * 100.0:.1f}%"
|
| 108 |
|
| 109 |
|
| 110 | def fmt_delta_lower_better(base: Optional[float], value: Optional[float], skip: bool = False) -> str:
|
| 111 | """Render delta_pct (positive = better) or empty when not comparable."""
|
| 112 | if skip or base is None or value is None:
|
| 113 | return ""
|
| 114 | return f"{delta_pct(base, value):+.1f}%"
|