| 1 | """Shared reporting math and formatting.
|
| 2 |
|
| 3 | One definition of every delta so summaries, regression reports, and the
|
| 4 | dashboard cannot disagree about what "vs baseline" means:
|
| 5 |
|
| 6 | - delta_pct: POSITIVE means the value is LOWER/BETTER than the baseline
|
| 7 | (used by summary tables labelled "positive deltas mean better").
|
| 8 | - slower_pct / lower_is_better_pct: POSITIVE means the value is HIGHER/WORSE
|
| 9 | than the baseline (used by regression flagging).
|
| 10 | """
|
| 11 |
|
| 12 | from __future__ import annotations
|
| 13 |
|
| 14 | import statistics
|
| 15 | from typing import Any, Optional
|
| 16 |
|
| 17 | from oakbench.stats import percentile_nearest
|
| 18 |
|
| 19 |
|
| 20 | def delta_pct(base: float, value: float) -> float:
|
| 21 | if base == 0:
|
| 22 | return 0.0
|
| 23 | return ((base - value) / base) * 100.0
|
| 24 |
|
| 25 |
|
| 26 | def slower_pct(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
|
| 27 | if baseline is None or value is None or baseline <= 0:
|
| 28 | return None
|
| 29 | return ((value - baseline) / baseline) * 100.0
|
| 30 |
|
| 31 |
|
| 32 | def lower_is_better_pct(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
|
| 33 | if baseline is None or value is None or baseline <= 0:
|
| 34 | return None
|
| 35 | return ((value - baseline) / baseline) * 100.0
|
| 36 |
|
| 37 |
|
| 38 | def delta_ms(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
|
| 39 | if baseline is None or value is None:
|
| 40 | return None
|
| 41 | return value - baseline
|
| 42 |
|
| 43 |
|
| 44 | # Minimum sample counts for a nearest-rank tail percentile to be more than a
|
| 45 | # relabelled max: below 1/(1-p) samples the percentile IS the max. Reporting
|
| 46 | # null instead (ADR-0002) keeps tail claims honest at small n.
|
| 47 | TAIL_PERCENTILE_MIN_SAMPLES = {"p95": 20, "p99": 100}
|
| 48 |
|
| 49 |
|
| 50 | def tail_latency_summary(values: list[float]) -> dict[str, Any]:
|
| 51 | """p50/p95/p99/max with sample-count honesty.
|
| 52 |
|
| 53 | Agents pay tail latency as stuck tool calls and timeouts; medians hide it.
|
| 54 | Unsupportable percentiles are null, and `tail_supported` says which levels
|
| 55 | this sample count can claim.
|
| 56 | """
|
| 57 | n = len(values)
|
| 58 | summary: dict[str, Any] = {
|
| 59 | "n": n,
|
| 60 | "p50_ms": median(values),
|
| 61 | "max_ms": max(values) if values else None,
|
| 62 | }
|
| 63 | supported: list[str] = []
|
| 64 | for label, pct in (("p95", 95.0), ("p99", 99.0)):
|
| 65 | if n >= TAIL_PERCENTILE_MIN_SAMPLES[label]:
|
| 66 | summary[f"{label}_ms"] = percentile_nearest(values, pct)
|
| 67 | supported.append(label)
|
| 68 | else:
|
| 69 | summary[f"{label}_ms"] = None
|
| 70 | summary["tail_supported"] = supported
|
| 71 | return summary
|
| 72 |
|
| 73 |
|
| 74 | def median(values: list[float]) -> Optional[float]:
|
| 75 | if not values:
|
| 76 | return None
|
| 77 | return float(statistics.median(values))
|
| 78 |
|
| 79 |
|
| 80 | def fmt_ms(value: Optional[float]) -> str:
|
| 81 | if value is None:
|
| 82 | return ""
|
| 83 | return f"{value:.1f}"
|
| 84 |
|
| 85 |
|
| 86 | def fmt_num(value: Optional[float], precision: int = 1) -> str:
|
| 87 | if value is None:
|
| 88 | return ""
|
| 89 | return f"{value:.{precision}f}"
|
| 90 |
|
| 91 |
|
| 92 | def fmt_pct(value: Optional[float]) -> str:
|
| 93 | if value is None:
|
| 94 | return ""
|
| 95 | return f"{value:+.1f}%"
|
| 96 |
|
| 97 |
|
| 98 | def fmt_rate(value: float) -> str:
|
| 99 | return f"{value * 100.0:.1f}%"
|
| 100 |
|
| 101 |
|
| 102 | def fmt_delta_lower_better(base: Optional[float], value: Optional[float], skip: bool = False) -> str:
|
| 103 | """Render delta_pct (positive = better) or empty when not comparable."""
|
| 104 | if skip or base is None or value is None:
|
| 105 | return ""
|
| 106 | return f"{delta_pct(base, value):+.1f}%"
|