Viewing files at commit 2e081b95a118Commit detailsView latest

scripts/oakbench/reporting.py 114 lines · 3.6 KB · python
1
"""Shared reporting math and formatting.
2
3
One definition of every delta so summaries, regression reports, and the
4
dashboard cannot disagree about what "vs baseline" means:
5
6
- delta_pct: POSITIVE means the value is LOWER/BETTER than the baseline
7
  (used by summary tables labelled "positive deltas mean better").
8
- slower_pct / lower_is_better_pct: POSITIVE means the value is HIGHER/WORSE
9
  than the baseline (used by regression flagging).
10
"""
11
12
from __future__ import annotations
13
14
import math
15
import statistics
16
from typing import Any, Optional
17
18
from oakbench.stats import percentile_nearest
19
20
21
def delta_pct(base: float, value: float) -> float:
22
    if base == 0:
23
        return 0.0
24
    return ((base - value) / base) * 100.0
25
26
27
def slower_pct(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
28
    if baseline is None or value is None or baseline <= 0:
29
        return None
30
    return ((value - baseline) / baseline) * 100.0
31
32
33
def symmetric_ratio_spread_pct(left: Optional[float], right: Optional[float]) -> Optional[float]:
34
    """Null-control spread with no privileged subject or denominator."""
35
    if left is None or right is None or not all(math.isfinite(value) and value > 0 for value in (left, right)):
36
        return None
37
    return slower_pct(min(left, right), max(left, right))
38
39
40
def lower_is_better_pct(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
41
    if baseline is None or value is None or baseline <= 0:
42
        return None
43
    return ((value - baseline) / baseline) * 100.0
44
45
46
def delta_ms(baseline: Optional[float], value: Optional[float]) -> Optional[float]:
47
    if baseline is None or value is None:
48
        return None
49
    return value - baseline
50
51
52
# Minimum sample counts for a nearest-rank tail percentile to be more than a
53
# relabelled max: below 1/(1-p) samples the percentile IS the max. Reporting
54
# null instead (ADR-0002) keeps tail claims honest at small n.
55
TAIL_PERCENTILE_MIN_SAMPLES = {"p95": 20, "p99": 100}
56
57
58
def tail_latency_summary(values: list[float]) -> dict[str, Any]:
59
    """p50/p95/p99/max with sample-count honesty.
60
61
    Agents pay tail latency as stuck tool calls and timeouts; medians hide it.
62
    Unsupportable percentiles are null, and `tail_supported` says which levels
63
    this sample count can claim.
64
    """
65
    n = len(values)
66
    summary: dict[str, Any] = {
67
        "n": n,
68
        "p50_ms": median(values),
69
        "max_ms": max(values) if values else None,
70
    }
71
    supported: list[str] = []
72
    for label, pct in (("p95", 95.0), ("p99", 99.0)):
73
        if n >= TAIL_PERCENTILE_MIN_SAMPLES[label]:
74
            summary[f"{label}_ms"] = percentile_nearest(values, pct)
75
            supported.append(label)
76
        else:
77
            summary[f"{label}_ms"] = None
78
    summary["tail_supported"] = supported
79
    return summary
80
81
82
def median(values: list[float]) -> Optional[float]:
83
    if not values:
84
        return None
85
    return float(statistics.median(values))
86
87
88
def fmt_ms(value: Optional[float]) -> str:
89
    if value is None:
90
        return ""
91
    return f"{value:.1f}"
92
93
94
def fmt_num(value: Optional[float], precision: int = 1) -> str:
95
    if value is None:
96
        return ""
97
    return f"{value:.{precision}f}"
98
99
100
def fmt_pct(value: Optional[float]) -> str:
101
    if value is None:
102
        return ""
103
    return f"{value:+.1f}%"
104
105
106
def fmt_rate(value: float) -> str:
107
    return f"{value * 100.0:.1f}%"
108
109
110
def fmt_delta_lower_better(base: Optional[float], value: Optional[float], skip: bool = False) -> str:
111
    """Render delta_pct (positive = better) or empty when not comparable."""
112
    if skip or base is None or value is None:
113
        return ""
114
    return f"{delta_pct(base, value):+.1f}%"