Log in
scripts/runner_calibration.py 139 lines · 4.3 KB · blame Source
88e86b3faccd Rebuild benchmarks as a clean single-root reposi 2 months ago
1
#!/usr/bin/env python3
2
"""Score the runner with a fixed ~20-second micro-suite (advisory only).
3
4
Three probes β€” CPU (sha256 over deterministic bytes), IO (write+fsync+read a
5
temp file), spawn (subprocess startup) β€” stamped with the runner identity from
6
oakbench.runner.  No thresholds and no exit-code judgment: calibration numbers
7
contextualize latency rows, they never gate them.
8
"""
9
10
from __future__ import annotations
11
12
import argparse
13
import hashlib
14
import json
15
import os
16
import random
17
import statistics
18
import subprocess
19
import sys
20
import tempfile
21
import time
22
from pathlib import Path
23
from typing import Any
24
25
ROOT = Path(__file__).resolve().parents[1]
26
sys.path.insert(0, str(ROOT / "scripts"))
27
28
from oakbench.runner import machine_profile, runner_class, runner_id  # noqa: E402
29
30
PROBE_BYTES = 64 * 1024 * 1024
31
QUICK_DIVISOR = 8
32
CPU_PASSES = 4
33
SPAWN_COUNT = 50
34
SPAWN_TIMEOUT_S = 30
35
DATA_SEED = 20260612
36
37
38
def parse_args() -> argparse.Namespace:
39
    parser = argparse.ArgumentParser(description=__doc__)
40
    parser.add_argument("--path", type=Path, default=Path.cwd(), help="Directory for the IO probe temp file")
41
    parser.add_argument("--json", action="store_true", default=True, help="Emit JSON (always on; kept for symmetry)")
42
    parser.add_argument("--quick", action="store_true", help="Scale probe sizes down 8x for tests")
43
    return parser.parse_args()
44
45
46
def deterministic_bytes(size: int, seed: int = DATA_SEED) -> bytes:
47
    return random.Random(seed).randbytes(size)
48
49
50
def probe_cpu_sha256(data: bytes, passes: int = CPU_PASSES) -> float | None:
51
    """sha256 throughput in MiB/s over fixed in-process bytes."""
52
    if not data or passes < 1:
53
        return None
54
    start = time.perf_counter()
55
    for _ in range(passes):
56
        hashlib.sha256(data).digest()
57
    elapsed = time.perf_counter() - start
58
    if elapsed <= 0:
59
        return None
60
    return (len(data) / (1024 * 1024)) * passes / elapsed
61
62
63
def probe_io_write_read(directory: Path, data: bytes) -> float | None:
64
    """write+fsync+read throughput in MiB/s (total bytes moved); None if the path is unusable."""
65
    if not data:
66
        return None
67
    temp_path: Path | None = None
68
    try:
69
        start = time.perf_counter()
70
        with tempfile.NamedTemporaryFile(dir=directory, prefix="oakbench-calib-", delete=False) as handle:
71
            temp_path = Path(handle.name)
72
            handle.write(data)
73
            handle.flush()
74
            os.fsync(handle.fileno())
75
        read_back = temp_path.read_bytes()
76
        elapsed = time.perf_counter() - start
77
    except Exception:
78
        return None
79
    finally:
80
        try:
81
            if temp_path is not None:
82
                temp_path.unlink()
83
        except Exception:
84
            pass
85
    if elapsed <= 0 or len(read_back) != len(data):
86
        return None
87
    return (len(data) * 2 / (1024 * 1024)) / elapsed
88
89
90
def _spawn_once_ms() -> float | None:
91
    try:
92
        start = time.perf_counter()
93
        subprocess.run(
94
            [sys.executable, "-c", "pass"],
95
            stdout=subprocess.DEVNULL,
96
            stderr=subprocess.DEVNULL,
97
            timeout=SPAWN_TIMEOUT_S,
98
            check=True,
99
        )
100
        return (time.perf_counter() - start) * 1000.0
101
    except Exception:
102
        return None
103
104
105
def probe_spawn_median_ms(count: int = SPAWN_COUNT) -> float | None:
106
    samples = [sample for _ in range(count) if (sample := _spawn_once_ms()) is not None]
107
    return statistics.median(samples) if samples else None
108
109
110
def calibration_row(path: Path, quick: bool) -> dict[str, Any]:
111
    divisor = QUICK_DIVISOR if quick else 1
112
    started = time.perf_counter()
113
    profile = machine_profile(path)
114
    try:
115
        klass = runner_class(profile)
116
    except Exception:
117
        klass = None
118
    data = deterministic_bytes(PROBE_BYTES // divisor)
119
    row: dict[str, Any] = {
120
        "schema_version": 1,
121
        "runner_id": runner_id(profile),
122
        "runner_class": klass,
123
        "cpu_sha256_mib_per_s": probe_cpu_sha256(data),
124
        "io_write_read_mib_per_s": probe_io_write_read(path, data),
125
        "spawn_ms_median": probe_spawn_median_ms(max(1, SPAWN_COUNT // divisor)),
126
    }
127
    row["elapsed_s"] = time.perf_counter() - started
128
    return row
129
130
131
def main() -> int:
132
    args = parse_args()
133
    row = calibration_row(args.path, args.quick)
134
    print(json.dumps(row, sort_keys=True))
135
    return 0
136
137
138
if __name__ == "__main__":
139
    raise SystemExit(main())