88e86b3faccd
Rebuild benchmarks as a clean single-root reposi
2 months ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Score the runner with a fixed ~20-second micro-suite (advisory only).
|
| 3 |
|
| 4 | Three probes β CPU (sha256 over deterministic bytes), IO (write+fsync+read a
|
| 5 | temp file), spawn (subprocess startup) β stamped with the runner identity from
|
| 6 | oakbench.runner. No thresholds and no exit-code judgment: calibration numbers
|
| 7 | contextualize latency rows, they never gate them.
|
| 8 | """
|
| 9 |
|
| 10 | from __future__ import annotations
|
| 11 |
|
| 12 | import argparse
|
| 13 | import hashlib
|
| 14 | import json
|
| 15 | import os
|
| 16 | import random
|
| 17 | import statistics
|
| 18 | import subprocess
|
| 19 | import sys
|
| 20 | import tempfile
|
| 21 | import time
|
| 22 | from pathlib import Path
|
| 23 | from typing import Any
|
| 24 |
|
| 25 | ROOT = Path(__file__).resolve().parents[1]
|
| 26 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 27 |
|
| 28 | from oakbench.runner import machine_profile, runner_class, runner_id # noqa: E402
|
| 29 |
|
| 30 | PROBE_BYTES = 64 * 1024 * 1024
|
| 31 | QUICK_DIVISOR = 8
|
| 32 | CPU_PASSES = 4
|
| 33 | SPAWN_COUNT = 50
|
| 34 | SPAWN_TIMEOUT_S = 30
|
| 35 | DATA_SEED = 20260612
|
| 36 |
|
| 37 |
|
| 38 | def parse_args() -> argparse.Namespace:
|
| 39 | parser = argparse.ArgumentParser(description=__doc__)
|
| 40 | parser.add_argument("--path", type=Path, default=Path.cwd(), help="Directory for the IO probe temp file")
|
| 41 | parser.add_argument("--json", action="store_true", default=True, help="Emit JSON (always on; kept for symmetry)")
|
| 42 | parser.add_argument("--quick", action="store_true", help="Scale probe sizes down 8x for tests")
|
| 43 | return parser.parse_args()
|
| 44 |
|
| 45 |
|
| 46 | def deterministic_bytes(size: int, seed: int = DATA_SEED) -> bytes:
|
| 47 | return random.Random(seed).randbytes(size)
|
| 48 |
|
| 49 |
|
| 50 | def probe_cpu_sha256(data: bytes, passes: int = CPU_PASSES) -> float | None:
|
| 51 | """sha256 throughput in MiB/s over fixed in-process bytes."""
|
| 52 | if not data or passes < 1:
|
| 53 | return None
|
| 54 | start = time.perf_counter()
|
| 55 | for _ in range(passes):
|
| 56 | hashlib.sha256(data).digest()
|
| 57 | elapsed = time.perf_counter() - start
|
| 58 | if elapsed <= 0:
|
| 59 | return None
|
| 60 | return (len(data) / (1024 * 1024)) * passes / elapsed
|
| 61 |
|
| 62 |
|
| 63 | def probe_io_write_read(directory: Path, data: bytes) -> float | None:
|
| 64 | """write+fsync+read throughput in MiB/s (total bytes moved); None if the path is unusable."""
|
| 65 | if not data:
|
| 66 | return None
|
| 67 | temp_path: Path | None = None
|
| 68 | try:
|
| 69 | start = time.perf_counter()
|
| 70 | with tempfile.NamedTemporaryFile(dir=directory, prefix="oakbench-calib-", delete=False) as handle:
|
| 71 | temp_path = Path(handle.name)
|
| 72 | handle.write(data)
|
| 73 | handle.flush()
|
| 74 | os.fsync(handle.fileno())
|
| 75 | read_back = temp_path.read_bytes()
|
| 76 | elapsed = time.perf_counter() - start
|
| 77 | except Exception:
|
| 78 | return None
|
| 79 | finally:
|
| 80 | try:
|
| 81 | if temp_path is not None:
|
| 82 | temp_path.unlink()
|
| 83 | except Exception:
|
| 84 | pass
|
| 85 | if elapsed <= 0 or len(read_back) != len(data):
|
| 86 | return None
|
| 87 | return (len(data) * 2 / (1024 * 1024)) / elapsed
|
| 88 |
|
| 89 |
|
| 90 | def _spawn_once_ms() -> float | None:
|
| 91 | try:
|
| 92 | start = time.perf_counter()
|
| 93 | subprocess.run(
|
| 94 | [sys.executable, "-c", "pass"],
|
| 95 | stdout=subprocess.DEVNULL,
|
| 96 | stderr=subprocess.DEVNULL,
|
| 97 | timeout=SPAWN_TIMEOUT_S,
|
| 98 | check=True,
|
| 99 | )
|
| 100 | return (time.perf_counter() - start) * 1000.0
|
| 101 | except Exception:
|
| 102 | return None
|
| 103 |
|
| 104 |
|
| 105 | def probe_spawn_median_ms(count: int = SPAWN_COUNT) -> float | None:
|
| 106 | samples = [sample for _ in range(count) if (sample := _spawn_once_ms()) is not None]
|
| 107 | return statistics.median(samples) if samples else None
|
| 108 |
|
| 109 |
|
| 110 | def calibration_row(path: Path, quick: bool) -> dict[str, Any]:
|
| 111 | divisor = QUICK_DIVISOR if quick else 1
|
| 112 | started = time.perf_counter()
|
| 113 | profile = machine_profile(path)
|
| 114 | try:
|
| 115 | klass = runner_class(profile)
|
| 116 | except Exception:
|
| 117 | klass = None
|
| 118 | data = deterministic_bytes(PROBE_BYTES // divisor)
|
| 119 | row: dict[str, Any] = {
|
| 120 | "schema_version": 1,
|
| 121 | "runner_id": runner_id(profile),
|
| 122 | "runner_class": klass,
|
| 123 | "cpu_sha256_mib_per_s": probe_cpu_sha256(data),
|
| 124 | "io_write_read_mib_per_s": probe_io_write_read(path, data),
|
| 125 | "spawn_ms_median": probe_spawn_median_ms(max(1, SPAWN_COUNT // divisor)),
|
| 126 | }
|
| 127 | row["elapsed_s"] = time.perf_counter() - started
|
| 128 | return row
|
| 129 |
|
| 130 |
|
| 131 | def main() -> int:
|
| 132 | args = parse_args()
|
| 133 | row = calibration_row(args.path, args.quick)
|
| 134 | print(json.dumps(row, sort_keys=True))
|
| 135 | return 0
|
| 136 |
|
| 137 |
|
| 138 | if __name__ == "__main__":
|
| 139 | raise SystemExit(main())
|