Viewing files at commit ef78687eef1aCommit detailsView latest

Browse files
scripts/oakbench/scale_calibration.py 124 lines · 5.7 KB · python
1
"""Quiet-host calibration records: the only way a scale-lane row becomes claimable.
2
3
A scale row's ``claim_eligible`` and ``evidence_class`` are DERIVED, never a
4
free flag: they are "calibrated_quiet_host"/true only when the run was given
5
a calibration record that validates here (content digest, schema, the same
6
runner, a quiet host, a clean identical-binary A/A campaign) AND the row's
7
own host-load sample is quiet. Every calibrated row embeds the full record
8
and its digest, so publish_gate re-validates each row from the row alone; a
9
JSONL hand-edited to ``claim_eligible: true`` (or to the calibrated evidence
10
class without a valid embedded record) is refused.
11
12
Record (JSON)::
13
14
    {"schema_version": 1, "kind": "oakbench_quiet_host_calibration",
15
     "runner_id": "...", "created_utc": "...", "calibration_bench_id": "...",
16
     "cpu_count": 12, "loadavg_per_core_max": 0.08,
17
     "aa_cells": 42, "aa_floor_median_pct": 4.1, "aa_false_claims": 0,
18
     "record_sha256": "<sha256 of the canonical JSON without this key>"}
19
"""
20
21
from __future__ import annotations
22
23
import hashlib
24
import json
25
from typing import Any, Dict, List, Optional, Tuple
26
27
SCHEMA_VERSION = 1
28
KIND = "oakbench_quiet_host_calibration"
29
CALIBRATED_EVIDENCE_CLASS = "calibrated_quiet_host"
30
DEVELOPMENT_EVIDENCE_CLASS = "development_only_loaded_host"
31
# Quiet-host thresholds (policy; change with a version bump).
32
POLICY_VERSION = "2026-09-29.1"
33
MAX_LOAD_PER_CORE = 0.25
34
MIN_AA_CELLS = 20
35
MAX_AA_FLOOR_MEDIAN_PCT = 10.0
36
REQUIRED = ("runner_id", "created_utc", "calibration_bench_id", "cpu_count", "loadavg_per_core_max", "aa_cells",
37
            "aa_floor_median_pct", "aa_false_claims")
38
39
40
def record_digest(record: Dict[str, Any]) -> str:
41
    body = {key: value for key, value in record.items() if key != "record_sha256"}
42
    return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
43
44
45
def seal(record: Dict[str, Any]) -> Dict[str, Any]:
46
    sealed = dict(record)
47
    sealed.update(schema_version=SCHEMA_VERSION, kind=KIND, policy_version=POLICY_VERSION)
48
    sealed["record_sha256"] = record_digest(sealed)
49
    return sealed
50
51
52
def validate(record: Any, runner_id: Optional[str]) -> List[str]:
53
    """Reasons the record does NOT make rows claimable ([] = valid)."""
54
    if not isinstance(record, dict):
55
        return ["calibration record missing or not an object"]
56
    reasons = []
57
    if record.get("schema_version") != SCHEMA_VERSION or record.get("kind") != KIND:
58
        reasons.append("wrong schema_version/kind")
59
    missing = [key for key in REQUIRED if key not in record]
60
    if missing:
61
        reasons.append(f"missing fields {missing}")
62
        return reasons
63
    if record.get("record_sha256") != record_digest(record):
64
        reasons.append("record_sha256 does not match content (edited record)")
65
    if runner_id is None or record.get("runner_id") != runner_id:
66
        reasons.append("calibration was taken on a different runner")
67
    load = record.get("loadavg_per_core_max")
68
    if not isinstance(load, (int, float)) or load > MAX_LOAD_PER_CORE:
69
        reasons.append(f"host not quiet: load/core {load} > {MAX_LOAD_PER_CORE}")
70
    cells = record.get("aa_cells")
71
    if not isinstance(cells, int) or cells < MIN_AA_CELLS:
72
        reasons.append(f"too few A/A cells ({cells} < {MIN_AA_CELLS})")
73
    floor = record.get("aa_floor_median_pct")
74
    if not isinstance(floor, (int, float)) or floor > MAX_AA_FLOOR_MEDIAN_PCT:
75
        reasons.append(f"A/A floor median {floor}% > {MAX_AA_FLOOR_MEDIAN_PCT}%")
76
    if record.get("aa_false_claims") != 0:
77
        reasons.append("identical-binary A/A produced claims")
78
    return reasons
79
80
81
def row_load_per_core(row: Dict[str, Any], cpu_count: Any) -> Optional[float]:
82
    sample = row.get("environment_sample") or {}
83
    load = sample.get("loadavg") if isinstance(sample, dict) else None
84
    if not isinstance(load, list) or not load or not isinstance(cpu_count, int) or cpu_count <= 0:
85
        return None
86
    return float(load[0]) / cpu_count
87
88
89
def derive(row: Dict[str, Any], record: Optional[Dict[str, Any]]) -> Tuple[str, bool, List[str]]:
90
    """(evidence_class, claim_eligible, reasons) for one row."""
91
    reasons = validate(record, row.get("runner_id")) if record is not None else ["no calibration record"]
92
    if not reasons:
93
        per_core = row_load_per_core(row, record.get("cpu_count"))
94
        if per_core is None or per_core > MAX_LOAD_PER_CORE:
95
            reasons.append(f"row host load/core {per_core} not quiet")
96
    if row.get("returncode") != 0:
97
        reasons.append("row is not a successful measurement")
98
    if reasons:
99
        return DEVELOPMENT_EVIDENCE_CLASS, False, reasons
100
    return CALIBRATED_EVIDENCE_CLASS, True, []
101
102
103
def stamp(row: Dict[str, Any], record: Optional[Dict[str, Any]]) -> Dict[str, Any]:
104
    evidence, eligible, reasons = derive(row, record)
105
    row["evidence_class"] = evidence
106
    row["claim_eligible"] = eligible
107
    row["claim_ineligible_reasons"] = reasons
108
    if eligible:
109
        row["calibration_record"] = record
110
        row["calibration_record_sha256"] = record["record_sha256"]
111
    return row
112
113
114
def gate_row(row: Dict[str, Any]) -> List[str]:
115
    """publish_gate: why a scale row may NOT be published ([] = publishable)."""
116
    if row.get("evidence_class") != CALIBRATED_EVIDENCE_CLASS:
117
        return [f"evidence_class {row.get('evidence_class')!r} is not {CALIBRATED_EVIDENCE_CLASS!r}"]
118
    record = row.get("calibration_record")
119
    if row.get("calibration_record_sha256") != (record or {}).get("record_sha256"):
120
        return ["calibration_record_sha256 does not match the embedded record"]
121
    evidence, eligible, reasons = derive(row, record)
122
    if not eligible or row.get("claim_eligible") is not True:
123
        return reasons or ["claim_eligible is not derived true"]
124
    return []