| 1 | """Quiet-host calibration records: the only way a scale-lane row becomes claimable.
|
| 2 |
|
| 3 | A scale row's ``claim_eligible`` and ``evidence_class`` are DERIVED, never a
|
| 4 | free flag: they are "calibrated_quiet_host"/true only when the run was given
|
| 5 | a calibration record that validates here (content digest, schema, the same
|
| 6 | runner, a quiet host, a clean identical-binary A/A campaign) AND the row's
|
| 7 | own host-load sample is quiet. Every calibrated row embeds the full record
|
| 8 | and its digest, so publish_gate re-validates each row from the row alone; a
|
| 9 | JSONL hand-edited to ``claim_eligible: true`` (or to the calibrated evidence
|
| 10 | class without a valid embedded record) is refused.
|
| 11 |
|
| 12 | Record (JSON)::
|
| 13 |
|
| 14 | {"schema_version": 1, "kind": "oakbench_quiet_host_calibration",
|
| 15 | "runner_id": "...", "created_utc": "...", "calibration_bench_id": "...",
|
| 16 | "cpu_count": 12, "loadavg_per_core_max": 0.08,
|
| 17 | "aa_cells": 42, "aa_floor_median_pct": 4.1, "aa_false_claims": 0,
|
| 18 | "record_sha256": "<sha256 of the canonical JSON without this key>"}
|
| 19 | """
|
| 20 |
|
| 21 | from __future__ import annotations
|
| 22 |
|
| 23 | import hashlib
|
| 24 | import json
|
| 25 | from typing import Any, Dict, List, Optional, Tuple
|
| 26 |
|
| 27 | SCHEMA_VERSION = 1
|
| 28 | KIND = "oakbench_quiet_host_calibration"
|
| 29 | CALIBRATED_EVIDENCE_CLASS = "calibrated_quiet_host"
|
| 30 | DEVELOPMENT_EVIDENCE_CLASS = "development_only_loaded_host"
|
| 31 | # Quiet-host thresholds (policy; change with a version bump).
|
| 32 | POLICY_VERSION = "2026-09-29.1"
|
| 33 | MAX_LOAD_PER_CORE = 0.25
|
| 34 | MIN_AA_CELLS = 20
|
| 35 | MAX_AA_FLOOR_MEDIAN_PCT = 10.0
|
| 36 | REQUIRED = ("runner_id", "created_utc", "calibration_bench_id", "cpu_count", "loadavg_per_core_max", "aa_cells",
|
| 37 | "aa_floor_median_pct", "aa_false_claims")
|
| 38 |
|
| 39 |
|
| 40 | def record_digest(record: Dict[str, Any]) -> str:
|
| 41 | body = {key: value for key, value in record.items() if key != "record_sha256"}
|
| 42 | return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
| 43 |
|
| 44 |
|
| 45 | def seal(record: Dict[str, Any]) -> Dict[str, Any]:
|
| 46 | sealed = dict(record)
|
| 47 | sealed.update(schema_version=SCHEMA_VERSION, kind=KIND, policy_version=POLICY_VERSION)
|
| 48 | sealed["record_sha256"] = record_digest(sealed)
|
| 49 | return sealed
|
| 50 |
|
| 51 |
|
| 52 | def validate(record: Any, runner_id: Optional[str]) -> List[str]:
|
| 53 | """Reasons the record does NOT make rows claimable ([] = valid)."""
|
| 54 | if not isinstance(record, dict):
|
| 55 | return ["calibration record missing or not an object"]
|
| 56 | reasons = []
|
| 57 | if record.get("schema_version") != SCHEMA_VERSION or record.get("kind") != KIND:
|
| 58 | reasons.append("wrong schema_version/kind")
|
| 59 | missing = [key for key in REQUIRED if key not in record]
|
| 60 | if missing:
|
| 61 | reasons.append(f"missing fields {missing}")
|
| 62 | return reasons
|
| 63 | if record.get("record_sha256") != record_digest(record):
|
| 64 | reasons.append("record_sha256 does not match content (edited record)")
|
| 65 | if runner_id is None or record.get("runner_id") != runner_id:
|
| 66 | reasons.append("calibration was taken on a different runner")
|
| 67 | load = record.get("loadavg_per_core_max")
|
| 68 | if not isinstance(load, (int, float)) or load > MAX_LOAD_PER_CORE:
|
| 69 | reasons.append(f"host not quiet: load/core {load} > {MAX_LOAD_PER_CORE}")
|
| 70 | cells = record.get("aa_cells")
|
| 71 | if not isinstance(cells, int) or cells < MIN_AA_CELLS:
|
| 72 | reasons.append(f"too few A/A cells ({cells} < {MIN_AA_CELLS})")
|
| 73 | floor = record.get("aa_floor_median_pct")
|
| 74 | if not isinstance(floor, (int, float)) or floor > MAX_AA_FLOOR_MEDIAN_PCT:
|
| 75 | reasons.append(f"A/A floor median {floor}% > {MAX_AA_FLOOR_MEDIAN_PCT}%")
|
| 76 | if record.get("aa_false_claims") != 0:
|
| 77 | reasons.append("identical-binary A/A produced claims")
|
| 78 | return reasons
|
| 79 |
|
| 80 |
|
| 81 | def row_load_per_core(row: Dict[str, Any], cpu_count: Any) -> Optional[float]:
|
| 82 | sample = row.get("environment_sample") or {}
|
| 83 | load = sample.get("loadavg") if isinstance(sample, dict) else None
|
| 84 | if not isinstance(load, list) or not load or not isinstance(cpu_count, int) or cpu_count <= 0:
|
| 85 | return None
|
| 86 | return float(load[0]) / cpu_count
|
| 87 |
|
| 88 |
|
| 89 | def derive(row: Dict[str, Any], record: Optional[Dict[str, Any]]) -> Tuple[str, bool, List[str]]:
|
| 90 | """(evidence_class, claim_eligible, reasons) for one row."""
|
| 91 | reasons = validate(record, row.get("runner_id")) if record is not None else ["no calibration record"]
|
| 92 | if not reasons:
|
| 93 | per_core = row_load_per_core(row, record.get("cpu_count"))
|
| 94 | if per_core is None or per_core > MAX_LOAD_PER_CORE:
|
| 95 | reasons.append(f"row host load/core {per_core} not quiet")
|
| 96 | if row.get("returncode") != 0:
|
| 97 | reasons.append("row is not a successful measurement")
|
| 98 | if reasons:
|
| 99 | return DEVELOPMENT_EVIDENCE_CLASS, False, reasons
|
| 100 | return CALIBRATED_EVIDENCE_CLASS, True, []
|
| 101 |
|
| 102 |
|
| 103 | def stamp(row: Dict[str, Any], record: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
| 104 | evidence, eligible, reasons = derive(row, record)
|
| 105 | row["evidence_class"] = evidence
|
| 106 | row["claim_eligible"] = eligible
|
| 107 | row["claim_ineligible_reasons"] = reasons
|
| 108 | if eligible:
|
| 109 | row["calibration_record"] = record
|
| 110 | row["calibration_record_sha256"] = record["record_sha256"]
|
| 111 | return row
|
| 112 |
|
| 113 |
|
| 114 | def gate_row(row: Dict[str, Any]) -> List[str]:
|
| 115 | """publish_gate: why a scale row may NOT be published ([] = publishable)."""
|
| 116 | if row.get("evidence_class") != CALIBRATED_EVIDENCE_CLASS:
|
| 117 | return [f"evidence_class {row.get('evidence_class')!r} is not {CALIBRATED_EVIDENCE_CLASS!r}"]
|
| 118 | record = row.get("calibration_record")
|
| 119 | if row.get("calibration_record_sha256") != (record or {}).get("record_sha256"):
|
| 120 | return ["calibration_record_sha256 does not match the embedded record"]
|
| 121 | evidence, eligible, reasons = derive(row, record)
|
| 122 | if not eligible or row.get("claim_eligible") is not True:
|
| 123 | return reasons or ["claim_eligible is not derived true"]
|
| 124 | return []
|