Log in
benchmarksPublic

Commit f08b4a24

· Browse files at this commit

f08b4a249972

Merged on mrmrs-d5ae98
mrmrs · 1 month ago · 4 files ~4 modified
M scripts/oakbench/scale_calibration.py
@@ -16,10 +16,12 @@
1616 registry-pinned fixtures. The published run must carry its own quiet disk
1717 probes too.
1818
19-This detects edited, resealed, copied or invented records (the sealed digest
20-is only an integrity check of the record's own content). It cannot detect a
21-fabricated calibration JSONL with invented timings; that would need a signed
22-runner, which the lane does not have. Do not describe it as forgery-proof.
19+This detects edited, resealed, copied or invented RECORDS (the sealed digest
20+is only an integrity check of the record's own content). It does not
21+authenticate the campaign JSONL: editing its load samples and probe values
22+(a few dozen numbers) and resealing a record to the new file digest passes, as
23+would a fabricated campaign. Closing that needs a signed runner, which the
24+lane does not have. Do not describe it as forgery-proof.
2325 """
2426
2527 from __future__ import annotations
@@ -36,18 +38,32 @@
3638 KIND = "oakbench_quiet_host_calibration"
3739 CALIBRATED_EVIDENCE_CLASS = "calibrated_quiet_host"
3840 DEVELOPMENT_EVIDENCE_CLASS = "development_only_loaded_host"
39-POLICY_VERSION = "2026-09-29.2"
41+POLICY_VERSION = "2026-09-29.3"
4042 # Quiet-host policy (provisional; re-derive on the dedicated runner and bump
4143 # POLICY_VERSION). Load/core 0.1 is ~1.4 runnable threads on a 14-core host.
4244 MAX_LOAD_PER_CORE = 0.10
4345 MIN_AA_CELLS = 20
44-MIN_RUNS = 5
46+# A calibration needs two independent halves of >= 5 runs: floors come from
47+# one, A/A claims are counted on the other (a floor computed from the same
48+# A/A pair it judges can never produce a claim).
49+MIN_RUNS = 10
4550 MAX_AA_FLOOR_MEDIAN_PCT = 10.0
51+# Per arm: the median A/A floor AND the share of cells whose floor exceeds
52+# MAX_AA_FLOOR_MEDIAN_PCT must be small (a broken A/A arm is systematic).
53+MAX_AA_CELLS_OVER_FLOOR_SHARE = 0.10
4654 # Disk contention (load average does not see it; Oak clone/switch are
4755 # fsync-bound, fb-549). Measured on the loaded dev host: F_FULLFSYNC median
4856 # 3.7 ms, file creation as low as 170 files/s.
49-MAX_FULLFSYNC_MEDIAN_MS = 2.5
57+# Per flush primitive; a probe must use the platform's full-flush primitive
58+# (F_FULLFSYNC on macOS: plain fsync there skips the drive-cache flush).
59+# PROVISIONAL: no idle baseline was obtainable on the shared dev host (internal
60+# SSD, loaded: F_FULLFSYNC 3.7-3.9 ms); 2.5 ms may be unreachable even idle.
61+# The dedicated runner must re-derive these (P6) and bump POLICY_VERSION.
62+MAX_FLUSH_MEDIAN_MS = {"F_FULLFSYNC": 2.5, "fsync": 2.5}
63+MAX_FULLFSYNC_MEDIAN_MS = MAX_FLUSH_MEDIAN_MS["F_FULLFSYNC"]
5064 MIN_CREATE_FILES_PER_S = 2000.0
65+# Work volumes whose flushes do not reach durable media cannot certify runs.
66+ACCEPTED_MEDIA = ("physical",)
5167 PROBE_OPERATIONS = ("host.disk_fullfsync", "host.file_create_rate")
5268 AA_PAIRS = (("git", "git_aa"), ("oak_installed", "oak_installed_aa"))
5369
@@ -65,6 +81,101 @@
6581 return {"host_cpu_count": os.cpu_count(), "host_mem_bytes": mem}
6682
6783
84+def expected_flush_kind(system: Optional[str] = None) -> str:
85+ import platform
86+ return "F_FULLFSYNC" if (system or platform.system()) == "Darwin" else "fsync"
87+
88+
89+def _diskutil(target: str) -> Dict[str, Any]:
90+ import plistlib
91+ import subprocess
92+ try:
93+ out = subprocess.run(["/usr/sbin/diskutil", "info", "-plist", target], capture_output=True, timeout=20)
94+ except (OSError, subprocess.TimeoutExpired):
95+ return {}
96+ if out.returncode != 0:
97+ return {}
98+ try:
99+ return plistlib.loads(out.stdout)
100+ except Exception: # noqa: BLE001 - malformed output means unknown medium
101+ return {}
102+
103+
104+def classify_medium(volume: Dict[str, Any], store: Dict[str, Any]) -> str:
105+ """physical | ram_or_disk_image | tmpfs | unknown (only physical certifies)."""
106+ fstype = str(volume.get("fstype") or "").lower()
107+ if fstype in ("tmpfs", "ramfs", "devfs", "overlay", "squashfs"):
108+ return "tmpfs"
109+ info = store or volume.get("_diskutil") or {}
110+ if not info:
111+ return "unknown"
112+ if info.get("BusProtocol") == "Disk Image" or info.get("VirtualOrPhysical") == "Virtual" \
113+ or "Disk Image" in str(info.get("MediaName") or ""):
114+ return "ram_or_disk_image"
115+ return "physical" if info.get("BusProtocol") else "unknown"
116+
117+
118+def volume_identity(path: Path) -> Dict[str, Any]:
119+ """Filesystem type, mount point and backing device/medium of ``path``'s volume."""
120+ import platform
121+ import subprocess
122+ path = Path(path)
123+ probe = path if path.exists() else path.parent
124+ identity: Dict[str, Any] = {"mount_point": None, "device": None, "fstype": None, "volume_uuid": None,
125+ "bus_protocol": None, "physical_store": None, "medium": "unknown"}
126+ try:
127+ df = subprocess.run(["df", "-P", str(probe)], capture_output=True, text=True, timeout=20).stdout
128+ fields = df.strip().splitlines()[-1].split()
129+ identity["device"], identity["mount_point"] = fields[0], fields[-1]
130+ except (OSError, IndexError, subprocess.TimeoutExpired):
131+ return identity
132+ if platform.system() == "Darwin":
133+ info = _diskutil(identity["mount_point"])
134+ identity["fstype"] = info.get("FilesystemType")
135+ identity["volume_uuid"] = info.get("VolumeUUID")
136+ store_info: Dict[str, Any] = {}
137+ stores = info.get("APFSPhysicalStores") or []
138+ if stores:
139+ store = stores[0].get("APFSPhysicalStore")
140+ identity["physical_store"] = store
141+ store_info = _diskutil(store) if store else {}
142+ else:
143+ store_info = info
144+ identity["bus_protocol"] = store_info.get("BusProtocol")
145+ identity["medium"] = classify_medium(identity, store_info)
146+ else:
147+ try:
148+ for line in Path("/proc/mounts").read_text().splitlines():
149+ parts = line.split()
150+ if len(parts) >= 3 and parts[1] == identity["mount_point"]:
151+ identity["fstype"] = parts[2]
152+ except OSError:
153+ pass
154+ fstype = str(identity["fstype"] or "").lower()
155+ if fstype in ("tmpfs", "ramfs", "overlay", "squashfs"):
156+ identity["medium"] = "tmpfs"
157+ elif str(identity["device"]).startswith("/dev/") and not str(identity["device"]).startswith("/dev/ram") \
158+ and not str(identity["device"]).startswith("/dev/zram"):
159+ identity["medium"] = "physical"
160+ identity["volume_uuid"] = identity["device"]
161+ return identity
162+
163+
164+def volume_problems(volume: Any) -> List[str]:
165+ if not isinstance(volume, dict):
166+ return ["work volume unrecorded"]
167+ if volume.get("medium") not in ACCEPTED_MEDIA:
168+ return [f"work volume medium {volume.get('medium')!r} ({volume.get('fstype')}, {volume.get('device')}) "
169+ "cannot certify durable-flush costs"]
170+ return []
171+
172+
173+def volume_key(volume: Any) -> Optional[str]:
174+ if not isinstance(volume, dict):
175+ return None
176+ return str(volume.get("volume_uuid") or f"{volume.get('device')}@{volume.get('mount_point')}")
177+
178+
68179 def disk_probe(directory: Path, samples: int = 20, files: int = 500) -> Dict[str, Any]:
69180 """F_FULLFSYNC (fsync elsewhere) latency of a 4 KiB write, and small-file creation rate."""
70181 import fcntl
@@ -98,7 +209,7 @@
98209 tree.rmdir()
99210 return {"fullfsync_ms": [round(v, 4) for v in latencies],
100211 "fullfsync_kind": "F_FULLFSYNC" if full is not None else "fsync",
101- "create_files": files, "create_elapsed_s": round(elapsed, 6)}
212+ "create_files": files, "create_elapsed_s": round(elapsed, 6), "work_volume": volume_identity(directory)}
102213
103214
104215 def probe_rows(probe: Dict[str, Any], base: Dict[str, Any], phase: str) -> List[Dict[str, Any]]:
@@ -106,7 +217,8 @@
106217 common = {**base, "subject": "harness", "subject_kind": "harness", "command": [], "returncode": 0,
107218 "diagnostic": True, "probe_phase": phase, "measurement_source": "harness_phase_wall_clock",
108219 "evidence_class": DEVELOPMENT_EVIDENCE_CLASS, "claim_eligible": False,
109- "claim_ineligible_reasons": ["host probe (diagnostic)"], **host_facts()}
220+ "claim_ineligible_reasons": ["host probe (diagnostic)"], **host_facts(),
221+ "work_volume": probe.get("work_volume")}
110222 rows = [dict(common, operation="host.disk_fullfsync", run=index, elapsed_ms=value,
111223 fsync_kind=probe["fullfsync_kind"]) for index, value in enumerate(probe["fullfsync_ms"])]
112224 rate = probe["create_files"] / probe["create_elapsed_s"] if probe["create_elapsed_s"] > 0 else 0.0
@@ -118,17 +230,34 @@
118230
119231 def probe_verdict(rows: Iterable[Dict[str, Any]]) -> Tuple[List[str], Dict[str, Any]]:
120232 rows = list(rows)
121- fsyncs = [r["elapsed_ms"] for r in rows if r.get("operation") == "host.disk_fullfsync"
122- and isinstance(r.get("elapsed_ms"), (int, float))]
233+ flush_rows = [r for r in rows if r.get("operation") == "host.disk_fullfsync"
234+ and isinstance(r.get("elapsed_ms"), (int, float))]
235+ fsyncs = [r["elapsed_ms"] for r in flush_rows]
123236 rates = [r.get("files_per_s") for r in rows if r.get("operation") == "host.file_create_rate"]
124237 reasons, facts = [], {}
125238 if len(fsyncs) < 10 or not rates:
126239 return ["disk probes missing"], facts
240+ kinds = {r.get("fsync_kind") for r in flush_rows}
241+ systems = {(r.get("runner_profile") or {}).get("system") for r in flush_rows}
242+ expected = {expected_flush_kind(system) for system in systems if system} or {None}
243+ if len(kinds) != 1 or kinds != expected or next(iter(kinds)) not in MAX_FLUSH_MEDIAN_MS:
244+ reasons.append(f"flush probe kind {sorted(map(str, kinds))} is not the platform's full-flush "
245+ f"primitive {sorted(map(str, expected))}")
246+ return reasons, facts
247+ kind = next(iter(kinds))
248+ for row in rows:
249+ if row.get("operation", "").startswith("host."):
250+ reasons += volume_problems(row.get("work_volume"))
251+ volumes = {volume_key(r.get("work_volume")) for r in rows if r.get("operation", "").startswith("host.")}
252+ if len(volumes) != 1:
253+ reasons.append(f"probes ran on several volumes {sorted(map(str, volumes))}")
254+ reasons = list(dict.fromkeys(reasons))
255+ facts["work_volume_key"] = next(iter(volumes)) if len(volumes) == 1 else None
127256 facts["fullfsync_median_ms"] = statistics.median(fsyncs)
128257 facts["create_files_per_s_min"] = min(r for r in rates if isinstance(r, (int, float)))
129- if facts["fullfsync_median_ms"] > MAX_FULLFSYNC_MEDIAN_MS:
130- reasons.append(f"disk contended: F_FULLFSYNC median {facts['fullfsync_median_ms']:.2f} ms > "
131- f"{MAX_FULLFSYNC_MEDIAN_MS} ms")
258+ limit = MAX_FLUSH_MEDIAN_MS[kind]
259+ if facts["fullfsync_median_ms"] > limit:
260+ reasons.append(f"disk contended: {kind} median {facts['fullfsync_median_ms']:.2f} ms > {limit} ms")
132261 if facts["create_files_per_s_min"] < MIN_CREATE_FILES_PER_S:
133262 reasons.append(f"disk contended: {facts['create_files_per_s_min']:.0f} files/s < {MIN_CREATE_FILES_PER_S:.0f}")
134263 return reasons, facts
@@ -182,6 +311,11 @@
182311 """Runner/host/binary identity shared by every row of a run (or problems)."""
183312 reasons: List[str] = []
184313 identity: Dict[str, Any] = {}
314+ volumes = {volume_key(row.get("work_volume")) for row in rows}
315+ if len(volumes) != 1 or None in volumes:
316+ reasons.append(f"rows disagree on or lack work_volume: {sorted(map(str, volumes))[:3]}")
317+ else:
318+ identity["work_volume_key"] = volumes.pop()
185319 for key in ("runner_id", "host_cpu_count", "host_mem_bytes"):
186320 values = {row.get(key) for row in rows}
187321 if len(values) != 1 or None in values:
@@ -208,21 +342,36 @@
208342 value = lambda r: r.get("elapsed_ms") # noqa: E731
209343 floors: List[float] = []
210344 false_claims = 0
345+ per_arm: Dict[str, Dict[str, Any]] = {}
211346 for arm, control in AA_PAIRS:
212- runs = {r.get("run") for r in measured if r.get("subject") == arm}
347+ runs = sorted({r.get("run") for r in measured if r.get("subject") == arm})
213348 if len(runs) < MIN_RUNS:
214- reasons.append(f"{arm}: fewer than {MIN_RUNS} runs")
349+ reasons.append(f"{arm}: fewer than {MIN_RUNS} runs (two independent halves of >= 5)")
215350 continue
216- cell = cellstats.cell_floors(measured, arm, control, value)
351+ pair = [r for r in measured if r.get("subject") in (arm, control)]
352+ # Per-arm floors over all runs: the A/A noise this arm pair shows.
353+ cell = cellstats.cell_floors(pair, arm, control, value)
217354 floors.extend(cell.values())
218- verdicts = cellstats.classify_cells(measured, control, arm, cell, value_of=value)
219- false_claims += sum(1 for item in verdicts.values() if item.get("verdict") in ("regression", "improvement"))
355+ arm_floors = sorted(cell.values())
356+ over = sum(1 for f in arm_floors if f > MAX_AA_FLOOR_MEDIAN_PCT)
357+ per_arm[arm] = {"cells": len(arm_floors),
358+ "floor_median_pct": round(statistics.median(arm_floors), 3) if arm_floors else None,
359+ "cells_over_threshold": over}
360+ # Held-out A/A claims: floors from the first half, verdicts on the second.
361+ first = set(runs[: len(runs) // 2])
362+ screen = [r for r in pair if r.get("run") in first]
363+ held = [r for r in pair if r.get("run") not in first]
364+ held_floors = cellstats.cell_floors(screen, arm, control, value)
365+ verdicts = cellstats.classify_cells(held, control, arm, held_floors, value_of=value)
366+ claims = sum(1 for item in verdicts.values() if item.get("verdict") in ("regression", "improvement"))
367+ per_arm[arm]["held_out_claims"] = claims
368+ false_claims += claims
220369 cpus = identity.get("host_cpu_count")
221370 loads = [r["environment_sample"]["loadavg"][0] for r in rows
222371 if isinstance((r.get("environment_sample") or {}).get("loadavg"), list)]
223372 load_per_core = max(loads) / cpus if loads and isinstance(cpus, int) and cpus > 0 else None
224373 disk_reasons, disk = probe_verdict(rows)
225- facts = {**identity, "aa_cells": len(floors),
374+ facts = {**identity, "aa_cells": len(floors), "aa_per_arm": per_arm,
226375 "aa_floor_median_pct": round(statistics.median(floors), 3) if floors else None,
227376 "aa_false_claims": false_claims,
228377 "loadavg_per_core_max": round(load_per_core, 4) if load_per_core is not None else None, **disk}
@@ -231,7 +380,13 @@
231380 if facts["aa_floor_median_pct"] is None or facts["aa_floor_median_pct"] > MAX_AA_FLOOR_MEDIAN_PCT:
232381 reasons.append(f"A/A floor median {facts['aa_floor_median_pct']}% > {MAX_AA_FLOOR_MEDIAN_PCT}%")
233382 if false_claims:
234- reasons.append(f"identical-binary A/A produced {false_claims} claim(s)")
383+ reasons.append(f"identical-binary A/A produced {false_claims} held-out claim(s)")
384+ for arm, info in per_arm.items():
385+ if info["floor_median_pct"] is None or info["floor_median_pct"] > MAX_AA_FLOOR_MEDIAN_PCT:
386+ reasons.append(f"{arm} A/A floor median {info['floor_median_pct']}% > {MAX_AA_FLOOR_MEDIAN_PCT}%")
387+ if info["cells"] and info["cells_over_threshold"] / info["cells"] > MAX_AA_CELLS_OVER_FLOOR_SHARE:
388+ reasons.append(f"{arm}: {info['cells_over_threshold']}/{info['cells']} cells have an A/A floor over "
389+ f"{MAX_AA_FLOOR_MEDIAN_PCT}% (broken or noisy A/A arm)")
235390 if load_per_core is None or load_per_core > MAX_LOAD_PER_CORE:
236391 reasons.append(f"host not quiet: load/core {load_per_core} > {MAX_LOAD_PER_CORE}")
237392 reasons += disk_reasons
@@ -257,7 +412,7 @@
257412 facts, problems = recompute(calibration_rows, registry)
258413 reasons += problems
259414 for key in ("runner_id", "host_cpu_count", "host_mem_bytes", "oak_binary_sha256", "git_binary_sha256",
260- "aa_cells", "aa_false_claims"):
415+ "work_volume_key", "aa_cells", "aa_false_claims"):
261416 if record.get(key) != facts.get(key):
262417 reasons.append(f"record {key} {record.get(key)!r} != recomputed {facts.get(key)!r}")
263418 return facts, reasons
@@ -269,6 +424,9 @@
269424 for key in ("runner_id", "host_cpu_count", "host_mem_bytes"):
270425 if row.get(key) != facts.get(key):
271426 reasons.append(f"row {key} {row.get(key)!r} != calibration {facts.get(key)!r}")
427+ if volume_key(row.get("work_volume")) != facts.get("work_volume_key"):
428+ reasons.append("row ran on a different work volume than the calibration")
429+ reasons += volume_problems(row.get("work_volume"))
272430 kind = row.get("subject_kind")
273431 if kind in ("git", "oak") and row.get("binary_sha256") != facts.get(f"{kind}_binary_sha256"):
274432 reasons.append(f"row {kind} binary differs from the calibrated binary")