Viewing files at commit 5717cc9fc7edCommit detailsView latest

scripts/oakbench/populations.py 112 lines · 5.7 KB · python
1
"""Prevent aggregation of observably incompatible measurement populations.
2
3
Legacy rows with uniformly absent metadata remain usable for diagnostics.
4
Missing and present metadata are distinct: absence never establishes equality
5
with a known environment. This check is not a public-provenance attestation.
6
"""
7
8
from __future__ import annotations
9
10
import json
11
from typing import Any
12
13
14
DIMENSIONS = (
15
    "lane", "benchmark_lane", "measurement_lane", "profile", "benchmark_track", "command_semantics_version",
16
    "fixture_version", "scenario_version", "oracle_version", "runner_class", "runner_id", "cache_state",
17
    "remote_transport", "remote_server", "netshape_profile", "env_isolation_version",
18
    "calibration_version", "platform_semantics_version", "binary_sha256", "load_tier",
19
    "instruction_level", "agent.name", "agent.model", "agent.adapter",
20
    "agent.provider", "agent.version", "agent.temperature", "agent.max_output_tokens",
21
    "agent.environment", "agent.instruction_level", "fixture.version", "fixture.fixture_version",
22
    "oracle.version", "task_prompt_id", "output_oracle_version", "patch_oracle_version",
23
)
24
25
26
def population_conflicts(rows: list[dict[str, Any]]) -> list[str]:
27
    """Return differing identity fields within ONE subject/scenario/op cell."""
28
    fields = {name: set() for name in DIMENSIONS}
29
    fields.update({"subject_details.version": set(), "subject_details.binary_sha256": set()})
30
    for row in rows:
31
        for name in DIMENSIONS:
32
            value: Any = row
33
            for part in name.split("."):
34
                value = value.get(part) if isinstance(value, dict) else None
35
            fields[name].add(json.dumps(value, sort_keys=True))
36
        details = row.get("subject_details") or {}
37
        if isinstance(details, dict):
38
            details = details.get(str(row.get("subject")), details)
39
        for name in ("version", "binary_sha256"):
40
            value = details.get(name) if isinstance(details, dict) else None
41
            fields[f"subject_details.{name}"].add(json.dumps(value, sort_keys=True))
42
    return sorted(name for name, values in fields.items() if len(values) > 1)
43
44
45
# Unlike within-subject identity, comparator identity deliberately excludes
46
# subject binary/version, names and subject-specific prompt IDs.
47
COMPARISON_FIELDS = (
48
    "benchmark_track", "command_semantics_version", "fixture_version", "scenario_version",
49
    "oracle_version", "workflow_integrity_evidence_class", "workflow_integrity_committed_scope", "runner_class", "cache_state", "remote_transport", "remote_server",
50
    "netshape_profile", "env_isolation_version", "calibration_version",
51
    "platform_semantics_version", "agent.model", "agent.adapter", "agent.provider",
52
    "agent.version", "agent.temperature", "agent.max_output_tokens", "agent.environment",
53
    "agent.instruction_level", "output_oracle_version", "patch_oracle_version", "load_tier",
54
)
55
COMPARISON_IDENTITY_VERSION = 1
56
57
58
def comparison_identity(row: dict[str, Any]) -> dict[str, Any]:
59
    identity = {"schema_version": COMPARISON_IDENTITY_VERSION}
60
    for name in COMPARISON_FIELDS:
61
        value: Any = row
62
        for part in name.split("."):
63
            value = value.get(part) if isinstance(value, dict) else None
64
        identity[name] = value
65
    fixture = row.get("fixture")
66
    if identity["fixture_version"] is None and isinstance(fixture, dict):
67
        identity["fixture_version"] = fixture.get("fixture_version", fixture.get("version"))
68
    if identity["agent.instruction_level"] is None:
69
        identity["agent.instruction_level"] = row.get("instruction_level")
70
    oracle = row.get("oracle")
71
    if identity["oracle_version"] is None and isinstance(oracle, dict):
72
        identity["oracle_version"] = oracle.get("version")
73
    return identity
74
75
76
def comparison_conflicts(baseline: dict[str, Any], candidate: dict[str, Any]) -> list[str]:
77
    return sorted(name for name in ("schema_version", *COMPARISON_FIELDS)
78
                  if baseline.get(name) != candidate.get(name))
79
80
81
def missing_claim_identity(identity: dict[str, Any], lane: str, metric: str, operation: str) -> list[str]:
82
    required = {"benchmark_track", "command_semantics_version", "fixture_version", "env_isolation_version"}
83
    if metric.endswith("_ms") or metric == "elapsed_ms":
84
        required.update(("runner_class", "cache_state"))
85
        if lane in {"core", "workflow"}:
86
            required.add("load_tier")
87
    if lane == "agent":
88
        required.update(("scenario_version", "oracle_version", "agent.model", "agent.adapter",
89
                         "agent.provider", "agent.version", "agent.environment", "agent.instruction_level"))
90
    if "remote" in operation or identity.get("remote_transport") is not None:
91
        required.update(("remote_transport", "remote_server"))
92
    if identity.get("remote_transport") == "network_shaped":
93
        required.add("netshape_profile")
94
    if metric == "tokens" or "token" in metric:
95
        required.add("calibration_version")
96
    unknown = {"", "unknown", "unmeasured", "unavailable", "unspecified", "not_measured"}
97
    missing = [name for name in required if identity.get(name) is None or
98
               (isinstance(identity.get(name), str) and identity[name].lower() in unknown)]
99
    if identity.get("schema_version") != COMPARISON_IDENTITY_VERSION:
100
        missing.append("schema_version")
101
    return sorted(missing)
102
103
104
def independent_run_count(rows: list[dict[str, Any]]) -> int | None:
105
    """Distinct recorded trial IDs, not proof of independence across sessions."""
106
    identifiers = set()
107
    for row in rows:
108
        if (not isinstance(row.get("bench_id"), str) or not row["bench_id"] or
109
                type(row.get("run")) is not int or row["run"] < 0):
110
            return None
111
        identifiers.add(json.dumps([row["bench_id"], row["run"]], sort_keys=True))
112
    return len(identifiers)