| 1 | """Prevent aggregation of observably incompatible measurement populations.
|
| 2 |
|
| 3 | Legacy rows with uniformly absent metadata remain usable for diagnostics.
|
| 4 | Missing and present metadata are distinct: absence never establishes equality
|
| 5 | with a known environment. This check is not a public-provenance attestation.
|
| 6 | """
|
| 7 |
|
| 8 | from __future__ import annotations
|
| 9 |
|
| 10 | import json
|
| 11 | from typing import Any
|
| 12 |
|
| 13 |
|
| 14 | DIMENSIONS = (
|
| 15 | "lane", "benchmark_lane", "measurement_lane", "profile", "benchmark_track", "command_semantics_version",
|
| 16 | "fixture_version", "scenario_version", "oracle_version", "runner_class", "runner_id", "cache_state",
|
| 17 | "remote_transport", "remote_server", "netshape_profile", "env_isolation_version",
|
| 18 | "calibration_version", "platform_semantics_version", "binary_sha256", "load_tier",
|
| 19 | "instruction_level", "agent.name", "agent.model", "agent.adapter",
|
| 20 | "agent.provider", "agent.version", "agent.temperature", "agent.max_output_tokens",
|
| 21 | "agent.environment", "agent.instruction_level", "fixture.version", "fixture.fixture_version",
|
| 22 | "oracle.version", "task_prompt_id", "output_oracle_version", "patch_oracle_version",
|
| 23 | )
|
| 24 |
|
| 25 |
|
| 26 | def population_conflicts(rows: list[dict[str, Any]]) -> list[str]:
|
| 27 | """Return differing identity fields within ONE subject/scenario/op cell."""
|
| 28 | fields = {name: set() for name in DIMENSIONS}
|
| 29 | fields.update({"subject_details.version": set(), "subject_details.binary_sha256": set()})
|
| 30 | for row in rows:
|
| 31 | for name in DIMENSIONS:
|
| 32 | value: Any = row
|
| 33 | for part in name.split("."):
|
| 34 | value = value.get(part) if isinstance(value, dict) else None
|
| 35 | fields[name].add(json.dumps(value, sort_keys=True))
|
| 36 | details = row.get("subject_details") or {}
|
| 37 | if isinstance(details, dict):
|
| 38 | details = details.get(str(row.get("subject")), details)
|
| 39 | for name in ("version", "binary_sha256"):
|
| 40 | value = details.get(name) if isinstance(details, dict) else None
|
| 41 | fields[f"subject_details.{name}"].add(json.dumps(value, sort_keys=True))
|
| 42 | return sorted(name for name, values in fields.items() if len(values) > 1)
|
| 43 |
|
| 44 |
|
| 45 | # Unlike within-subject identity, comparator identity deliberately excludes
|
| 46 | # subject binary/version, names and subject-specific prompt IDs.
|
| 47 | COMPARISON_FIELDS = (
|
| 48 | "benchmark_track", "command_semantics_version", "fixture_version", "scenario_version",
|
| 49 | "oracle_version", "runner_class", "cache_state", "remote_transport", "remote_server",
|
| 50 | "netshape_profile", "env_isolation_version", "calibration_version",
|
| 51 | "platform_semantics_version", "agent.model", "agent.adapter", "agent.provider",
|
| 52 | "agent.version", "agent.temperature", "agent.max_output_tokens", "agent.environment",
|
| 53 | "agent.instruction_level", "output_oracle_version", "patch_oracle_version", "load_tier",
|
| 54 | )
|
| 55 | COMPARISON_IDENTITY_VERSION = 1
|
| 56 |
|
| 57 |
|
| 58 | def comparison_identity(row: dict[str, Any]) -> dict[str, Any]:
|
| 59 | identity = {"schema_version": COMPARISON_IDENTITY_VERSION}
|
| 60 | for name in COMPARISON_FIELDS:
|
| 61 | value: Any = row
|
| 62 | for part in name.split("."):
|
| 63 | value = value.get(part) if isinstance(value, dict) else None
|
| 64 | identity[name] = value
|
| 65 | fixture = row.get("fixture")
|
| 66 | if identity["fixture_version"] is None and isinstance(fixture, dict):
|
| 67 | identity["fixture_version"] = fixture.get("fixture_version", fixture.get("version"))
|
| 68 | if identity["agent.instruction_level"] is None:
|
| 69 | identity["agent.instruction_level"] = row.get("instruction_level")
|
| 70 | oracle = row.get("oracle")
|
| 71 | if identity["oracle_version"] is None and isinstance(oracle, dict):
|
| 72 | identity["oracle_version"] = oracle.get("version")
|
| 73 | return identity
|
| 74 |
|
| 75 |
|
| 76 | def comparison_conflicts(baseline: dict[str, Any], candidate: dict[str, Any]) -> list[str]:
|
| 77 | return sorted(name for name in ("schema_version", *COMPARISON_FIELDS)
|
| 78 | if baseline.get(name) != candidate.get(name))
|
| 79 |
|
| 80 |
|
| 81 | def missing_claim_identity(identity: dict[str, Any], lane: str, metric: str, operation: str) -> list[str]:
|
| 82 | required = {"benchmark_track", "command_semantics_version", "fixture_version", "env_isolation_version"}
|
| 83 | if metric.endswith("_ms") or metric == "elapsed_ms":
|
| 84 | required.update(("runner_class", "cache_state"))
|
| 85 | if lane in {"core", "workflow"}:
|
| 86 | required.add("load_tier")
|
| 87 | if lane == "agent":
|
| 88 | required.update(("scenario_version", "oracle_version", "agent.model", "agent.adapter",
|
| 89 | "agent.provider", "agent.version", "agent.environment", "agent.instruction_level"))
|
| 90 | if "remote" in operation or identity.get("remote_transport") is not None:
|
| 91 | required.update(("remote_transport", "remote_server"))
|
| 92 | if identity.get("remote_transport") == "network_shaped":
|
| 93 | required.add("netshape_profile")
|
| 94 | if metric == "tokens" or "token" in metric:
|
| 95 | required.add("calibration_version")
|
| 96 | unknown = {"", "unknown", "unmeasured", "unavailable", "unspecified", "not_measured"}
|
| 97 | missing = [name for name in required if identity.get(name) is None or
|
| 98 | (isinstance(identity.get(name), str) and identity[name].lower() in unknown)]
|
| 99 | if identity.get("schema_version") != COMPARISON_IDENTITY_VERSION:
|
| 100 | missing.append("schema_version")
|
| 101 | return sorted(missing)
|
| 102 |
|
| 103 |
|
| 104 | def independent_run_count(rows: list[dict[str, Any]]) -> int | None:
|
| 105 | """Distinct recorded trial IDs, not proof of independence across sessions."""
|
| 106 | identifiers = set()
|
| 107 | for row in rows:
|
| 108 | if (not isinstance(row.get("bench_id"), str) or not row["bench_id"] or
|
| 109 | type(row.get("run")) is not int or row["run"] < 0):
|
| 110 | return None
|
| 111 | identifiers.add(json.dumps([row["bench_id"], row["run"]], sort_keys=True))
|
| 112 | return len(identifiers)
|