| 1 | """The row contract: canonical accessors and per-lane validation.
|
| 2 |
|
| 3 | A Row is one JSONL object for one measured operation — the durable source of
|
| 4 | truth (CONTEXT.md). This module is the single definition of how consumers read
|
| 5 | the best-available token/tool/output values from any lane's rows, and of what
|
| 6 | every lane's emitter must include. ResultsStore validates at write time, so a
|
| 7 | contract violation fails the producing run instead of corrupting a dashboard.
|
| 8 |
|
| 9 | Null means unmeasured (ADR-0002): validation checks presence and types, and
|
| 10 | treats null as a legitimate value wherever a signal is probe-dependent.
|
| 11 | """
|
| 12 |
|
| 13 | from __future__ import annotations
|
| 14 |
|
| 15 | import math
|
| 16 | from typing import Any, Optional
|
| 17 |
|
| 18 | LANES = ("core", "workflow", "agent", "contention", "mount", "task-loop", "platform")
|
| 19 |
|
| 20 | # Required top-level keys per lane. Deliberately the intersection that every
|
| 21 | # row of the lane carries — including diagnostic rows (determinism probes,
|
| 22 | # skip rows), which is why token fields are not required at this layer.
|
| 23 | LANE_REQUIRED_KEYS: dict[str, tuple[str, ...]] = {
|
| 24 | "core": (
|
| 25 | "bench_id",
|
| 26 | "profile",
|
| 27 | "scenario",
|
| 28 | "operation",
|
| 29 | "run",
|
| 30 | "subject",
|
| 31 | "subject_kind",
|
| 32 | "elapsed_ms",
|
| 33 | "returncode",
|
| 34 | "command",
|
| 35 | ),
|
| 36 | "workflow": (
|
| 37 | "bench_id",
|
| 38 | "profile",
|
| 39 | "scenario",
|
| 40 | "workflow",
|
| 41 | "phase",
|
| 42 | "operation",
|
| 43 | "run",
|
| 44 | "subject",
|
| 45 | "subject_kind",
|
| 46 | "elapsed_ms",
|
| 47 | "returncode",
|
| 48 | "process_returncode",
|
| 49 | "command",
|
| 50 | ),
|
| 51 | "agent": (
|
| 52 | "schema_version",
|
| 53 | "bench_id",
|
| 54 | "scenario",
|
| 55 | "operation",
|
| 56 | "run",
|
| 57 | "subject",
|
| 58 | "subject_kind",
|
| 59 | "elapsed_ms",
|
| 60 | "returncode",
|
| 61 | "token_metrics",
|
| 62 | "turn_metrics",
|
| 63 | "tool_call_metrics",
|
| 64 | ),
|
| 65 | "contention": (
|
| 66 | "bench_id",
|
| 67 | "profile",
|
| 68 | "scenario",
|
| 69 | "contention_mode",
|
| 70 | "workers",
|
| 71 | "operation",
|
| 72 | "run",
|
| 73 | "subject",
|
| 74 | "subject_kind",
|
| 75 | "elapsed_ms",
|
| 76 | "returncode",
|
| 77 | ),
|
| 78 | "mount": (
|
| 79 | "bench_id",
|
| 80 | "profile",
|
| 81 | "scenario",
|
| 82 | "operation",
|
| 83 | "run",
|
| 84 | "subject",
|
| 85 | "elapsed_ms",
|
| 86 | "returncode",
|
| 87 | ),
|
| 88 | "task-loop": (
|
| 89 | "bench_id",
|
| 90 | "profile",
|
| 91 | "scenario",
|
| 92 | "operation",
|
| 93 | "run",
|
| 94 | "subject",
|
| 95 | "subject_kind",
|
| 96 | "elapsed_ms",
|
| 97 | "returncode",
|
| 98 | "command",
|
| 99 | ),
|
| 100 | # Platform lane: hosted-workflow rows are only comparable when both sides
|
| 101 | # ran the same semantics class under the same protection variant, so that
|
| 102 | # fairness context is part of every row — including skip rows. Runner
|
| 103 | # identity (ADR-0007) is required because this lane measures network
|
| 104 | # services from a specific vantage point.
|
| 105 | "platform": (
|
| 106 | "bench_id",
|
| 107 | "profile",
|
| 108 | "scenario",
|
| 109 | "operation",
|
| 110 | "run",
|
| 111 | "platform",
|
| 112 | "driver",
|
| 113 | "platform_semantics_version",
|
| 114 | "platform_semantics_class",
|
| 115 | "protection_variant",
|
| 116 | "platform_comparison_key",
|
| 117 | "semantic_contract_match",
|
| 118 | "elapsed_ms",
|
| 119 | "returncode",
|
| 120 | "command",
|
| 121 | "runner_id",
|
| 122 | "runner_class",
|
| 123 | ),
|
| 124 | }
|
| 125 |
|
| 126 | NUMERIC_KEYS = ("elapsed_ms",)
|
| 127 | INTEGER_KEYS = ("returncode", "run")
|
| 128 | SKIP_RETURNCODE = 77
|
| 129 |
|
| 130 | # Optional metrics with constrained types/ranges when present. None is always
|
| 131 | # legal (ADR-0002); a present value must be well-formed so consumers never
|
| 132 | # need defensive parsing of the durable format.
|
| 133 | OPTIONAL_NON_NEGATIVE_INT_KEYS = (
|
| 134 | "peak_rss_bytes",
|
| 135 | "tool_call_count",
|
| 136 | "probe_output_bytes",
|
| 137 | "changed_files_expected",
|
| 138 | "changed_files_recovered",
|
| 139 | "changed_files_reported",
|
| 140 | "changed_files_false_positive",
|
| 141 | "first_payload_bytes_server_to_client",
|
| 142 | "poll_iterations",
|
| 143 | "false_close_count",
|
| 144 | "close_eligible_n",
|
| 145 | "false_vcs_merge_safe_count",
|
| 146 | "vcs_merge_safe_n",
|
| 147 | "false_merge_allowed_count",
|
| 148 | "merge_allowed_n",
|
| 149 | "branches_scored",
|
| 150 | "branches_expected",
|
| 151 | )
|
| 152 | OPTIONAL_NON_NEGATIVE_NUMBER_KEYS = (
|
| 153 | "ttfd_ms",
|
| 154 | "observed_wait_ms",
|
| 155 | "settle_elapsed_ms",
|
| 156 | "poll_interval_s",
|
| 157 | "poll_quantization_ms",
|
| 158 | "status_post_elapsed_ms",
|
| 159 | "pacing_injected_ms",
|
| 160 | )
|
| 161 | OPTIONAL_UNIT_INTERVAL_KEYS = ("information_recall", "information_precision", "triage_recall", "triage_precision")
|
| 162 | OPTIONAL_STRING_KEYS = (
|
| 163 | "runner_id",
|
| 164 | "runner_class",
|
| 165 | "fixture_id",
|
| 166 | "fixture_generator",
|
| 167 | "fixture_version",
|
| 168 | "fixture_seed",
|
| 169 | "fixture_manifest_sha256",
|
| 170 | "binary_sha256",
|
| 171 | "env_isolation_version",
|
| 172 | "platform_semantics_version",
|
| 173 | "platform_semantics_class",
|
| 174 | "protection_variant",
|
| 175 | "platform_comparison_key",
|
| 176 | )
|
| 177 | OPTIONAL_OBJECT_KEYS = ("runner_profile",)
|
| 178 |
|
| 179 |
|
| 180 | class RowContractViolation(SystemExit):
|
| 181 | """Raised after rows are persisted; fails the producing run loudly."""
|
| 182 |
|
| 183 |
|
| 184 | def validate_rows(rows: list[dict[str, Any]], lane: str) -> list[str]:
|
| 185 | if lane not in LANE_REQUIRED_KEYS:
|
| 186 | return [f"unknown lane {lane!r}; known: {', '.join(LANES)}"]
|
| 187 | required = LANE_REQUIRED_KEYS[lane]
|
| 188 | errors: list[str] = []
|
| 189 | for index, row in enumerate(rows, 1):
|
| 190 | if not isinstance(row, dict):
|
| 191 | errors.append(f"row {index}: expected object, got {type(row).__name__}")
|
| 192 | continue
|
| 193 | for key in required:
|
| 194 | if key not in row:
|
| 195 | errors.append(f"row {index} ({row.get('operation', '?')}): missing required key {key!r}")
|
| 196 | for key in NUMERIC_KEYS:
|
| 197 | value = row.get(key)
|
| 198 | diagnostic_timing = (lane == "core" and key == "elapsed_ms" and value is None
|
| 199 | and row.get("measurement_phase") == "output_diagnostic"
|
| 200 | and row.get("operation") in {
|
| 201 | f"{operation}.{probe}" for operation in ("status.clean", "status.dirty", "diff.dirty", "diff.full")
|
| 202 | for probe in ("inforecall", "determinism")})
|
| 203 | if diagnostic_timing:
|
| 204 | continue
|
| 205 | if key in row and (
|
| 206 | isinstance(value, bool)
|
| 207 | or not isinstance(value, (int, float))
|
| 208 | or not math.isfinite(float(value))
|
| 209 | or float(value) < 0
|
| 210 | ):
|
| 211 | errors.append(f"row {index}: {key} must be numeric, got {type(value).__name__}")
|
| 212 | for key in INTEGER_KEYS:
|
| 213 | value = row.get(key)
|
| 214 | if key in row and (isinstance(value, bool) or not isinstance(value, int)):
|
| 215 | errors.append(f"row {index}: {key} must be an integer, got {type(value).__name__}")
|
| 216 | for key in OPTIONAL_NON_NEGATIVE_INT_KEYS:
|
| 217 | value = row.get(key)
|
| 218 | if key in row and value is not None:
|
| 219 | if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
| 220 | errors.append(f"row {index}: {key} must be a non-negative integer or null, got {value!r}")
|
| 221 | for key in OPTIONAL_NON_NEGATIVE_NUMBER_KEYS:
|
| 222 | value = row.get(key)
|
| 223 | if key in row and value is not None:
|
| 224 | if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
|
| 225 | errors.append(f"row {index}: {key} must be a non-negative finite number or null, got {value!r}")
|
| 226 | for key in OPTIONAL_UNIT_INTERVAL_KEYS:
|
| 227 | value = row.get(key)
|
| 228 | if key in row and value is not None:
|
| 229 | if not isinstance(value, (int, float)) or isinstance(value, bool) or not 0.0 <= float(value) <= 1.0:
|
| 230 | errors.append(f"row {index}: {key} must be in [0, 1] or null, got {value!r}")
|
| 231 | for key in OPTIONAL_STRING_KEYS:
|
| 232 | value = row.get(key)
|
| 233 | if key in row and value is not None and not isinstance(value, str):
|
| 234 | errors.append(f"row {index}: {key} must be a string or null, got {type(value).__name__}")
|
| 235 | for key in OPTIONAL_OBJECT_KEYS:
|
| 236 | value = row.get(key)
|
| 237 | if key in row and value is not None and not isinstance(value, dict):
|
| 238 | errors.append(f"row {index}: {key} must be an object or null, got {type(value).__name__}")
|
| 239 | # Skip-row coherence: a skip is a recorded coverage gap, and consumers
|
| 240 | # key off both signals. Half-marked skips silently become failures
|
| 241 | # (or worse, successes) downstream.
|
| 242 | returncode = row.get("returncode")
|
| 243 | if returncode == SKIP_RETURNCODE:
|
| 244 | if not row.get("skipped"):
|
| 245 | errors.append(f"row {index} ({row.get('operation', '?')}): returncode 77 requires skipped: true")
|
| 246 | if not str(row.get("skip_reason") or "").strip():
|
| 247 | errors.append(f"row {index} ({row.get('operation', '?')}): skip rows require a non-empty skip_reason")
|
| 248 | elif row.get("skipped"):
|
| 249 | errors.append(
|
| 250 | f"row {index} ({row.get('operation', '?')}): skipped: true requires returncode {SKIP_RETURNCODE}"
|
| 251 | )
|
| 252 | return errors
|
| 253 |
|
| 254 |
|
| 255 | def number_or_none(value: Any) -> Optional[float]:
|
| 256 | if value is None or isinstance(value, bool):
|
| 257 | return None
|
| 258 | try:
|
| 259 | parsed = float(value)
|
| 260 | except (TypeError, ValueError):
|
| 261 | return None
|
| 262 | if not math.isfinite(parsed):
|
| 263 | return None
|
| 264 | return parsed
|
| 265 |
|
| 266 |
|
| 267 | def row_returncode(row: dict[str, Any]) -> int:
|
| 268 | """Row returncode with missing/null treated as failure, never a crash."""
|
| 269 | value = row.get("returncode")
|
| 270 | if value is None:
|
| 271 | return 1
|
| 272 | try:
|
| 273 | return int(value)
|
| 274 | except (TypeError, ValueError):
|
| 275 | return 1
|
| 276 |
|
| 277 |
|
| 278 | def transcript_metrics_complete(row: dict[str, Any]) -> bool:
|
| 279 | source = row.get("source")
|
| 280 | execution = source.get("agent_execution") if isinstance(source, dict) else None
|
| 281 | is_agent = isinstance(row.get("agent"), dict) or str(row.get("operation", "")).startswith("agent.") or isinstance(execution, dict)
|
| 282 | return not ((is_agent and row.get("output_truncated") is True) or (isinstance(execution, dict) and
|
| 283 | (execution.get("usage_complete") is False or execution.get("transcript_metrics_complete") is False)))
|
| 284 |
|
| 285 |
|
| 286 | def token_total(row: dict[str, Any]) -> Optional[float]:
|
| 287 | """Best-available token total for a row, preferring provider-reported usage."""
|
| 288 | if not transcript_metrics_complete(row):
|
| 289 | return None
|
| 290 | token_metrics = row.get("token_metrics")
|
| 291 | if isinstance(token_metrics, dict):
|
| 292 | value = number_or_none(token_metrics.get("total_tokens_reported"))
|
| 293 | if value is not None:
|
| 294 | return value
|
| 295 | reported_parts = [
|
| 296 | number_or_none(token_metrics.get("input_tokens_reported")),
|
| 297 | number_or_none(token_metrics.get("output_tokens_reported")),
|
| 298 | number_or_none(token_metrics.get("reasoning_tokens_reported")),
|
| 299 | ]
|
| 300 | if any(part is not None for part in reported_parts):
|
| 301 | return sum(part or 0.0 for part in reported_parts)
|
| 302 |
|
| 303 | for key in ("estimated_tokens_total", "token_count", "total_tokens"):
|
| 304 | value = number_or_none(row.get(key))
|
| 305 | if value is not None:
|
| 306 | return value
|
| 307 |
|
| 308 | token_estimate = row.get("token_estimate")
|
| 309 | if isinstance(token_estimate, dict):
|
| 310 | value = number_or_none(token_estimate.get("total"))
|
| 311 | if value is not None:
|
| 312 | return value
|
| 313 |
|
| 314 | tokens = row.get("tokens")
|
| 315 | if isinstance(tokens, dict):
|
| 316 | for key in ("total", "total_tokens", "provider_total", "estimated_total"):
|
| 317 | value = number_or_none(tokens.get(key))
|
| 318 | if value is not None:
|
| 319 | return value
|
| 320 | provider = tokens.get("provider_reported")
|
| 321 | if isinstance(provider, dict):
|
| 322 | value = number_or_none(provider.get("total"))
|
| 323 | if value is not None:
|
| 324 | return value
|
| 325 |
|
| 326 | if isinstance(token_metrics, dict):
|
| 327 | chars = number_or_none(token_metrics.get("transcript_chars"))
|
| 328 | if chars is None:
|
| 329 | input_chars = number_or_none(token_metrics.get("normalized_input_chars"))
|
| 330 | output_chars = number_or_none(token_metrics.get("normalized_output_chars"))
|
| 331 | if input_chars is not None or output_chars is not None:
|
| 332 | chars = (input_chars or 0.0) + (output_chars or 0.0)
|
| 333 | if chars is not None:
|
| 334 | return max(1.0, math.ceil(chars / 4.0))
|
| 335 | return None
|
| 336 |
|
| 337 |
|
| 338 | def tool_calls(row: dict[str, Any]) -> Optional[float]:
|
| 339 | if not transcript_metrics_complete(row):
|
| 340 | return None
|
| 341 | for key in ("tool_call_count", "total_tool_calls"):
|
| 342 | value = number_or_none(row.get(key))
|
| 343 | if value is not None:
|
| 344 | return value
|
| 345 |
|
| 346 | nested = row.get("tool_calls")
|
| 347 | if isinstance(nested, dict):
|
| 348 | for key in ("total", "all"):
|
| 349 | value = number_or_none(nested.get(key))
|
| 350 | if value is not None:
|
| 351 | return value
|
| 352 |
|
| 353 | tool_call_metrics = row.get("tool_call_metrics")
|
| 354 | if isinstance(tool_call_metrics, dict):
|
| 355 | value = number_or_none(tool_call_metrics.get("tool_calls_total"))
|
| 356 | if value is not None:
|
| 357 | return value
|
| 358 | return None
|
| 359 |
|
| 360 |
|
| 361 | def output_bytes(row: dict[str, Any]) -> Optional[float]:
|
| 362 | value = number_or_none(row.get("raw_output_bytes"))
|
| 363 | if value is not None:
|
| 364 | return value
|
| 365 |
|
| 366 | stdout = number_or_none(row.get("stdout_bytes"))
|
| 367 | stderr = number_or_none(row.get("stderr_bytes"))
|
| 368 | if stdout is not None or stderr is not None:
|
| 369 | return (stdout or 0.0) + (stderr or 0.0)
|
| 370 |
|
| 371 | token_estimate = row.get("token_estimate")
|
| 372 | if isinstance(token_estimate, dict):
|
| 373 | stdout = number_or_none(token_estimate.get("stdout_bytes"))
|
| 374 | stderr = number_or_none(token_estimate.get("stderr_bytes"))
|
| 375 | if stdout is not None or stderr is not None:
|
| 376 | return (stdout or 0.0) + (stderr or 0.0)
|
| 377 | return None
|
| 378 |
|
| 379 |
|
| 380 | def output_truncated(row: dict[str, Any]) -> bool:
|
| 381 | if bool(row.get("output_truncated")):
|
| 382 | return True
|
| 383 | if bool(row.get("stdout_truncated")) or bool(row.get("stderr_truncated")):
|
| 384 | return True
|
| 385 | token_estimate = row.get("token_estimate")
|
| 386 | if isinstance(token_estimate, dict):
|
| 387 | return bool(token_estimate.get("stdout_truncated")) or bool(token_estimate.get("stderr_truncated"))
|
| 388 | return False
|