| 1 | """Diagnostic campaign accounting from a frozen schedule and durable journal.
|
| 2 |
|
| 3 | No publication claim is certified here. Block bootstrap describes recorded
|
| 4 | paired trials; repeated fixed fixtures are not independent task populations.
|
| 5 | """
|
| 6 | from __future__ import annotations
|
| 7 |
|
| 8 | import itertools
|
| 9 | import math
|
| 10 | import statistics
|
| 11 | from statistics import NormalDist
|
| 12 |
|
| 13 | from oakbench.stats import bootstrap_ci, validate_interval_settings
|
| 14 | from oakbench.populations import population_conflicts, comparison_identity, comparison_conflicts
|
| 15 |
|
| 16 | CONDITION_FIELDS = ("workflow", "instruction_level", "agent", "model")
|
| 17 | OUTCOMES = ("success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped", "unstarted")
|
| 18 | COMPLETE = {"success", "failure", "timeout", "infrastructure_failure"}
|
| 19 | METRICS = {
|
| 20 | "agent_active_ms": ("wall_clock_metrics", "agent_active_ms"),
|
| 21 | "task_total_ms": ("wall_clock_metrics", "total_ms"),
|
| 22 | "reported_tokens": ("token_metrics", "total_tokens_reported"),
|
| 23 | "cost_weighted_token_proxy": ("token_metrics", "cost_weighted_total"),
|
| 24 | "assistant_turns": ("turn_metrics", "assistant_turns_total"),
|
| 25 | "tool_calls": ("tool_call_metrics", "tool_calls_total"),
|
| 26 | }
|
| 27 |
|
| 28 |
|
| 29 | def _number(value):
|
| 30 | return float(value) if type(value) in (int, float) and math.isfinite(value) and value >= 0 else None
|
| 31 |
|
| 32 |
|
| 33 | def _metric(row, path):
|
| 34 | if isinstance(row, dict) and path[0] in {"token_metrics", "turn_metrics", "tool_call_metrics"}:
|
| 35 | execution = (row.get("source") or {}).get("agent_execution", {})
|
| 36 | incomplete = (isinstance(execution, dict) and
|
| 37 | (execution.get("capture_complete") is False or execution.get("incomplete") is True or
|
| 38 | execution.get("usage_complete") is False or execution.get("transcript_metrics_complete") is False or
|
| 39 | execution.get("counters_coverage") == "observed_lower_bounds" or execution.get("output_limited") is True or
|
| 40 | execution.get("timed_out") is True or execution.get("output_truncated") is True))
|
| 41 | if row.get("outcome") == "timeout" or row.get("output_truncated") is True or incomplete:
|
| 42 | return None
|
| 43 | value = row
|
| 44 | for part in path:
|
| 45 | value = value.get(part) if isinstance(value, dict) else None
|
| 46 | return _number(value)
|
| 47 |
|
| 48 |
|
| 49 | def _wilson(successes, n, confidence):
|
| 50 | if n == 0:
|
| 51 | return None
|
| 52 | z = NormalDist().inv_cdf((1 + confidence) / 2)
|
| 53 | p = successes / n
|
| 54 | scale = 1 + z * z / n
|
| 55 | centre = (p + z * z / (2 * n)) / scale
|
| 56 | radius = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / scale
|
| 57 | return [max(0.0, centre - radius), min(1.0, centre + radius)]
|
| 58 |
|
| 59 |
|
| 60 | def _outcome(event):
|
| 61 | row = event.get("row")
|
| 62 | explicit = event.get("outcome")
|
| 63 | if explicit is not None and isinstance(row, dict) and explicit != row.get("outcome"):
|
| 64 | raise ValueError("event outcome contradicts terminal row")
|
| 65 | if event.get("measurement_invalidated") is True:
|
| 66 | if explicit != "error" or row is not None or not event.get("invalidation_reason"):
|
| 67 | raise ValueError("invalidated measurement has contradictory terminal evidence")
|
| 68 | return "skipped", None
|
| 69 | if isinstance(row, dict) and row.get("returncode") == 77:
|
| 70 | if row.get("outcome") != "error" or row.get("success") is not False or not row.get("skip_reason"):
|
| 71 | raise ValueError("unmeasured instrument row has contradictory outcome")
|
| 72 | return "skipped", row
|
| 73 | if explicit in {"skipped", "skip"}:
|
| 74 | return "skipped", row
|
| 75 | if explicit == "interrupted":
|
| 76 | return "interrupted", row
|
| 77 | if explicit in {"error", "infrastructure_failure"}:
|
| 78 | if row is not None and (not isinstance(row, dict) or row.get("outcome") != "error" or row.get("success") is not False):
|
| 79 | raise ValueError("infrastructure error contradicts terminal task row")
|
| 80 | return "infrastructure_failure", row
|
| 81 | if not isinstance(row, dict) or type(row.get("success")) is not bool:
|
| 82 | raise ValueError("finished trial lacks an explicit row success outcome")
|
| 83 | outcome = row.get("outcome")
|
| 84 | if outcome not in {"pass", "fail", "timeout", "error"} or (row["success"] != (outcome == "pass")):
|
| 85 | raise ValueError("contradictory or unknown row outcome")
|
| 86 | return {"pass": "success", "fail": "failure", "timeout": "timeout", "error": "infrastructure_failure"}[outcome], row
|
| 87 |
|
| 88 |
|
| 89 | def _trials(plan, events):
|
| 90 | if plan.get("schema_version") != 1 or not isinstance(plan.get("schedule"), list):
|
| 91 | raise ValueError("unsupported campaign plan")
|
| 92 | if not isinstance(plan.get("plan_sha256"), str) or not plan["plan_sha256"]:
|
| 93 | raise ValueError("campaign plan digest missing")
|
| 94 | trials = {}
|
| 95 | cells = set()
|
| 96 | for item in plan["schedule"]:
|
| 97 | if not all(key in item for key in ("trial_id", "block_id", "subject", *CONDITION_FIELDS)):
|
| 98 | raise ValueError("schedule trial missing identity")
|
| 99 | trial_id = item["trial_id"]
|
| 100 | cell = tuple(item[key] for key in (*CONDITION_FIELDS, "block_id", "subject"))
|
| 101 | if trial_id in trials or cell in cells:
|
| 102 | raise ValueError("duplicate scheduled trial or subject in paired block")
|
| 103 | cells.add(cell)
|
| 104 | trials[trial_id] = {"trial": item, "started": False, "finished": False,
|
| 105 | "outcome": "unstarted", "row": None}
|
| 106 | for event in events:
|
| 107 | if event.get("plan_sha256") != plan["plan_sha256"]:
|
| 108 | raise ValueError("journal plan digest mismatch")
|
| 109 | if event.get("event") == "journal_recovery":
|
| 110 | continue
|
| 111 | if event.get("trial_id") not in trials:
|
| 112 | raise ValueError("journal names unscheduled trial")
|
| 113 | state = trials[event["trial_id"]]
|
| 114 | if event.get("event") == "started":
|
| 115 | if state["started"] or state["finished"]:
|
| 116 | raise ValueError("duplicate or out-of-order trial start")
|
| 117 | state.update(started=True, outcome="interrupted")
|
| 118 | frozen_timeout = plan.get("manifest", {}).get("trial_timeout_seconds")
|
| 119 | state["budget_truncated"] = frozen_timeout is not None and event.get("timeout_seconds") != frozen_timeout
|
| 120 | elif event.get("event") == "finished":
|
| 121 | if state["finished"]:
|
| 122 | raise ValueError("duplicate terminal trial event")
|
| 123 | outcome, row = _outcome(event)
|
| 124 | if not state["started"] and outcome != "skipped":
|
| 125 | raise ValueError("finished trial has no start event")
|
| 126 | if isinstance(row, dict):
|
| 127 | trial = state["trial"]
|
| 128 | for row_key, expected in (("task_id", trial["trial_id"]), ("bench_id", plan.get("campaign_id")),
|
| 129 | ("run", trial.get("repetition"))):
|
| 130 | if row_key in row and row[row_key] != expected:
|
| 131 | raise ValueError("row trial identity contradicts schedule")
|
| 132 | for row_key, trial_key in (("subject", "subject"), ("scenario", "workflow")):
|
| 133 | if row_key in row and row[row_key] != trial[trial_key]:
|
| 134 | raise ValueError("row identity contradicts scheduled trial")
|
| 135 | agent = row.get("agent", {})
|
| 136 | if isinstance(agent, dict):
|
| 137 | for key, planned in (("name", "agent"), ("model", "model"), ("instruction_level", "instruction_level")):
|
| 138 | if agent.get(key) is not None and agent[key] != trial[planned]:
|
| 139 | raise ValueError("row agent condition contradicts scheduled trial")
|
| 140 | state.update(finished=True, outcome=outcome, row=row)
|
| 141 | if state.get("budget_truncated"):
|
| 142 | state["outcome"] = "interrupted"
|
| 143 | else:
|
| 144 | raise ValueError("unknown journal event")
|
| 145 | return list(trials.values())
|
| 146 |
|
| 147 |
|
| 148 | def _counts(states):
|
| 149 | result = {"scheduled": len(states), "started": sum(state["started"] for state in states),
|
| 150 | "finished": sum(state["finished"] for state in states)}
|
| 151 | result["budget_truncated"] = sum(bool(state.get("budget_truncated")) for state in states)
|
| 152 | result.update({name: sum(state["outcome"] == name for state in states) for name in OUTCOMES})
|
| 153 | return result
|
| 154 |
|
| 155 |
|
| 156 | def _resources(states):
|
| 157 | attempted = [state for state in states if state["started"]]
|
| 158 | successful = [state for state in states if state["outcome"] == "success"]
|
| 159 | resources = {}
|
| 160 | for name, path in METRICS.items():
|
| 161 | values = [value for state in attempted if (value := _metric(state["row"], path)) is not None]
|
| 162 | success_values = [value for state in successful if (value := _metric(state["row"], path)) is not None]
|
| 163 | observed_sum = sum(values) if values else None
|
| 164 | total = observed_sum if attempted and len(values) == len(attempted) else None
|
| 165 | resources[name] = {
|
| 166 | "attempted_n": len(attempted), "measured_n": len(values), "missing_n": len(attempted) - len(values),
|
| 167 | "observed_sum": observed_sum, "complete_total": total,
|
| 168 | "observed_cost_per_success": total / len(successful) if total is not None and successful else None,
|
| 169 | "success_conditioned": {
|
| 170 | "successful_n": len(successful), "measured_n": len(success_values),
|
| 171 | "missing_n": len(successful) - len(success_values),
|
| 172 | "median": statistics.median(success_values) if success_values else None,
|
| 173 | "mean": statistics.fmean(success_values) if success_values else None,
|
| 174 | },
|
| 175 | }
|
| 176 | return resources
|
| 177 |
|
| 178 |
|
| 179 | def _paired(states, bootstrap_samples, confidence, seed):
|
| 180 | by_subject = {}
|
| 181 | for state in states:
|
| 182 | trial = state["trial"]
|
| 183 | by_subject.setdefault(trial["subject"], {})[trial["block_id"]] = state
|
| 184 | comparisons = []
|
| 185 | for reference, candidate in itertools.combinations(sorted(by_subject), 2):
|
| 186 | left, right = by_subject[reference], by_subject[candidate]
|
| 187 | blocks = sorted(set(left) | set(right))
|
| 188 | complete = [block for block in blocks if block in left and block in right
|
| 189 | and left[block]["outcome"] in COMPLETE and right[block]["outcome"] in COMPLETE]
|
| 190 | for block in complete:
|
| 191 | if isinstance(left[block]["row"], dict) and isinstance(right[block]["row"], dict):
|
| 192 | mismatch = comparison_conflicts(comparison_identity(left[block]["row"]), comparison_identity(right[block]["row"]))
|
| 193 | if mismatch:
|
| 194 | raise ValueError("incompatible paired evidence: " + ", ".join(mismatch))
|
| 195 | deltas = [int(right[block]["outcome"] == "success") - int(left[block]["outcome"] == "success")
|
| 196 | for block in complete]
|
| 197 | interval = None
|
| 198 | if len(deltas) >= 2:
|
| 199 | interval = list(bootstrap_ci([100 * delta for delta in deltas], statistics.fmean,
|
| 200 | samples=bootstrap_samples, confidence=confidence, seed=seed))
|
| 201 | comparisons.append({
|
| 202 | "reference": reference, "candidate": candidate, "scheduled_blocks": len(blocks),
|
| 203 | "complete_pairs": len(complete), "incomplete_pairs": len(blocks) - len(complete),
|
| 204 | "success_delta_percentage_points": 100 * statistics.fmean(deltas) if deltas else None,
|
| 205 | "confidence_interval": interval, "confidence": confidence,
|
| 206 | "method": "percentile_bootstrap_recorded_trial_blocks",
|
| 207 | "estimand": "candidate_minus_reference_verified_success_on_complete_pairs",
|
| 208 | "multiplicity_adjusted": False, "claim_eligible": False,
|
| 209 | "success_conditioned_efficiency": _paired_efficiency(left, right, complete, bootstrap_samples, confidence, seed),
|
| 210 | })
|
| 211 | return comparisons
|
| 212 |
|
| 213 |
|
| 214 | def _paired_efficiency(left, right, complete, samples, confidence, seed):
|
| 215 | successes = [block for block in complete if left[block]["outcome"] == right[block]["outcome"] == "success"]
|
| 216 | result = {}
|
| 217 | for metric, source_path in (("agent_active_ms", ("wall_clock_metrics", "measurement_source")),
|
| 218 | ("reported_tokens", ("token_metrics", "token_source"))):
|
| 219 | pairs = []
|
| 220 | missing = incompatible = 0
|
| 221 | conventions = set()
|
| 222 | for block in successes:
|
| 223 | a, b = left[block]["row"], right[block]["row"]
|
| 224 | av, bv = _metric(a, METRICS[metric]), _metric(b, METRICS[metric])
|
| 225 | if av is None or bv is None:
|
| 226 | missing += 1
|
| 227 | continue
|
| 228 | a_source = a.get(source_path[0], {}).get(source_path[1])
|
| 229 | b_source = b.get(source_path[0], {}).get(source_path[1])
|
| 230 | if not isinstance(a_source, str) or not a_source or a_source != b_source:
|
| 231 | incompatible += 1
|
| 232 | continue
|
| 233 | conventions.add(a_source)
|
| 234 | pairs.append((av, bv))
|
| 235 | # Different conventions are not one population even if each pair agrees.
|
| 236 | if len(conventions) > 1:
|
| 237 | incompatible += len(pairs)
|
| 238 | pairs = []
|
| 239 | def difference(values):
|
| 240 | return statistics.fmean(b - a for a, b in values)
|
| 241 | interval = None
|
| 242 | if len(pairs) >= 2:
|
| 243 | interval = list(bootstrap_ci([b - a for a, b in pairs], statistics.fmean,
|
| 244 | samples=samples, confidence=confidence, seed=seed))
|
| 245 | result[metric] = {"jointly_successful_pairs": len(successes), "measured_pairs": len(pairs),
|
| 246 | "missing_metric_pairs": missing, "incompatible_or_unknown_source_pairs": incompatible,
|
| 247 | "measurement_source": next(iter(conventions)) if len(conventions) == 1 else None,
|
| 248 | "candidate_minus_reference_mean_delta": difference(pairs) if pairs else None,
|
| 249 | "confidence_interval": interval, "confidence": confidence,
|
| 250 | "estimand": "paired_mean_difference_conditioned_on_both_subjects_succeeding",
|
| 251 | "lower_is_better": True, "claim_eligible": False}
|
| 252 | return result
|
| 253 |
|
| 254 |
|
| 255 | def report_campaign(plan: dict, events: list[dict], *, bootstrap_samples=2000, confidence=.95, seed=20260910) -> dict:
|
| 256 | validate_interval_settings(bootstrap_samples, confidence)
|
| 257 | states = _trials(plan, events)
|
| 258 | grouped = {}
|
| 259 | for state in states:
|
| 260 | key = tuple(state["trial"][field] for field in CONDITION_FIELDS)
|
| 261 | grouped.setdefault(key, []).append(state)
|
| 262 | groups = []
|
| 263 | for key, condition_states in sorted(grouped.items(), key=lambda item: repr(item[0])):
|
| 264 | subjects = []
|
| 265 | for subject in sorted({state["trial"]["subject"] for state in condition_states}):
|
| 266 | selected = [state for state in condition_states if state["trial"]["subject"] == subject]
|
| 267 | conflicts = population_conflicts([state["row"] for state in selected if isinstance(state["row"], dict)])
|
| 268 | if conflicts:
|
| 269 | raise ValueError("incompatible campaign population: " + ", ".join(conflicts))
|
| 270 | counts = _counts(selected)
|
| 271 | complete = counts["started"] > 0 and not counts["interrupted"] and not counts["skipped"]
|
| 272 | subjects.append({"subject": subject, "counts": counts,
|
| 273 | "verified_success_fraction_of_started": counts["success"] / counts["started"] if counts["started"] else None,
|
| 274 | "success_wilson_interval": _wilson(counts["success"], counts["started"], confidence) if complete else None,
|
| 275 | "success_interval_reason": None if complete else "no_attempts_or_unresolved_started_trials",
|
| 276 | "resources": _resources(selected)})
|
| 277 | groups.append({"condition": dict(zip(CONDITION_FIELDS, key)), "subjects": subjects,
|
| 278 | "paired_comparisons": _paired(condition_states, bootstrap_samples, confidence, seed)})
|
| 279 | counts = _counts(states)
|
| 280 | reasons = ["diagnostic_report_only_no_confirmatory_inference", "independent_durable_evidence_not_certified_by_report"]
|
| 281 | if any(state["trial"]["agent"] == "mock" or
|
| 282 | (isinstance(state["row"], dict) and (state["row"].get("agent") or {}).get("adapter") == "deterministic_mock")
|
| 283 | for state in states):
|
| 284 | reasons.append("mock_agent_measurements_are_instrument_checks")
|
| 285 | if counts["budget_truncated"]:
|
| 286 | reasons.append("recorded_timeout_differs_from_frozen_budget")
|
| 287 | if any(event.get("event") == "journal_recovery" for event in events):
|
| 288 | reasons.append("journal_recovery_contains_uncertain_evidence")
|
| 289 | if counts["interrupted"] or counts["unstarted"] or counts["skipped"]:
|
| 290 | reasons.append("schedule_or_attempt_accounting_incomplete")
|
| 291 | if any(not state["trial"].get("model") for state in states):
|
| 292 | reasons.append("model_identity_unresolved")
|
| 293 | return {"schema_version": 1, "campaign_id": plan.get("campaign_id"), "plan_sha256": plan["plan_sha256"],
|
| 294 | "journal_recoveries": [event for event in events if event.get("event") == "journal_recovery"],
|
| 295 | "invalidated_count": sum(event.get("measurement_invalidated") is True for event in events),
|
| 296 | "invalidated_trials": [{"trial_id": event["trial_id"], "reason": event["invalidation_reason"],
|
| 297 | "rejected_row_retained": event.get("rejected_row") is not None}
|
| 298 | for event in events if event.get("measurement_invalidated") is True],
|
| 299 | "evidence_class": "diagnostic_agent_campaign", "claim_eligible": False,
|
| 300 | "claim_ineligibility_reasons": reasons, "counts": counts, "groups": groups,
|
| 301 | "confidence": confidence, "bootstrap_samples": bootstrap_samples, "analysis_seed": seed,
|
| 302 | "notes": ["Agent active time is outer CLI time including tool waits, not isolated model time.",
|
| 303 | "Cost-weighted tokens are a proxy, never USD. Missing usage makes complete totals null.",
|
| 304 | "Paired intervals condition on complete recorded blocks; incomplete pairs remain in denominators.",
|
| 305 | "A nonempty requested model does not establish provider-resolved model identity.",
|
| 306 | "Budget-truncated terminal trials are retained as interruptions and excluded from complete pairs.",
|
| 307 | "Uncertain torn starts are conservative reservations, not observed execution or elapsed time.",
|
| 308 | "Fixed-task repetitions do not establish independent-session or unseen-task generalization."]}
|
| 309 |
|
| 310 |
|
| 311 | def render_markdown(summary: dict) -> str:
|
| 312 | lines = [f"# Agent campaign: {summary['campaign_id']}", "", "Diagnostic evidence only; no public claim is certified.", ""]
|
| 313 | lines.append(f"Invalidated measurements: {summary['invalidated_count']} (excluded from complete pairs).")
|
| 314 | for item in summary["invalidated_trials"]:
|
| 315 | lines.append(f"- {item['trial_id']}: {item['reason']}; rejected row retained in journal: {item['rejected_row_retained']}")
|
| 316 | lines.append("")
|
| 317 | def cell(value):
|
| 318 | return "unmeasured" if value is None else f"{value:.3f}"
|
| 319 | for group in summary["groups"]:
|
| 320 | lines.extend(["Condition: " + ", ".join(f"{key}={value}" for key, value in group["condition"].items()), "",
|
| 321 | "| Subject | Scheduled | Started | Success | Failure | Timeout | Infrastructure | Interrupted | Skipped | Unstarted | Verified / started | Wilson interval |",
|
| 322 | "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |"])
|
| 323 | for subject in group["subjects"]:
|
| 324 | counts = subject["counts"]
|
| 325 | lines.append("| " + subject["subject"] + " | " + " | ".join(str(counts[name]) for name in
|
| 326 | ("scheduled", "started", "success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped", "unstarted")) +
|
| 327 | f" | {cell(subject['verified_success_fraction_of_started'])} | {subject['success_wilson_interval'] or 'unmeasured'} |")
|
| 328 | lines.extend(["", "Efficiency below is conditioned on successful trials; totals include all started attempts.", "",
|
| 329 | "| Subject / metric | Successful median | Measured successes | All-attempt measured / started | Complete total |",
|
| 330 | "| --- | ---: | ---: | --- | ---: |"])
|
| 331 | for subject in group["subjects"]:
|
| 332 | for metric, values in subject["resources"].items():
|
| 333 | success = values["success_conditioned"]
|
| 334 | lines.append(f"| {subject['subject']} / {metric} | {cell(success['median'])} | {success['measured_n']}/{success['successful_n']} | {values['measured_n']}/{values['attempted_n']} | {cell(values['complete_total'])} |")
|
| 335 | lines.append("")
|
| 336 | for pair in group["paired_comparisons"]:
|
| 337 | lines.append(f"{pair['candidate']} minus {pair['reference']}: {cell(pair['success_delta_percentage_points'])} percentage points on {pair['complete_pairs']}/{pair['scheduled_blocks']} complete pairs; descriptive bootstrap CI: {pair['confidence_interval'] or 'unmeasured'}; multiplicity unadjusted.")
|
| 338 | for metric, evidence in pair["success_conditioned_efficiency"].items():
|
| 339 | lines.append(f"Paired {metric} mean difference, conditioned on both succeeding: {cell(evidence['candidate_minus_reference_mean_delta'])}; CI {evidence['confidence_interval'] or 'unmeasured'}; measured {evidence['measured_pairs']}/{evidence['jointly_successful_pairs']} pairs; missing metrics {evidence['missing_metric_pairs']}; incompatible/unknown sources {evidence['incompatible_or_unknown_source_pairs']}. Negative favors candidate.")
|
| 340 | lines.append("")
|
| 341 | lines.extend(summary["notes"])
|
| 342 | lines.extend(["", "Claim ineligibility: " + "; ".join(summary["claim_ineligibility_reasons"]), ""])
|
| 343 | return "\n".join(lines)
|