"""Diagnostic campaign accounting from a frozen schedule and durable journal.

No publication claim is certified here. Block bootstrap describes recorded

paired trials; repeated fixed fixtures are not independent task populations.

"""

from __future__ import annotations

import itertools

import math

import statistics

from statistics import NormalDist

from oakbench.stats import bootstrap_ci, validate_interval_settings

from oakbench.populations import population_conflicts, comparison_identity, comparison_conflicts

CONDITION_FIELDS = ("workflow", "instruction_level", "agent", "model")

OUTCOMES = ("success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped", "unstarted")

COMPLETE = {"success", "failure", "timeout"}

METRICS = {

    "agent_active_ms": ("wall_clock_metrics", "agent_active_ms"),

    "task_total_ms": ("wall_clock_metrics", "total_ms"),

    "reported_tokens": ("token_metrics", "total_tokens_reported"),

    "cost_weighted_token_proxy": ("token_metrics", "cost_weighted_total"),

    "assistant_turns": ("turn_metrics", "assistant_turns_total"),

    "tool_calls": ("tool_call_metrics", "tool_calls_total"),

}

def _number(value):

    return float(value) if type(value) in (int, float) and math.isfinite(value) and value >= 0 else None

def _metric(row, path):

    if isinstance(row, dict) and path[0] in {"token_metrics", "turn_metrics", "tool_call_metrics"}:

        execution = (row.get("source") or {}).get("agent_execution", {})

        incomplete = (isinstance(execution, dict) and

                      (execution.get("capture_complete") is False or execution.get("incomplete") is True or

                       execution.get("usage_complete") is False or execution.get("transcript_metrics_complete") is False or

                       execution.get("counters_coverage") == "observed_lower_bounds" or execution.get("output_limited") is True or

                       execution.get("timed_out") is True or execution.get("output_truncated") is True))

        if row.get("outcome") == "timeout" or row.get("output_truncated") is True or incomplete:

            return None

    value = row

    for part in path:

        value = value.get(part) if isinstance(value, dict) else None

    return _number(value)

def _wilson(successes, n, confidence):

    if n == 0:

        return None

    z = NormalDist().inv_cdf((1 + confidence) / 2)

    p = successes / n

    scale = 1 + z * z / n

    centre = (p + z * z / (2 * n)) / scale

    radius = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / scale

    return [max(0.0, centre - radius), min(1.0, centre + radius)]

def _outcome(event):

    row = event.get("row")

    explicit = event.get("outcome")

    if explicit is not None and isinstance(row, dict) and explicit != row.get("outcome"):

        raise ValueError("event outcome contradicts terminal row")

    if event.get("measurement_invalidated") is True:

        if explicit != "error" or row is not None or not event.get("invalidation_reason"):

            raise ValueError("invalidated measurement has contradictory terminal evidence")

        return "skipped", None

    if isinstance(row, dict) and row.get("returncode") == 77:

        if row.get("outcome") != "error" or row.get("success") is not False or not row.get("skip_reason"):

            raise ValueError("unmeasured instrument row has contradictory outcome")

        return "skipped", row

    if isinstance(row, dict) and (row.get("outcome_attribution") in {"harness", "subject"} or row.get("agent_launched") is False):

        return "infrastructure_failure", row

    if explicit in {"skipped", "skip"}:

        return "skipped", row

    if explicit == "interrupted":

        return "interrupted", row

    if explicit in {"error", "infrastructure_failure"}:

        if row is not None and (not isinstance(row, dict) or row.get("outcome") != "error" or row.get("success") is not False):

            raise ValueError("infrastructure error contradicts terminal task row")

        return "infrastructure_failure", row

    if not isinstance(row, dict) or type(row.get("success")) is not bool:

        raise ValueError("finished trial lacks an explicit row success outcome")

    outcome = row.get("outcome")

    if outcome not in {"pass", "fail", "timeout", "error"} or (row["success"] != (outcome == "pass")):

        raise ValueError("contradictory or unknown row outcome")

    return {"pass": "success", "fail": "failure", "timeout": "timeout", "error": "infrastructure_failure"}[outcome], row

def _trials(plan, events):

    if plan.get("schema_version") != 1 or not isinstance(plan.get("schedule"), list):

        raise ValueError("unsupported campaign plan")

    if not isinstance(plan.get("plan_sha256"), str) or not plan["plan_sha256"]:

        raise ValueError("campaign plan digest missing")

    trials = {}

    cells = set()

    for item in plan["schedule"]:

        if not all(key in item for key in ("trial_id", "block_id", "subject", *CONDITION_FIELDS)):

            raise ValueError("schedule trial missing identity")

        trial_id = item["trial_id"]

        cell = tuple(item[key] for key in (*CONDITION_FIELDS, "block_id", "subject"))

        if trial_id in trials or cell in cells:

            raise ValueError("duplicate scheduled trial or subject in paired block")

        cells.add(cell)

        trials[trial_id] = {"trial": item, "started": False, "finished": False,

                            "outcome": "unstarted", "row": None}

    for event in events:

        if event.get("plan_sha256") != plan["plan_sha256"]:

            raise ValueError("journal plan digest mismatch")

        if event.get("event") == "journal_recovery":

            continue

        if event.get("trial_id") not in trials:

            raise ValueError("journal names unscheduled trial")

        state = trials[event["trial_id"]]

        if event.get("event") == "started":

            if state["started"] or state["finished"]:

                raise ValueError("duplicate or out-of-order trial start")

            state.update(started=True, outcome="interrupted")

            frozen_timeout = plan.get("manifest", {}).get("trial_timeout_seconds")

            state["budget_truncated"] = frozen_timeout is not None and event.get("timeout_seconds") != frozen_timeout

        elif event.get("event") == "finished":

            if state["finished"]:

                raise ValueError("duplicate terminal trial event")

            outcome, row = _outcome(event)

            if not state["started"] and outcome != "skipped":

                raise ValueError("finished trial has no start event")

            if isinstance(row, dict):

                trial = state["trial"]

                for row_key, expected in (("task_id", trial["trial_id"]), ("bench_id", plan.get("campaign_id")),

                                          ("run", trial.get("repetition"))):

                    if row_key in row and row[row_key] != expected:

                        raise ValueError("row trial identity contradicts schedule")

                for row_key, trial_key in (("subject", "subject"), ("scenario", "workflow")):

                    if row_key in row and row[row_key] != trial[trial_key]:

                        raise ValueError("row identity contradicts scheduled trial")

                agent = row.get("agent", {})

                if isinstance(agent, dict):

                    for key, planned in (("name", "agent"), ("model", "model"), ("instruction_level", "instruction_level")):

                        if agent.get(key) is not None and agent[key] != trial[planned]:

                            raise ValueError("row agent condition contradicts scheduled trial")

            state.update(finished=True, outcome=outcome, row=row)

            if state.get("budget_truncated"):

                state["outcome"] = "interrupted"

        else:

            raise ValueError("unknown journal event")

    return list(trials.values())

def _counts(states):

    result = {"scheduled": len(states), "started": sum(state["started"] for state in states),

              "finished": sum(state["finished"] for state in states)}

    result["budget_truncated"] = sum(bool(state.get("budget_truncated")) for state in states)

    result.update({name: sum(state["outcome"] == name for state in states) for name in OUTCOMES})

    return result

def _resources(states):

    attempted = [state for state in states if state["started"]]

    successful = [state for state in states if state["outcome"] == "success"]

    resources = {}

    for name, path in METRICS.items():

        values = [value for state in attempted if (value := _metric(state["row"], path)) is not None]

        success_values = [value for state in successful if (value := _metric(state["row"], path)) is not None]

        observed_sum = sum(values) if values else None

        total = observed_sum if attempted and len(values) == len(attempted) else None

        resources[name] = {

            "attempted_n": len(attempted), "measured_n": len(values), "missing_n": len(attempted) - len(values),

            "observed_sum": observed_sum, "complete_total": total,

            "observed_cost_per_success": total / len(successful) if total is not None and successful else None,

            "success_conditioned": {

                "successful_n": len(successful), "measured_n": len(success_values),

                "missing_n": len(successful) - len(success_values),

                "median": statistics.median(success_values) if success_values else None,

                "mean": statistics.fmean(success_values) if success_values else None,

            },

        }

    return resources

def _paired(states, bootstrap_samples, confidence, seed):

    by_subject = {}

    for state in states:

        trial = state["trial"]

        by_subject.setdefault(trial["subject"], {})[trial["block_id"]] = state

    comparisons = []

    for reference, candidate in itertools.combinations(sorted(by_subject), 2):

        left, right = by_subject[reference], by_subject[candidate]

        blocks = sorted(set(left) | set(right))

        complete = [block for block in blocks if block in left and block in right

                    and left[block]["outcome"] in COMPLETE and right[block]["outcome"] in COMPLETE]

        for block in complete:

            if isinstance(left[block]["row"], dict) and isinstance(right[block]["row"], dict):

                mismatch = comparison_conflicts(comparison_identity(left[block]["row"]), comparison_identity(right[block]["row"]))

                # A measured incorrect worktree need not reach a committed-tree

                # proof. Keep that failure in the denominator; successful proof

                # classes must still agree before comparison.

                if any(row.get("workflow_integrity_passed") is False

                       and row.get("agent_launched") is True and row.get("outcome_attribution") == "agent"

                       for row in (left[block]["row"], right[block]["row"])):

                    mismatch = [field for field in mismatch if field != "workflow_integrity_evidence_class"]

                if mismatch:

                    raise ValueError("incompatible paired evidence: " + ", ".join(mismatch))

        deltas = [int(right[block]["outcome"] == "success") - int(left[block]["outcome"] == "success")

                  for block in complete]

        interval = None

        if len(deltas) >= 2:

            interval = list(bootstrap_ci([100 * delta for delta in deltas], statistics.fmean,

                                         samples=bootstrap_samples, confidence=confidence, seed=seed))

        comparisons.append({

            "reference": reference, "candidate": candidate, "scheduled_blocks": len(blocks),

            "complete_pairs": len(complete), "incomplete_pairs": len(blocks) - len(complete),

            "success_delta_percentage_points": 100 * statistics.fmean(deltas) if deltas else None,

            "confidence_interval": interval, "confidence": confidence,

            "method": "percentile_bootstrap_recorded_trial_blocks",

            "estimand": "candidate_minus_reference_verified_success_on_complete_pairs",

            "multiplicity_adjusted": False, "claim_eligible": False,

            "success_conditioned_efficiency": _paired_efficiency(left, right, complete, bootstrap_samples, confidence, seed),

        })

    return comparisons

def _paired_efficiency(left, right, complete, samples, confidence, seed):

    successes = [block for block in complete if left[block]["outcome"] == right[block]["outcome"] == "success"]

    result = {}

    for metric, source_path in (("agent_active_ms", ("wall_clock_metrics", "measurement_source")),

                                ("reported_tokens", ("token_metrics", "token_source"))):

        pairs = []

        missing = incompatible = 0

        conventions = set()

        for block in successes:

            a, b = left[block]["row"], right[block]["row"]

            av, bv = _metric(a, METRICS[metric]), _metric(b, METRICS[metric])

            if av is None or bv is None:

                missing += 1

                continue

            a_source = a.get(source_path[0], {}).get(source_path[1])

            b_source = b.get(source_path[0], {}).get(source_path[1])

            if not isinstance(a_source, str) or not a_source or a_source != b_source:

                incompatible += 1

                continue

            conventions.add(a_source)

            pairs.append((av, bv))

        # Different conventions are not one population even if each pair agrees.

        if len(conventions) > 1:

            incompatible += len(pairs)

            pairs = []

        def difference(values):

            return statistics.fmean(b - a for a, b in values)

        interval = None

        if len(pairs) >= 2:

            interval = list(bootstrap_ci([b - a for a, b in pairs], statistics.fmean,

                                         samples=samples, confidence=confidence, seed=seed))

        result[metric] = {"jointly_successful_pairs": len(successes), "measured_pairs": len(pairs),

                          "missing_metric_pairs": missing, "incompatible_or_unknown_source_pairs": incompatible,

                          "measurement_source": next(iter(conventions)) if len(conventions) == 1 else None,

                          "candidate_minus_reference_mean_delta": difference(pairs) if pairs else None,

                          "confidence_interval": interval, "confidence": confidence,

                          "estimand": "paired_mean_difference_conditioned_on_both_subjects_succeeding",

                          "lower_is_better": True, "claim_eligible": False}

    return result

def report_campaign(plan: dict, events: list[dict], *, bootstrap_samples=2000, confidence=.95, seed=20260910) -> dict:

    validate_interval_settings(bootstrap_samples, confidence)

    states = _trials(plan, events)

    grouped = {}

    for state in states:

        key = tuple(state["trial"][field] for field in CONDITION_FIELDS)

        grouped.setdefault(key, []).append(state)

    groups = []

    for key, condition_states in sorted(grouped.items(), key=lambda item: repr(item[0])):

        subjects = []

        for subject in sorted({state["trial"]["subject"] for state in condition_states}):

            selected = [state for state in condition_states if state["trial"]["subject"] == subject]

            conflicts = population_conflicts([state["row"] for state in selected if isinstance(state["row"], dict)])

            if conflicts:

                raise ValueError("incompatible campaign population: " + ", ".join(conflicts))

            counts = _counts(selected)

            complete = counts["started"] > 0 and not counts["interrupted"] and not counts["skipped"] and not counts["infrastructure_failure"]

            subjects.append({"subject": subject, "counts": counts,

                             "verified_success_fraction_of_started": counts["success"] / counts["started"] if counts["started"] else None,

                             "success_wilson_interval": _wilson(counts["success"], counts["started"], confidence) if complete else None,

                             "success_interval_reason": None if complete else "no_attempts_or_unresolved_started_trials",

                             "resources": _resources(selected)})

        groups.append({"condition": dict(zip(CONDITION_FIELDS, key)), "subjects": subjects,

                       "paired_comparisons": _paired(condition_states, bootstrap_samples, confidence, seed)})

    counts = _counts(states)

    reasons = ["diagnostic_report_only_no_confirmatory_inference", "independent_durable_evidence_not_certified_by_report"]

    if any(state["trial"]["agent"] == "mock" or

           (isinstance(state["row"], dict) and (state["row"].get("agent") or {}).get("adapter") == "deterministic_mock")

           for state in states):

        reasons.append("mock_agent_measurements_are_instrument_checks")

    if counts["budget_truncated"]:

        reasons.append("recorded_timeout_differs_from_frozen_budget")

    if any(event.get("event") == "journal_recovery" for event in events):

        reasons.append("journal_recovery_contains_uncertain_evidence")

    if counts["interrupted"] or counts["unstarted"] or counts["skipped"] or counts["infrastructure_failure"]:

        reasons.append("schedule_or_attempt_accounting_incomplete")

    if any(not state["trial"].get("model") for state in states):

        reasons.append("model_identity_unresolved")

    return {"schema_version": 1, "campaign_id": plan.get("campaign_id"), "plan_sha256": plan["plan_sha256"],

            "journal_recoveries": [event for event in events if event.get("event") == "journal_recovery"],

            "invalidated_count": sum(event.get("measurement_invalidated") is True for event in events),

            "invalidated_trials": [{"trial_id": event["trial_id"], "reason": event["invalidation_reason"],

                                    "rejected_row_retained": event.get("rejected_row") is not None}

                                   for event in events if event.get("measurement_invalidated") is True],

            "evidence_class": "diagnostic_agent_campaign", "claim_eligible": False,

            "claim_ineligibility_reasons": reasons, "counts": counts, "groups": groups,

            "confidence": confidence, "bootstrap_samples": bootstrap_samples, "analysis_seed": seed,

            "notes": ["Agent active time is outer CLI time including tool waits, not isolated model time.",

                      "Cost-weighted tokens are a proxy, never USD. Missing usage makes complete totals null.",

                      "Paired intervals condition on complete recorded blocks; incomplete pairs remain in denominators.",

                      "A nonempty requested model does not establish provider-resolved model identity.",

                      "Budget-truncated terminal trials are retained as interruptions and excluded from complete pairs.",

                      "Uncertain torn starts are conservative reservations, not observed execution or elapsed time.",

                      "Fixed-task repetitions do not establish independent-session or unseen-task generalization."]}

def render_markdown(summary: dict) -> str:

    lines = [f"# Agent campaign: {summary['campaign_id']}", "", "Diagnostic evidence only; no public claim is certified.", ""]

    lines.append(f"Invalidated measurements: {summary['invalidated_count']} (excluded from complete pairs).")

    for item in summary["invalidated_trials"]:

        lines.append(f"- {item['trial_id']}: {item['reason']}; rejected row retained in journal: {item['rejected_row_retained']}")

    lines.append("")

    def cell(value):

        return "unmeasured" if value is None else f"{value:.3f}"

    for group in summary["groups"]:

        lines.extend(["Condition: " + ", ".join(f"{key}={value}" for key, value in group["condition"].items()), "",

                      "| Subject | Scheduled | Started | Success | Failure | Timeout | Infrastructure | Interrupted | Skipped | Unstarted | Verified / started | Wilson interval |",

                      "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |"])

        for subject in group["subjects"]:

            counts = subject["counts"]

            lines.append("| " + subject["subject"] + " | " + " | ".join(str(counts[name]) for name in

                         ("scheduled", "started", "success", "failure", "timeout", "infrastructure_failure", "interrupted", "skipped", "unstarted")) +

                         f" | {cell(subject['verified_success_fraction_of_started'])} | {subject['success_wilson_interval'] or 'unmeasured'} |")

        lines.extend(["", "Efficiency below is conditioned on successful trials; totals include all started attempts.", "",

                      "| Subject / metric | Successful median | Measured successes | All-attempt measured / started | Complete total |",

                      "| --- | ---: | ---: | --- | ---: |"])

        for subject in group["subjects"]:

            for metric, values in subject["resources"].items():

                success = values["success_conditioned"]

                lines.append(f"| {subject['subject']} / {metric} | {cell(success['median'])} | {success['measured_n']}/{success['successful_n']} | {values['measured_n']}/{values['attempted_n']} | {cell(values['complete_total'])} |")

        lines.append("")

        for pair in group["paired_comparisons"]:

            lines.append(f"{pair['candidate']} minus {pair['reference']}: {cell(pair['success_delta_percentage_points'])} percentage points on {pair['complete_pairs']}/{pair['scheduled_blocks']} complete pairs; descriptive bootstrap CI: {pair['confidence_interval'] or 'unmeasured'}; multiplicity unadjusted.")

            for metric, evidence in pair["success_conditioned_efficiency"].items():

                lines.append(f"Paired {metric} mean difference, conditioned on both succeeding: {cell(evidence['candidate_minus_reference_mean_delta'])}; CI {evidence['confidence_interval'] or 'unmeasured'}; measured {evidence['measured_pairs']}/{evidence['jointly_successful_pairs']} pairs; missing metrics {evidence['missing_metric_pairs']}; incompatible/unknown sources {evidence['incompatible_or_unknown_source_pairs']}. Negative favors candidate.")

        lines.append("")

    lines.extend(summary["notes"])

    lines.extend(["", "Claim ineligibility: " + "; ".join(summary["claim_ineligibility_reasons"]), ""])

    return "\n".join(lines)