"""Target scorecard evaluation."""

from __future__ import annotations

from typing import Any

import math

from oakbench.baseline_book import BaselineCell, Target, iter_jsonl, metric_value

from oakbench.integrity import content_integrity_public_trust_passed

from oakbench.rows import number_or_none, row_returncode

from oakbench.populations import (population_conflicts, comparison_identity, comparison_conflicts,

                                 independent_run_count, missing_claim_identity)

from oakbench.stats import benjamini_hochberg_detail, bootstrap_ci, named_statistic, required_n_for_relative_delta, validate_interval_settings

VERDICTS = ("GREEN", "RED", "AMBER", "UNDECIDABLE", "UNMEASURED", "INVALID")

# One JSONL loader for scorecard inputs; baseline_book owns the implementation.

load_jsonl = iter_jsonl

def evaluate_targets(

    rows: list[dict[str, Any]],

    targets: list[Target],

    baseline: dict[tuple[str, str | None, str, str, str, str, str], BaselineCell],

    *,

    bootstrap_samples: int = 2000,

    confidence: float = 0.95,

    seed: int = 20260609,

    mode: str = "diagnostic",

    fdr: float = 0.05,

) -> list[dict[str, Any]]:

    validate_interval_settings(bootstrap_samples, confidence)

    if mode not in {"diagnostic", "claim"}:

        raise ValueError("mode must be diagnostic or claim")

    results = [

        evaluate_target(

            rows,

            target,

            baseline.get(target.baseline_key),

            bootstrap_samples=bootstrap_samples,

            confidence=confidence,

            seed=seed,

            mode=mode,

        )

        for target in targets

    ]

    if mode == "claim":

        apply_claim_fdr(results, fdr=fdr)

    return results

def apply_claim_fdr(results: list[dict[str, Any]], *, fdr: float = 0.05) -> None:

    """Benjamini-Hochberg across every decisive claim-mode target (ADR-0011).

    The family is all GREEN/RED verdicts in one claim evaluation; each carries

    a one-sided bootstrap p-value for its decision. A decisive verdict that BH

    does not confirm is capped at AMBER. There is no global floor here: each

    cell's noise floor is its own (claim eligibility requires per-cell A/A).

    """

    decisive = [item for item in results if item.get("verdict") in ("GREEN", "RED") and item.get("p_value") is not None]

    if not decisive:

        return

    detail = benjamini_hochberg_detail([item["p_value"] for item in decisive], fdr=fdr)

    for item, rejected in zip(decisive, detail["rejected"]):

        item["fdr_family_size"] = len(decisive)

        item["fdr_rejected"] = rejected

        if not rejected:

            item["messages"] = list(item.get("messages") or []) + [

                f"not significant after Benjamini-Hochberg (q={fdr}) across {len(decisive)} claim targets"]

            item["verdict"] = "AMBER"

            item["claim_eligible"] = False

def evaluate_target(

    rows: list[dict[str, Any]],

    target: Target,

    baseline: BaselineCell | None,

    *,

    bootstrap_samples: int,

    confidence: float,

    seed: int,

    mode: str = "diagnostic",

) -> dict[str, Any]:

    matched = [row for row in rows if _row_matches(row, target)]

    skipped = [row for row in matched if row_returncode(row) == 77]

    successful = [row for row in matched if row_returncode(row) == 0]

    failures = [row for row in matched if row_returncode(row) not in (0, 77)]

    values = [value for row in successful if (value := metric_value(row, target.metric)) is not None]

    base = {

        "target_id": target.target_id,

        "verdict": "UNMEASURED",

        "lane": target.lane,

        "profile": target.profile,

        "scenario": target.scenario,

        "operation": target.operation,

        "subject": target.subject,

        "comparator": target.comparator,

        "metric": target.metric,

        "goal_type": target.goal_type,

        "goal_value": target.goal_value,

        # "n" is the number of rows that actually contributed to the statistic

        # (metric measured, per ADR-0002 null-means-unmeasured); "n_successful"

        # is the pool it was drawn from. CI consumers should read "n".

        "n": len(values),

        "n_successful": len(successful),

        "skip_count": len(skipped),

        "failure_count": len(failures),

        "messages": [],

        "mode": mode,

        "claim_eligible": False,

        "independent_n": independent_run_count(successful),

        "confidence": confidence,

        "bootstrap_samples": bootstrap_samples,

    }

    if not matched:

        base["messages"].append("no matching rows")

        return base

    if skipped and not successful and not failures:

        base["messages"].append("; ".join(sorted({str(row.get("skip_reason") or "skipped") for row in skipped})))

        return base

    if baseline is None:

        base["messages"].append("missing baseline cell")

        return base

    guard_errors = guard_violations(matched, successful, failures, target.guards)

    # Outcome denominators are part of the experiment, not a separate identity

    # population. Failed/skipped trials cannot silently come from another host.

    conflicts = population_conflicts(matched)

    if conflicts:

        guard_errors.append(f"incompatible measurement population: {', '.join(conflicts)}")

    candidate_identity = comparison_identity(successful[0]) if successful else None

    if baseline.comparison_identity is not None and candidate_identity is not None:

        mismatch = comparison_conflicts(baseline.comparison_identity, candidate_identity)

        if mismatch:

            guard_errors.append(f"baseline comparison identity mismatch: {', '.join(mismatch)}")

    if guard_errors:

        base.update({"verdict": "INVALID", "messages": guard_errors})

        return base

    if not values:

        base["messages"].append("metric unmeasured in matching successful rows")

        return base

    eligibility_errors = claim_eligibility_errors(successful, matched, target, baseline, confidence)

    base["claim_ineligibility_reasons"] = eligibility_errors

    if mode == "claim" and eligibility_errors:

        base.update(verdict="AMBER", messages=eligibility_errors)

        return base

    base["claim_eligible"] = mode == "claim"

    if mode == "diagnostic":

        base["messages"].append("diagnostic only; not eligible for public claims")

    unmeasured = len(successful) - len(values)

    if unmeasured:

        base["messages"].append(f"metric unmeasured on {unmeasured} of {len(successful)} successful rows")

    if baseline.value == 0:

        base.update({"verdict": "UNDECIDABLE", "messages": ["baseline value is zero; ratio target is undefined"]})

        return base

    observed = statistic_value(values, target.statistic)

    ci_low, ci_high = bootstrap_ci(

        values,

        statistic=target.statistic,

        samples=bootstrap_samples,

        confidence=confidence,

        seed=seed,

    )

    baseline_denominators = _baseline_denominators(baseline)

    if baseline_denominators is None:

        base.update(

            {

                "verdict": "UNDECIDABLE",

                "messages": ["baseline confidence interval crosses zero; ratio target is undefined"],

            }

        )

        return base

    baseline_low, baseline_high = baseline_denominators

    ratio = observed / baseline.value

    ratio_low = ci_low / baseline_high

    ratio_high = ci_high / baseline_low

    base.update(

        {

            "value": observed,

            "ci_low": ci_low,

            "ci_high": ci_high,

            "baseline_value": baseline.value,

            "baseline_ci_low": baseline.ci_low,

            "baseline_ci_high": baseline.ci_high,

            "ratio": ratio,

            "ratio_ci_low": ratio_low,

            "ratio_ci_high": ratio_high,

        }

    )

    # GREEN/RED must clear the goal by more than the baseline noise floor:

    # the bootstrap CI only sees within-sample variance, so a narrow CI that

    # lands inside the noise band is instrument luck, not a verdict (ADR-0008:

    # a target can never out-resolve the instrument). Decisive results — CI

    # bounds that survive noise inflation — stay GREEN/RED.

    noise = float(baseline.noise_floor_pct or 0.0) / 100.0

    decisively_green = ratio_high * (1.0 + noise) <= target.goal_value

    decisively_red = ratio_low * (1.0 - noise) > target.goal_value

    # Partial metric coverage caps decisive verdicts at AMBER (ADR-0008): the

    # statistic only saw the rows where the metric was measured, so a decisive

    # bound may be an artifact of which rows happened to record it. Dropping

    # information can never show green — and, symmetrically, missing data

    # cannot fail a target either, so RED is capped to AMBER as well.

    if (decisively_green or decisively_red) and unmeasured:

        required = required_n_for_relative_delta(float(baseline.noise_floor_pct or 0.0), _target_delta_pct(target), len(values))

        base.update({"verdict": "AMBER", "required_n": required})

    elif decisively_green:

        base["verdict"] = "GREEN"

        base["p_value"] = _one_sided_p(values, target, baseline, noise, "green", bootstrap_samples, seed)

    elif decisively_red:

        base["verdict"] = "RED"

        base["p_value"] = _one_sided_p(values, target, baseline, noise, "red", bootstrap_samples, seed)

    elif _noise_makes_undecidable(target, baseline):

        required = required_n_for_relative_delta(float(baseline.noise_floor_pct or 0.0), _target_delta_pct(target), len(values))

        base.update(

            {

                "verdict": "UNDECIDABLE",

                "required_n": required,

                "messages": ["target is tighter than the baseline noise floor"],

            }

        )

    else:

        required = required_n_for_relative_delta(float(baseline.noise_floor_pct or 0.0), _target_delta_pct(target), len(values))

        base.update({"verdict": "AMBER", "required_n": required})

        base["messages"].append("confidence interval straddles target")

    return base

def _one_sided_p(values, target, baseline, noise, direction, samples, seed) -> float:

    """Bootstrap p that the noise-inflated ratio is on the wrong side of the goal.

    Candidate uncertainty is resampled; the saved baseline enters through its

    recorded CI (normal approximation), since books keep summaries, not samples.

    """

    import random as _random

    rng = _random.Random(seed)

    spread = 0.0

    if baseline.ci_low is not None and baseline.ci_high is not None:

        spread = max(0.0, (baseline.ci_high - baseline.ci_low) / (2 * 1.959964))

    stat = named_statistic(target.statistic)

    wrong = 0

    for _ in range(samples):

        sample = [values[rng.randrange(len(values))] for _ in values]

        denominator = rng.gauss(baseline.value, spread) if spread else baseline.value

        if denominator <= 0:

            wrong += 1

            continue

        ratio = stat(sample) / denominator

        if direction == "green":

            wrong += ratio * (1.0 + noise) > target.goal_value

        else:

            wrong += ratio * (1.0 - noise) <= target.goal_value

    return (wrong + 1) / (samples + 1)

def claim_eligibility_errors(successful, matched, target, baseline, confidence) -> list[str]:

    errors = []

    if confidence < 0.95:

        errors.append("candidate confidence must be at least 0.95 for claims")

    if (baseline.confidence is None or not math.isfinite(baseline.confidence) or

            not 0.95 <= baseline.confidence < 1):

        errors.append("recorded baseline confidence must be at least 0.95 and below 1 for claims")

    measured = [row for row in successful if metric_value(row, target.metric) is not None]

    # n=30 is a conservative eligibility floor, not a precision guarantee.

    minimum = max(30, {"p95": 20, "p99": 100}.get(target.statistic, 0))

    n = independent_run_count(measured)

    if n is None or n < minimum:

        errors.append(f"candidate requires at least {minimum} distinct recorded trials; observed {n}")

    if n is not None and n != len(measured):

        errors.append("candidate contains duplicate trial IDs")

    if baseline.independent_n is None or baseline.independent_n < minimum:

        errors.append(f"baseline requires at least {minimum} distinct recorded trials")

    if baseline.independent_n != baseline.n:

        errors.append("baseline sample count differs from distinct trial count")

    if len(measured) != len(matched):

        errors.append("candidate contains failed, skipped or unmeasured trials")

    if (baseline.failure_n != 0 or baseline.skipped_n != 0 or

            baseline.attempted_n != baseline.n):

        errors.append("baseline outcomes missing or contain failed, skipped or unmeasured trials")

    noise = baseline.noise_floor_pct

    if noise is None or not math.isfinite(noise) or noise < 0:

        errors.append("measured finite nonnegative baseline noise floor required")

    elif target.metric.endswith("_ms") and getattr(baseline, "noise_floor_source", None) != "per_cell_aa":

        # ADR-0011: one operator-supplied floor for every cell is the flaw that

        # published improvements of a binary over itself; claims need the

        # cell's own interleaved A/A floor.

        errors.append("per-cell A/A noise floor required for latency claims (baseline floor is global or unknown)")

    if (baseline.ci_low is None or baseline.ci_high is None or

            not all(math.isfinite(x) for x in (baseline.ci_low, baseline.ci_high)) or

            not baseline.ci_low <= baseline.value <= baseline.ci_high):

        errors.append("valid measured baseline confidence interval required")

    identities = [("candidate", comparison_identity(successful[0]))] if successful else []

    if baseline.comparison_identity is None:

        errors.append("versioned baseline comparison identity required")

    else:

        identities.append(("baseline", baseline.comparison_identity))

    for label, identity in identities:

        missing = missing_claim_identity(identity, target.lane, target.metric, target.operation)

        if any(name in target.guards for name in ("information_recall_at_least", "information_precision_at_least")):

            if not identity.get("output_oracle_version"):

                missing.append("output_oracle_version")

        if target.guards.get("patch_apply_ok") and not identity.get("patch_oracle_version"):

            missing.append("patch_oracle_version")

        if missing:

            errors.append(f"{label} claim identity missing: {', '.join(missing)}")

    return errors

def statistic_value(values: list[float], statistic: str) -> float:

    return float(named_statistic(statistic)(values))

def guard_violations(

    matched: list[dict[str, Any]],

    successful: list[dict[str, Any]],

    failures: list[dict[str, Any]],

    guards: dict[str, Any],

) -> list[str]:

    errors: list[str] = []

    if failures:

        errors.append(f"{len(failures)} matching rows failed")

    if "success_rate_at_least" in guards:

        success_rate = len(successful) / len(matched) if matched else 0.0

        if success_rate < float(guards["success_rate_at_least"]):

            errors.append(f"success_rate {success_rate:.3f} below guard {float(guards['success_rate_at_least']):.3f}")

    if "information_recall_at_least" in guards:

        _minimum_guard(successful, "information_recall", float(guards["information_recall_at_least"]), errors)

    if "information_precision_at_least" in guards:

        _minimum_guard(successful, "information_precision", float(guards["information_precision_at_least"]), errors)

    if guards.get("pipe_compatible_unified") is True:

        missing = [row for row in successful if row.get("pipe_compatible_unified") is None]

        bad = [row for row in successful if row.get("pipe_compatible_unified") is False]

        if missing:

            errors.append(f"{len(missing)} rows have unmeasured pipe_compatible_unified guard")

        if bad:

            errors.append(f"{len(bad)} rows failed pipe_compatible_unified guard")

    if guards.get("patch_apply_ok") is True:

        if any(row.get("patch_apply_ok") is not True for row in successful):

            errors.append("patch_apply_ok guard is failed or unmeasured")

    if guards.get("content_integrity_public_trust") is True:

        missing = [row for row in successful if row.get("content_integrity_check_passed") is None]

        failed = [row for row in successful if row.get("content_integrity_check_passed") is False]

        weak = [

            row

            for row in successful

            if row.get("content_integrity_check_passed") is True

            and not content_integrity_public_trust_passed(row)

        ]

        if missing:

            errors.append(f"{len(missing)} rows have unmeasured content_integrity_public_trust guard")

        if failed:

            errors.append(f"{len(failed)} rows failed content_integrity_public_trust guard")

        if weak:

            errors.append(f"{len(weak)} rows have weak content_integrity_source for public trust")

    return errors

def _minimum_guard(rows: list[dict[str, Any]], key: str, minimum: float, errors: list[str]) -> None:

    if not rows:

        errors.append(f"{key} guard has no successful observations")

        return

    values = [value for row in rows if (value := _number_or_none(row.get(key))) is not None]

    if len(values) != len(rows):

        errors.append(f"{key} guard is unmeasured on {len(rows) - len(values)} rows")

    elif min(values) < minimum:

        errors.append(f"{key} minimum {min(values):.3f} below guard {minimum:.3f}")

def _row_matches(row: dict[str, Any], target: Target) -> bool:

    lane = _row_lane(row)

    return (

        lane == target.lane

        and str(row.get("subject")) == target.subject

        and str(row.get("scenario")) == target.scenario

        and str(row.get("operation")) == target.operation

        and (

            (target.profile is None and row.get("profile") is None)

            or (target.profile is not None and str(row.get("profile")) == target.profile)

        )

    )

def _row_lane(row: dict[str, Any]) -> str | None:

    for key in ("lane", "benchmark_lane", "measurement_lane"):

        if row.get(key) is not None:

            return str(row[key])

    if "workflow" in row or "phase" in row:

        return "workflow"

    if all(key in row for key in ("token_metrics", "turn_metrics", "tool_call_metrics")):

        return "agent"

    if "contention_mode" in row or "workers" in row:

        return "contention"

    if str(row.get("scenario") or "") == "task_loop":

        return "task-loop"

    if row.get("subject_kind") is None and row.get("subject") is not None:

        return "mount"

    if all(key in row for key in ("profile", "scenario", "operation", "subject", "subject_kind", "command")):

        return "core"

    return None

def _noise_makes_undecidable(target: Target, baseline: BaselineCell) -> bool:

    if baseline.noise_floor_pct is None:

        return False

    return baseline.noise_floor_pct > _target_delta_pct(target)

def _baseline_denominators(baseline: BaselineCell) -> tuple[float, float] | None:

    low = baseline.ci_low if baseline.ci_low is not None else baseline.value

    high = baseline.ci_high if baseline.ci_high is not None else baseline.value

    if low <= 0 or high <= 0:

        return None

    return low, high

def _target_delta_pct(target: Target) -> float:

    # The margin the instrument must resolve is the distance from parity in

    # either direction: a goal of 1.5x demands the same 50% resolution a goal

    # of 0.5x does. A parity goal (1.0) has zero margin and is undecidable

    # whenever any noise floor exists.

    if target.goal_type == "ratio_at_most":

        return abs(1.0 - target.goal_value) * 100.0

    return 0.0

_number_or_none = number_or_none