"""Target scorecard evaluation."""

from __future__ import annotations

from typing import Any

import math

from oakbench.baseline_book import BaselineCell, Target, iter_jsonl, metric_value

from oakbench.integrity import content_integrity_public_trust_passed

from oakbench.rows import number_or_none, row_returncode

from oakbench.populations import (population_conflicts, comparison_identity, comparison_conflicts,

                                 independent_run_count, missing_claim_identity)

from oakbench.stats import bootstrap_ci, named_statistic, required_n_for_relative_delta, validate_interval_settings

VERDICTS = ("GREEN", "RED", "AMBER", "UNDECIDABLE", "UNMEASURED", "INVALID")

# One JSONL loader for scorecard inputs; baseline_book owns the implementation.

load_jsonl = iter_jsonl

def evaluate_targets(

    rows: list[dict[str, Any]],

    targets: list[Target],

    baseline: dict[tuple[str, str | None, str, str, str, str, str], BaselineCell],

    *,

    bootstrap_samples: int = 2000,

    confidence: float = 0.95,

    seed: int = 20260609,

    mode: str = "diagnostic",

) -> list[dict[str, Any]]:

    validate_interval_settings(bootstrap_samples, confidence)

    if mode not in {"diagnostic", "claim"}:

        raise ValueError("mode must be diagnostic or claim")

    return [

        evaluate_target(

            rows,

            target,

            baseline.get(target.baseline_key),

            bootstrap_samples=bootstrap_samples,

            confidence=confidence,

            seed=seed,

            mode=mode,

        )

        for target in targets

    ]

def evaluate_target(

    rows: list[dict[str, Any]],

    target: Target,

    baseline: BaselineCell | None,

    *,

    bootstrap_samples: int,

    confidence: float,

    seed: int,

    mode: str = "diagnostic",

) -> dict[str, Any]:

    matched = [row for row in rows if _row_matches(row, target)]

    skipped = [row for row in matched if row_returncode(row) == 77]

    successful = [row for row in matched if row_returncode(row) == 0]

    failures = [row for row in matched if row_returncode(row) not in (0, 77)]

    values = [value for row in successful if (value := metric_value(row, target.metric)) is not None]

    base = {

        "target_id": target.target_id,

        "verdict": "UNMEASURED",

        "lane": target.lane,

        "profile": target.profile,

        "scenario": target.scenario,

        "operation": target.operation,

        "subject": target.subject,

        "comparator": target.comparator,

        "metric": target.metric,

        "goal_type": target.goal_type,

        "goal_value": target.goal_value,

        # "n" is the number of rows that actually contributed to the statistic

        # (metric measured, per ADR-0002 null-means-unmeasured); "n_successful"

        # is the pool it was drawn from. CI consumers should read "n".

        "n": len(values),

        "n_successful": len(successful),

        "skip_count": len(skipped),

        "failure_count": len(failures),

        "messages": [],

        "mode": mode,

        "claim_eligible": False,

        "independent_n": independent_run_count(successful),

        "confidence": confidence,

        "bootstrap_samples": bootstrap_samples,

    }

    if not matched:

        base["messages"].append("no matching rows")

        return base

    if skipped and not successful and not failures:

        base["messages"].append("; ".join(sorted({str(row.get("skip_reason") or "skipped") for row in skipped})))

        return base

    if baseline is None:

        base["messages"].append("missing baseline cell")

        return base

    guard_errors = guard_violations(matched, successful, failures, target.guards)

    # Outcome denominators are part of the experiment, not a separate identity

    # population. Failed/skipped trials cannot silently come from another host.

    conflicts = population_conflicts(matched)

    if conflicts:

        guard_errors.append(f"incompatible measurement population: {', '.join(conflicts)}")

    candidate_identity = comparison_identity(successful[0]) if successful else None

    if baseline.comparison_identity is not None and candidate_identity is not None:

        mismatch = comparison_conflicts(baseline.comparison_identity, candidate_identity)

        if mismatch:

            guard_errors.append(f"baseline comparison identity mismatch: {', '.join(mismatch)}")

    if guard_errors:

        base.update({"verdict": "INVALID", "messages": guard_errors})

        return base

    if not values:

        base["messages"].append("metric unmeasured in matching successful rows")

        return base

    eligibility_errors = claim_eligibility_errors(successful, matched, target, baseline, confidence)

    base["claim_ineligibility_reasons"] = eligibility_errors

    if mode == "claim" and eligibility_errors:

        base.update(verdict="AMBER", messages=eligibility_errors)

        return base

    base["claim_eligible"] = mode == "claim"

    if mode == "diagnostic":

        base["messages"].append("diagnostic only; not eligible for public claims")

    unmeasured = len(successful) - len(values)

    if unmeasured:

        base["messages"].append(f"metric unmeasured on {unmeasured} of {len(successful)} successful rows")

    if baseline.value == 0:

        base.update({"verdict": "UNDECIDABLE", "messages": ["baseline value is zero; ratio target is undefined"]})

        return base

    observed = statistic_value(values, target.statistic)

    ci_low, ci_high = bootstrap_ci(

        values,

        statistic=target.statistic,

        samples=bootstrap_samples,

        confidence=confidence,

        seed=seed,

    )

    baseline_denominators = _baseline_denominators(baseline)

    if baseline_denominators is None:

        base.update(

            {

                "verdict": "UNDECIDABLE",

                "messages": ["baseline confidence interval crosses zero; ratio target is undefined"],

            }

        )

        return base

    baseline_low, baseline_high = baseline_denominators

    ratio = observed / baseline.value

    ratio_low = ci_low / baseline_high

    ratio_high = ci_high / baseline_low

    base.update(

        {

            "value": observed,

            "ci_low": ci_low,

            "ci_high": ci_high,

            "baseline_value": baseline.value,

            "baseline_ci_low": baseline.ci_low,

            "baseline_ci_high": baseline.ci_high,

            "ratio": ratio,

            "ratio_ci_low": ratio_low,

            "ratio_ci_high": ratio_high,

        }

    )

    # GREEN/RED must clear the goal by more than the baseline noise floor:

    # the bootstrap CI only sees within-sample variance, so a narrow CI that

    # lands inside the noise band is instrument luck, not a verdict (ADR-0008:

    # a target can never out-resolve the instrument). Decisive results — CI

    # bounds that survive noise inflation — stay GREEN/RED.

    noise = float(baseline.noise_floor_pct or 0.0) / 100.0

    decisively_green = ratio_high * (1.0 + noise) <= target.goal_value

    decisively_red = ratio_low * (1.0 - noise) > target.goal_value

    # Partial metric coverage caps decisive verdicts at AMBER (ADR-0008): the

    # statistic only saw the rows where the metric was measured, so a decisive

    # bound may be an artifact of which rows happened to record it. Dropping

    # information can never show green — and, symmetrically, missing data

    # cannot fail a target either, so RED is capped to AMBER as well.

    if (decisively_green or decisively_red) and unmeasured:

        required = required_n_for_relative_delta(float(baseline.noise_floor_pct or 0.0), _target_delta_pct(target), len(values))

        base.update({"verdict": "AMBER", "required_n": required})

    elif decisively_green:

        base["verdict"] = "GREEN"

    elif decisively_red:

        base["verdict"] = "RED"

    elif _noise_makes_undecidable(target, baseline):

        required = required_n_for_relative_delta(float(baseline.noise_floor_pct or 0.0), _target_delta_pct(target), len(values))

        base.update(

            {

                "verdict": "UNDECIDABLE",

                "required_n": required,

                "messages": ["target is tighter than the baseline noise floor"],

            }

        )

    else:

        required = required_n_for_relative_delta(float(baseline.noise_floor_pct or 0.0), _target_delta_pct(target), len(values))

        base.update({"verdict": "AMBER", "required_n": required})

        base["messages"].append("confidence interval straddles target")

    return base

def claim_eligibility_errors(successful, matched, target, baseline, confidence) -> list[str]:

    errors = []

    if confidence < 0.95:

        errors.append("candidate confidence must be at least 0.95 for claims")

    if (baseline.confidence is None or not math.isfinite(baseline.confidence) or

            not 0.95 <= baseline.confidence < 1):

        errors.append("recorded baseline confidence must be at least 0.95 and below 1 for claims")

    measured = [row for row in successful if metric_value(row, target.metric) is not None]

    # n=30 is a conservative eligibility floor, not a precision guarantee.

    minimum = max(30, {"p95": 20, "p99": 100}.get(target.statistic, 0))

    n = independent_run_count(measured)

    if n is None or n < minimum:

        errors.append(f"candidate requires at least {minimum} distinct recorded trials; observed {n}")

    if n is not None and n != len(measured):

        errors.append("candidate contains duplicate trial IDs")

    if baseline.independent_n is None or baseline.independent_n < minimum:

        errors.append(f"baseline requires at least {minimum} distinct recorded trials")

    if baseline.independent_n != baseline.n:

        errors.append("baseline sample count differs from distinct trial count")

    if len(measured) != len(matched):

        errors.append("candidate contains failed, skipped or unmeasured trials")

    if (baseline.failure_n != 0 or baseline.skipped_n != 0 or

            baseline.attempted_n != baseline.n):

        errors.append("baseline outcomes missing or contain failed, skipped or unmeasured trials")

    noise = baseline.noise_floor_pct

    if noise is None or not math.isfinite(noise) or noise < 0:

        errors.append("measured finite nonnegative baseline noise floor required")

    if (baseline.ci_low is None or baseline.ci_high is None or

            not all(math.isfinite(x) for x in (baseline.ci_low, baseline.ci_high)) or

            not baseline.ci_low <= baseline.value <= baseline.ci_high):

        errors.append("valid measured baseline confidence interval required")

    identities = [("candidate", comparison_identity(successful[0]))] if successful else []

    if baseline.comparison_identity is None:

        errors.append("versioned baseline comparison identity required")

    else:

        identities.append(("baseline", baseline.comparison_identity))

    for label, identity in identities:

        missing = missing_claim_identity(identity, target.lane, target.metric, target.operation)

        if any(name in target.guards for name in ("information_recall_at_least", "information_precision_at_least")):

            if not identity.get("output_oracle_version"):

                missing.append("output_oracle_version")

        if target.guards.get("patch_apply_ok") and not identity.get("patch_oracle_version"):

            missing.append("patch_oracle_version")

        if missing:

            errors.append(f"{label} claim identity missing: {', '.join(missing)}")

    return errors

def statistic_value(values: list[float], statistic: str) -> float:

    return float(named_statistic(statistic)(values))

def guard_violations(

    matched: list[dict[str, Any]],

    successful: list[dict[str, Any]],

    failures: list[dict[str, Any]],

    guards: dict[str, Any],

) -> list[str]:

    errors: list[str] = []

    if failures:

        errors.append(f"{len(failures)} matching rows failed")

    if "success_rate_at_least" in guards:

        success_rate = len(successful) / len(matched) if matched else 0.0

        if success_rate < float(guards["success_rate_at_least"]):

            errors.append(f"success_rate {success_rate:.3f} below guard {float(guards['success_rate_at_least']):.3f}")

    if "information_recall_at_least" in guards:

        _minimum_guard(successful, "information_recall", float(guards["information_recall_at_least"]), errors)

    if "information_precision_at_least" in guards:

        _minimum_guard(successful, "information_precision", float(guards["information_precision_at_least"]), errors)

    if guards.get("pipe_compatible_unified") is True:

        missing = [row for row in successful if row.get("pipe_compatible_unified") is None]

        bad = [row for row in successful if row.get("pipe_compatible_unified") is False]

        if missing:

            errors.append(f"{len(missing)} rows have unmeasured pipe_compatible_unified guard")

        if bad:

            errors.append(f"{len(bad)} rows failed pipe_compatible_unified guard")

    if guards.get("patch_apply_ok") is True:

        if any(row.get("patch_apply_ok") is not True for row in successful):

            errors.append("patch_apply_ok guard is failed or unmeasured")

    if guards.get("content_integrity_public_trust") is True:

        missing = [row for row in successful if row.get("content_integrity_check_passed") is None]

        failed = [row for row in successful if row.get("content_integrity_check_passed") is False]

        weak = [

            row

            for row in successful

            if row.get("content_integrity_check_passed") is True

            and not content_integrity_public_trust_passed(row)

        ]

        if missing:

            errors.append(f"{len(missing)} rows have unmeasured content_integrity_public_trust guard")

        if failed:

            errors.append(f"{len(failed)} rows failed content_integrity_public_trust guard")

        if weak:

            errors.append(f"{len(weak)} rows have weak content_integrity_source for public trust")

    return errors

def _minimum_guard(rows: list[dict[str, Any]], key: str, minimum: float, errors: list[str]) -> None:

    if not rows:

        errors.append(f"{key} guard has no successful observations")

        return

    values = [value for row in rows if (value := _number_or_none(row.get(key))) is not None]

    if len(values) != len(rows):

        errors.append(f"{key} guard is unmeasured on {len(rows) - len(values)} rows")

    elif min(values) < minimum:

        errors.append(f"{key} minimum {min(values):.3f} below guard {minimum:.3f}")

def _row_matches(row: dict[str, Any], target: Target) -> bool:

    lane = _row_lane(row)

    return (

        lane == target.lane

        and str(row.get("subject")) == target.subject

        and str(row.get("scenario")) == target.scenario

        and str(row.get("operation")) == target.operation

        and (

            (target.profile is None and row.get("profile") is None)

            or (target.profile is not None and str(row.get("profile")) == target.profile)

        )

    )

def _row_lane(row: dict[str, Any]) -> str | None:

    for key in ("lane", "benchmark_lane", "measurement_lane"):

        if row.get(key) is not None:

            return str(row[key])

    if "workflow" in row or "phase" in row:

        return "workflow"

    if all(key in row for key in ("token_metrics", "turn_metrics", "tool_call_metrics")):

        return "agent"

    if "contention_mode" in row or "workers" in row:

        return "contention"

    if str(row.get("scenario") or "") == "task_loop":

        return "task-loop"

    if row.get("subject_kind") is None and row.get("subject") is not None:

        return "mount"

    if all(key in row for key in ("profile", "scenario", "operation", "subject", "subject_kind", "command")):

        return "core"

    return None

def _noise_makes_undecidable(target: Target, baseline: BaselineCell) -> bool:

    if baseline.noise_floor_pct is None:

        return False

    return baseline.noise_floor_pct > _target_delta_pct(target)

def _baseline_denominators(baseline: BaselineCell) -> tuple[float, float] | None:

    low = baseline.ci_low if baseline.ci_low is not None else baseline.value

    high = baseline.ci_high if baseline.ci_high is not None else baseline.value

    if low <= 0 or high <= 0:

        return None

    return low, high

def _target_delta_pct(target: Target) -> float:

    # The margin the instrument must resolve is the distance from parity in

    # either direction: a goal of 1.5x demands the same 50% resolution a goal

    # of 0.5x does. A parity goal (1.0) has zero margin and is undecidable

    # whenever any noise floor exists.

    if target.goal_type == "ratio_at_most":

        return abs(1.0 - target.goal_value) * 100.0

    return 0.0

_number_or_none = number_or_none