#!/usr/bin/env python3

"""Re-derive public benchmark claims from raw JSONL evidence.

This is deliberately separate from publish_gate.py. publish_gate checks whether

rows are mechanically publishable; this script checks whether a specific public

number still matches the raw rows it cites.

"""

from __future__ import annotations

import argparse

import json

import statistics

import sys

from pathlib import Path

from typing import Any

ROOT = Path(__file__).resolve().parents[1]

sys.path.insert(0, str(ROOT / "scripts"))

from oakbench.rows import row_returncode  # noqa: E402

KNOWN_TRANSPORTS = {"network", "local_file", "network_shaped"}

TRANSPORT_FIELDS = ("remote_transport", "workspace_transport")

FILTER_OPERATORS = {"exists", "eq", "ne", "in"}

CLEANUP_FAILURE_FIELDS = ("cleanup_failures", "cleanup_failure_count")

class ClaimError(Exception):

    pass

def load_rows(paths: list[Path]) -> list[dict[str, Any]]:

    rows: list[dict[str, Any]] = []

    for path in paths:

        for line_no, line in enumerate(path.read_text().splitlines(), start=1):

            if not line.strip():

                continue

            try:

                row = json.loads(line)

            except json.JSONDecodeError as exc:

                raise ClaimError(f"{path}:{line_no}: invalid JSON: {exc}") from exc

            rows.append(row)

    return rows

def dotted_get(row: dict[str, Any], key: str) -> Any:

    cur: Any = row

    for part in key.split("."):

        if not isinstance(cur, dict) or part not in cur:

            return None

        cur = cur[part]

    return cur

def row_matches(row: dict[str, Any], filters: dict[str, Any]) -> bool:

    for key, expected in filters.items():

        actual = dotted_get(row, key)

        if isinstance(expected, list):

            if actual not in expected:

                return False

        elif isinstance(expected, dict):

            unknown = set(expected) - FILTER_OPERATORS

            if unknown:

                raise ClaimError(

                    f"invalid filter operator(s) for {key}: {sorted(unknown)}; "

                    f"known: {sorted(FILTER_OPERATORS)}"

                )

            if "exists" in expected and (actual is not None) is not bool(expected["exists"]):

                return False

            if "eq" in expected and actual != expected["eq"]:

                return False

            if "ne" in expected and actual == expected["ne"]:

                return False

            if "in" in expected and actual not in expected["in"]:

                return False

        elif actual != expected:

            return False

    return True

def number(value: Any) -> float | None:

    if isinstance(value, bool) or value is None:

        return None

    if isinstance(value, (int, float)):

        return float(value)

    return None

def metric_values(rows: list[dict[str, Any]], metric: str) -> list[float]:

    values = []

    for row in rows:

        value = number(dotted_get(row, metric))

        if value is not None:

            values.append(value)

    return values

def statistic(values: list[float], name: str) -> float:

    if name == "median":

        return float(statistics.median(values))

    if name == "mean":

        return float(statistics.mean(values))

    if name == "sum":

        return float(sum(values))

    if name == "min":

        return float(min(values))

    if name == "max":

        return float(max(values))

    if name == "count":

        return float(len(values))

    raise ClaimError(f"unsupported statistic: {name}")

def enforce_row_guards(claim: dict[str, Any], rows: list[dict[str, Any]], label: str) -> list[str]:

    errors: list[str] = []

    if not rows:

        errors.append(f"{label}: no matching rows")

        return errors

    guards = claim.get("guards") if isinstance(claim.get("guards"), dict) else {}

    require_success = bool(guards.get("require_success", True))

    if require_success:

        failed = [row for row in rows if row_returncode(row) != 0]

        if failed:

            preview = [(row.get("operation"), row.get("returncode"), row.get("skip_reason")) for row in failed[:5]]

            errors.append(f"{label}: {len(failed)} row(s) not successful: {preview}")

    min_rows = int(guards.get("min_rows", 1))

    if len(rows) < min_rows:

        errors.append(f"{label}: {len(rows)} row(s) below min_rows={min_rows}")

    if not bool(guards.get("allow_fake_provider", False)):

        fake = [

            row

            for row in rows

            if row.get("driver") == "fake-provider" or row.get("branch_triage_provider") == "fake"

        ]

        if fake:

            errors.append(f"{label}: fake-provider rows are not public claim evidence")

    if any("json_oracle_passed" in row for row in rows):

        bad = [row for row in rows if row.get("json_oracle_passed") is not True]

        if bad:

            errors.append(f"{label}: {len(bad)} row(s) have json_oracle_passed != true")

    for cleanup_field in CLEANUP_FAILURE_FIELDS:

        bad = [

            row

            for row in rows

            if cleanup_field in row and number(row.get(cleanup_field)) != 0.0

        ]

        if bad:

            errors.append(f"{label}: {len(bad)} row(s) have {cleanup_field} != 0")

    for field in guards.get("fields_true", []):

        bad = [row for row in rows if dotted_get(row, field) is not True]

        if bad:

            errors.append(f"{label}: {len(bad)} row(s) have {field} != true")

    for field in guards.get("fields_false", []):

        bad = [row for row in rows if dotted_get(row, field) is not False]

        if bad:

            errors.append(f"{label}: {len(bad)} row(s) have {field} != false")

    for field in guards.get("fields_zero", []):

        bad = [row for row in rows if number(dotted_get(row, field)) != 0.0]

        if bad:

            errors.append(f"{label}: {len(bad)} row(s) have {field} != 0")

    for field in guards.get("fields_present", []):

        bad = [row for row in rows if dotted_get(row, field) is None]

        if bad:

            errors.append(f"{label}: {len(bad)} row(s) are missing {field}")

    for item in guards.get("fields_equal", []):

        if not isinstance(item, dict) or "field" not in item or "value" not in item:

            errors.append(f"{label}: invalid fields_equal guard {item!r}")

            continue

        field = str(item["field"])

        expected = item["value"]

        bad = [row for row in rows if dotted_get(row, field) != expected]

        if bad:

            errors.append(f"{label}: {len(bad)} row(s) have {field} != {expected!r}")

    return errors

def row_transport(row: dict[str, Any]) -> str | None:

    for field in TRANSPORT_FIELDS:

        value = row.get(field)

        if value:

            return str(value)

    return None

def transport_identity(row: dict[str, Any]) -> tuple[str, str | None, str | None]:

    return (

        row_transport(row) or "<missing>",

        str(row.get("netshape_profile")) if row.get("netshape_profile") is not None else None,

        str(row.get("remote_server") or row.get("server") or "")

        or None,

    )

def transport_set(rows: list[dict[str, Any]]) -> set[tuple[str, str | None, str | None]]:

    return {transport_identity(row) for row in rows if transport_identity(row)[0] != "<missing>"}

def transport_errors(rows: list[dict[str, Any]], label: str) -> list[str]:

    errors: list[str] = []

    identities: set[tuple[str, str | None, str | None]] = set()

    for index, row in enumerate(rows, 1):

        transport = row_transport(row)

        if (row.get("remote_repo") or row.get("remote_server")) and transport is None:

            errors.append(f"{label}: row {index} remote-touching row missing transport")

        if transport is not None and transport not in KNOWN_TRANSPORTS:

            errors.append(

                f"{label}: row {index} unknown transport {transport!r}; known: {sorted(KNOWN_TRANSPORTS)}"

            )

        if transport == "network_shaped" and not row.get("netshape_profile"):

            errors.append(f"{label}: row {index} shaped transport missing netshape_profile")

        if transport is not None:

            identities.add(transport_identity(row))

    if len(identities) > 1:

        errors.append(f"{label}: matched rows mix transport identities: {sorted(identities)}")

    return errors

def metric_coverage_errors(rows: list[dict[str, Any]], metric: str, values: list[float], label: str) -> list[str]:

    successful = [row for row in rows if row_returncode(row) == 0]

    if len(values) != len(successful):

        return [

            f"{label}: metric {metric} measured on {len(values)} of "

            f"{len(successful)} successful row(s)"

        ]

    return []

def evaluate_claim(claim: dict[str, Any], rows: list[dict[str, Any]]) -> dict[str, Any]:

    claim_id = str(claim.get("id") or "<missing-id>")

    filters = claim.get("row_filter")

    if not isinstance(filters, dict):

        raise ClaimError(f"{claim_id}: row_filter is required")

    metric = str(claim.get("metric") or "")

    if not metric:

        raise ClaimError(f"{claim_id}: metric is required")

    stat_name = str(claim.get("statistic") or "median")

    expected = claim.get("expected")

    if not isinstance(expected, dict):

        raise ClaimError(f"{claim_id}: expected is required")

    matched = [row for row in rows if row_matches(row, filters)]

    errors = enforce_row_guards(claim, matched, "candidate")

    errors.extend(transport_errors(matched, "candidate"))

    values = metric_values(matched, metric)

    errors.extend(metric_coverage_errors(matched, metric, values, "candidate"))

    if not values:

        errors.append(f"candidate: metric {metric} unmeasured")

        candidate_value = None

    else:

        candidate_value = statistic(values, stat_name)

    baseline_value = None

    ratio = None

    delta_pct = None

    reduction_pct = None

    baseline_filter = claim.get("baseline_filter")

    if isinstance(baseline_filter, dict):

        baseline = [row for row in rows if row_matches(row, baseline_filter)]

        errors.extend(enforce_row_guards(claim, baseline, "baseline"))

        errors.extend(transport_errors(baseline, "baseline"))

        baseline_values = metric_values(baseline, metric)

        errors.extend(metric_coverage_errors(baseline, metric, baseline_values, "baseline"))

        if not baseline_values:

            errors.append(f"baseline: metric {metric} unmeasured")

        else:

            baseline_value = statistic(baseline_values, stat_name)

            if baseline_value == 0:

                errors.append("baseline: value is zero; ratio/delta undefined")

            elif candidate_value is not None:

                ratio = candidate_value / baseline_value

                delta_pct = (candidate_value - baseline_value) / baseline_value * 100.0

                reduction_pct = (baseline_value - candidate_value) / baseline_value * 100.0

        if bool(claim.get("require_same_transport", True)):

            candidate_transport = transport_set(matched)

            baseline_transport = transport_set(baseline)

            if candidate_transport or baseline_transport:

                if candidate_transport != baseline_transport:

                    errors.append(

                        "transport mismatch: "

                        f"candidate={sorted(candidate_transport)} baseline={sorted(baseline_transport)}"

                    )

    actuals = {

        "value": candidate_value,

        "baseline_value": baseline_value,

        "ratio": ratio,

        "delta_pct": delta_pct,

        "reduction_pct": reduction_pct,

    }

    field = str(expected.get("field") or "value")

    actual = actuals.get(field)

    if actual is None:

        errors.append(f"expected field {field} is unavailable")

    else:

        if "value" in expected:

            target = float(expected["value"])

            tolerance_pct = float(expected.get("tolerance_pct", 0.0))

            allowed = abs(target) * tolerance_pct / 100.0

            if abs(actual - target) > allowed:

                errors.append(

                    f"{field} drifted: actual={actual:.6g} expected={target:.6g} "

                    f"tolerance_pct={tolerance_pct:.3g}"

                )

        if "min" in expected and actual < float(expected["min"]):

            errors.append(f"{field} below min: actual={actual:.6g} min={float(expected['min']):.6g}")

        if "max" in expected and actual > float(expected["max"]):

            errors.append(f"{field} above max: actual={actual:.6g} max={float(expected['max']):.6g}")

    return {

        "id": claim_id,

        "description": claim.get("description"),

        "metric": metric,

        "statistic": stat_name,

        "n": len(values),

        "matched_rows": len(matched),

        **actuals,

        "expected": expected,

        "verdict": "PASS" if not errors else "FAIL",

        "errors": errors,

    }

def load_spec(path: Path) -> list[dict[str, Any]]:

    spec = json.loads(path.read_text())

    if spec.get("schema_version") != 1:

        raise ClaimError(f"{path}: unsupported schema_version {spec.get('schema_version')!r}")

    claims = spec.get("claims")

    if not isinstance(claims, list):

        raise ClaimError(f"{path}: claims must be a list")

    return claims

def parse_args() -> argparse.Namespace:

    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument("jsonl", nargs="+", type=Path, help="raw evidence JSONL")

    parser.add_argument("--claims", required=True, type=Path, help="claim spec JSON")

    parser.add_argument("--json", action="store_true", help="emit JSON results")

    return parser.parse_args()

def main() -> int:

    args = parse_args()

    try:

        rows = load_rows(args.jsonl)

        results = [evaluate_claim(claim, rows) for claim in load_spec(args.claims)]

    except ClaimError as exc:

        print(f"FAIL   {exc}", file=sys.stderr)

        return 2

    if args.json:

        print(json.dumps({"schema_version": 1, "claims": results}, indent=2, sort_keys=True))

    else:

        for result in results:

            print(f"{result['verdict']}   {result['id']}")

            print(

                "       "

                f"value={result.get('value')} baseline={result.get('baseline_value')} "

                f"ratio={result.get('ratio')} reduction_pct={result.get('reduction_pct')}"

            )

            for error in result["errors"]:

                print(f"       {error}")

    return 1 if any(result["verdict"] != "PASS" for result in results) else 0

if __name__ == "__main__":

    raise SystemExit(main())