#!/usr/bin/env python3

"""Summarize the biggest metric changes between two benchmark JSONL files.

The report is intentionally compact: it compares matching row identities,

aggregates metric values per subject/scenario/operation group, and prints the

largest improvements first. Positive percentage changes mean the new file is

better for lower-is-better metrics and worse for higher-is-better metrics.

"""

from __future__ import annotations

import argparse

import json

import math

import statistics

from pathlib import Path

from typing import Any

IDENTITY_FIELDS = (

    "lane",

    "profile",

    "scenario",

    "subject",

    "subject_kind",

    "benchmark_track",

    "instruction_level",

    "agent_environment",

    "transport",

    "remote_transport",

    "git_mode",

    "mode",

    "contention_mode",

    "workers",

    "platform",

    "driver",

    "protection_variant",

    "platform_comparison_key",

    "operation",

    "run",

    "phase",

    "workflow",

    "worker",

    "task_id",

)

DEFAULT_METRICS = (

    "elapsed_ms",

    "estimated_tokens_total",

    "estimated_tokens_input",

    "estimated_tokens_output",

    "estimated_cost_weighted_tokens",

    "raw_output_bytes",

    "probe_output_bytes",

    "tool_call_count",

    "vcs_tool_call_count",

    "terminal_tool_call_count",

    "peak_rss_bytes",

)

HIGHER_IS_BETTER = {

    "information_recall",

    "information_precision",

    "triage_recall",

    "triage_precision",

    "json_parse_success",

    "json_oracle_passed",

    "output_stable",

}

def parse_args() -> argparse.Namespace:

    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument("old", type=Path, help="Baseline JSONL")

    parser.add_argument("new", type=Path, help="Candidate JSONL")

    parser.add_argument("--limit", type=int, default=10, help="Maximum rows to print")

    parser.add_argument(

        "--metrics",

        default=",".join(DEFAULT_METRICS),

        help="Comma-separated metric fields to compare",

    )

    return parser.parse_args()

def load_rows(path: Path) -> list[dict[str, Any]]:

    rows: list[dict[str, Any]] = []

    for line_number, line in enumerate(path.read_text().splitlines(), 1):

        if not line.strip():

            continue

        try:

            row = json.loads(line)

        except json.JSONDecodeError as exc:

            raise SystemExit(f"{path}:{line_number}: invalid JSON: {exc}") from exc

        if isinstance(row, dict):

            rows.append(row)

    return rows

def identity_fields_for(old_rows: list[dict[str, Any]], new_rows: list[dict[str, Any]]) -> tuple[str, ...]:

    old_fields = {key for row in old_rows for key in row}

    new_fields = {key for row in new_rows for key in row}

    fields = tuple(field for field in IDENTITY_FIELDS if field in old_fields and field in new_fields)

    return fields or ("scenario", "subject", "operation", "run")

def identity(row: dict[str, Any], fields: tuple[str, ...]) -> tuple[str, ...]:

    return tuple(json.dumps(row.get(field), sort_keys=True) for field in fields)

def identity_display(row: dict[str, Any], fields: tuple[str, ...]) -> str:

    parts = [f"{field}={row.get(field)}" for field in fields if row.get(field) is not None]

    return " ".join(parts)

def index_rows(rows: list[dict[str, Any]], fields: tuple[str, ...]) -> dict[tuple[str, ...], dict[str, Any]]:

    indexed: dict[tuple[str, ...], dict[str, Any]] = {}

    for row in rows:

        indexed[identity(row, fields)] = row

    return indexed

def metric_value(row: dict[str, Any], metric: str) -> float | None:

    value = row.get(metric)

    if value is None or isinstance(value, bool):

        return None

    try:

        parsed = float(value)

    except (TypeError, ValueError):

        return None

    return parsed if math.isfinite(parsed) else None

def improvement_pct(metric: str, old: float, new: float) -> float | None:

    if old == 0:

        return None

    if metric in HIGHER_IS_BETTER:

        return ((new - old) / abs(old)) * 100.0

    return ((old - new) / abs(old)) * 100.0

def main() -> int:

    args = parse_args()

    old_path = args.old

    new_path = args.new

    for path in (old_path, new_path):

        if not path.exists():

            raise SystemExit(f"input not found: {path}")

    metrics = [item.strip() for item in args.metrics.split(",") if item.strip()]

    old_raw = load_rows(old_path)

    new_raw = load_rows(new_path)

    identity_fields = identity_fields_for(old_raw, new_raw)

    old_rows = index_rows(old_raw, identity_fields)

    new_rows = index_rows(new_raw, identity_fields)

    shared = [key for key in old_rows if key in new_rows]

    changes: list[dict[str, Any]] = []

    for key in shared:

        old_row = old_rows[key]

        new_row = new_rows[key]

        for metric in metrics:

            old_value = metric_value(old_row, metric)

            new_value = metric_value(new_row, metric)

            if old_value is None or new_value is None:

                continue

            pct = improvement_pct(metric, old_value, new_value)

            if pct is None:

                continue

            delta = new_value - old_value

            if delta == 0:

                continue

            changes.append(

                {

                    "pct": pct,

                    "delta": delta,

                    "metric": metric,

                    "identity": identity_display(old_row, identity_fields),

                    "old": old_value,

                    "new": new_value,

                }

            )

    improvements = sorted([item for item in changes if item["pct"] > 0], key=lambda item: item["pct"], reverse=True)

    regressions = sorted([item for item in changes if item["pct"] < 0], key=lambda item: item["pct"])

    print("# Result Delta Report")

    print()

    print(f"- Old: `{old_path}`")

    print(f"- New: `{new_path}`")

    print(f"- Shared row identities: {len(shared)}")

    print(f"- Identity fields: {', '.join(identity_fields)}")

    print(f"- Metrics compared: {', '.join(metrics)}")

    print()

    if old_raw and new_raw and not shared:

        print("No shared row identities; inputs may use incompatible scenario/subject/operation dimensions.")

        return 1

    def section(title: str, rows: list[dict[str, Any]]) -> None:

        print(f"## {title} ({len(rows)})")

        print()

        print("| Rank | Metric | Identity | Old | New | Delta | Improvement |")

        print("| --- | --- | --- | ---: | ---: | ---: | ---: |")

        for index, item in enumerate(rows[: args.limit], 1):

            print(

                f"| {index} | `{item['metric']}` | {item['identity']} | "

                f"{item['old']:.3f} | {item['new']:.3f} | {item['delta']:+.3f} | {item['pct']:+.1f}% |"

            )

        if not rows:

            print("| - | none | none |  |  |  |  |")

        print()

    section("Top Improvements", improvements)

    section("Top Regressions", regressions)

    return 0

if __name__ == "__main__":

    raise SystemExit(main())