#!/usr/bin/env python3

"""Print a ranked, evidence-linked attack list from the latest benchmark rows.

Standing orders for the Oak improvement swarm: instead of every agent

re-deriving priorities from raw JSONL, this script turns the latest per-lane

results into one ranked opportunity report.

Sections, in order:

1. Where Oak loses to Git — per-(scenario, operation) median gaps on

   elapsed_ms, cost-weighted tokens, output bytes, and tool calls, scored by

   pct-worse times a documented operation-family frequency weight.

2. Absolute quality bar — output-quality violations independent of Git:

   broken pipe compatibility, information recall below 1.0, unstable output,

   ANSI escapes in non-TTY output. Oak subjects only.

3. Oak-only trends — remote.net.* and mount-lane medians with no Git

   comparator; the goal is making them smaller release over release.

4. Unmeasured — skip rows grouped by skip_reason: coverage to unlock.

5. Low confidence — every line above with sample count n < 3, repeated.

Honesty rules (ADR-0002): skip rows (returncode 77 / skipped: true) never

enter medians or section 1-3 counts; null metric values are unmeasured, never

zero; the header states which input files were read and which were missing.

Output is deterministic: the same inputs produce a byte-identical report.

"""

from __future__ import annotations

import argparse

import json

import sys

from dataclasses import dataclass

from pathlib import Path

from typing import Any, Callable, Optional

from oakbench.reporting import median

from oakbench.rows import (

    number_or_none,

    output_bytes as row_output_bytes,

    tool_calls as row_tool_calls,

)

SKIP_RETURNCODE = 77

LOW_N_THRESHOLD = 3

# Input files, all optional: a missing file is reported as "lane not run".

LANE_FILES = (

    ("core", "latest.jsonl"),

    ("mount", "latest.mount.jsonl"),

    ("contention", "parallel-contention/latest.jsonl"),

)

# How often agent fleets run each operation family. Static and documented on

# purpose: opportunity scores must be reproducible from the rows alone, not

# from telemetry that moves between runs. Keys match either the full operation

# name or any dotted component of it; the highest matching weight wins.

FREQUENCY_WEIGHTS = {

    "status": 10,

    "diff": 10,

    "commit": 10,

    "snapshot": 10,

    "proc.spawn": 10,

    "branch": 5,

    "log": 5,

    "show": 5,

    "init": 2,

    "clone": 2,

    "push": 2,

    "pull": 2,

    "fetch": 2,

}

DEFAULT_FREQUENCY_WEIGHT = 1

def frequency_weight(operation: str) -> int:

    if operation in FREQUENCY_WEIGHTS:

        return FREQUENCY_WEIGHTS[operation]

    weight = DEFAULT_FREQUENCY_WEIGHT

    for part in operation.split("."):

        weight = max(weight, FREQUENCY_WEIGHTS.get(part, DEFAULT_FREQUENCY_WEIGHT))

    return weight

def cost_weighted_tokens(row: dict[str, Any]) -> Optional[float]:

    # The envelope-priced total is the truer agent cost (tool-use/tool-result

    # framing per call); prefer it when the row carries it.

    value = number_or_none(row.get("estimated_cost_weighted_tokens_with_envelope"))

    if value is not None:

        return value

    return number_or_none(row.get("estimated_cost_weighted_tokens"))

# (metric name, per-row extractor, display precision). Extractors return None

# for unmeasured rows, which are excluded from medians, never treated as 0.

METRICS: tuple[tuple[str, Callable[[dict[str, Any]], Optional[float]], int], ...] = (

    ("elapsed_ms", lambda row: number_or_none(row.get("elapsed_ms")), 1),

    ("estimated_cost_weighted_tokens", cost_weighted_tokens, 1),

    ("raw_output_bytes", row_output_bytes, 0),

    ("tool_call_count", row_tool_calls, 2),

)

METRIC_PRECISION = {name: precision for name, _, precision in METRICS}

# Known skip reasons mapped to what would unlock the measurement. Unknown

# reasons fall back to the reason text itself, which producers are required

# to make actionable (rows.py rejects empty skip_reason).

SKIP_REMEDIES = {

    "mount_not_started": (

        "make `oak mount` start on this host (macOS FSKit app enabled / Linux fuse3 installed); "

        "every in-mount measurement is gated on a successful mount.start"

    ),

    "requires_external_offline_harness": (

        "build the offline network harness so offline-after-partial-hydration can run"

    ),

    "huge_file_path_not_configured": (

        "configure the huge-file fixture path in scenarios/mount.yaml for the partial-read scenario"

    ),

}

DEFAULT_SKIP_REMEDY = "provide the capability described by the skip reason"

@dataclass(frozen=True)

class Opportunity:

    section: str

    scenario: Optional[str]

    operation: Optional[str]

    metric: str

    subject: Optional[str]

    value: Optional[float]

    baseline_value: Optional[float]

    pct_gap: Optional[float]

    score: Optional[float]

    n: int

    evidence_file: str

    detail: str = ""

    @property

    def low_n(self) -> bool:

        # Skip-row counts are exact tallies, not samples; low-n applies to

        # the median-based sections only.

        return self.section != "unmeasured" and self.n < LOW_N_THRESHOLD

@dataclass

class Lane:

    name: str

    path: Path

    missing: bool

    row_count: int = 0

    # (scenario, operation, subject) -> successful rows only

    success: Optional[dict[tuple[str, str, str], list[dict[str, Any]]]] = None

    skips: Optional[list[dict[str, Any]]] = None

def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:

    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])

    parser.add_argument(

        "jsonl",

        nargs="*",

        type=Path,

        help=(

            "Explicit benchmark JSONL files to read. When omitted, reads the "

            "standard latest lane files under --results."

        ),

    )

    parser.add_argument(

        "--results",

        type=Path,

        default=Path(__file__).resolve().parent.parent / "results",

        help="Results directory for default latest lane files (default: the repository's results/)",

    )

    parser.add_argument("--json", type=Path, help="Also write machine-readable JSON to this path")

    parser.add_argument("--git-subject", default="git")

    parser.add_argument(

        "--oak-baseline",

        help="Default: oak_main when present, otherwise oak_installed",

    )

    return parser.parse_args(argv)

def is_skip(row: dict[str, Any]) -> bool:

    return row.get("returncode") == SKIP_RETURNCODE or bool(row.get("skipped"))

def is_oak_subject(subject: str, rows: list[dict[str, Any]]) -> bool:

    kind = rows[0].get("subject_kind")

    if isinstance(kind, str) and kind:

        return kind == "oak"

    return subject.startswith("oak")

def load_lane(name: str, path: Path) -> Lane:

    if not path.exists():

        return Lane(name=name, path=path, missing=True)

    success: dict[tuple[str, str, str], list[dict[str, Any]]] = {}

    skips: list[dict[str, Any]] = []

    row_count = 0

    with path.open() as fh:

        for line_number, raw_line in enumerate(fh, start=1):

            line = raw_line.strip()

            if not line:

                continue

            try:

                row = json.loads(line)

            except json.JSONDecodeError as exc:

                raise SystemExit(f"{path}:{line_number}: invalid JSON: {exc}") from exc

            if not isinstance(row, dict):

                raise SystemExit(f"{path}:{line_number}: expected a JSON object")

            row_count += 1

            if is_skip(row):

                skips.append(row)

                continue

            if row.get("returncode") != 0:

                continue

            subject = str(row.get("subject") or "")

            scenario = str(row.get("scenario") or "")

            operation = str(row.get("operation") or "")

            if not subject or not scenario or not operation:

                continue

            success.setdefault((scenario, operation, subject), []).append(row)

    return Lane(name=name, path=path, missing=False, row_count=row_count, success=success, skips=skips)

def lane_name_for_path(path: Path, index: int) -> str:

    parts = path.parts

    if path.name == "latest.mount.jsonl":

        return "mount"

    if "parallel-contention" in parts:

        return "contention"

    if path.name == "latest.jsonl":

        return "core" if index == 0 else f"input-{index + 1}"

    return path.stem or f"input-{index + 1}"

def load_lanes(results_dir: Path, input_paths: Optional[list[Path]] = None) -> list[Lane]:

    if input_paths:

        return [load_lane(lane_name_for_path(path, index), path) for index, path in enumerate(input_paths)]

    return [load_lane(name, results_dir / rel) for name, rel in LANE_FILES]

def choose_oak_baseline(lanes: list[Lane], explicit: Optional[str]) -> Optional[str]:

    subjects = {

        subject

        for lane in lanes

        if not lane.missing

        for (_, _, subject) in lane.success

    }

    if explicit:

        return explicit if explicit in subjects else None

    if "oak_main" in subjects:

        return "oak_main"

    if "oak_installed" in subjects:

        return "oak_installed"

    return None

def metric_samples(

    rows: list[dict[str, Any]],

    extract: Callable[[dict[str, Any]], Optional[float]],

) -> list[float]:

    return [value for value in (extract(row) for row in rows) if value is not None]

def build_losses(lanes: list[Lane], git_subject: str, oak_baseline: Optional[str]) -> list[Opportunity]:

    """Section 1: scored per-(scenario, operation, metric) gaps vs Git."""

    if oak_baseline is None:

        return []

    entries: list[Opportunity] = []

    for lane in lanes:

        if lane.missing:

            continue

        scenario_ops = sorted({(scenario, op) for (scenario, op, _) in lane.success})

        for scenario, operation in scenario_ops:

            git_rows = lane.success.get((scenario, operation, git_subject))

            oak_rows = lane.success.get((scenario, operation, oak_baseline))

            if not git_rows or not oak_rows:

                continue

            for metric, extract, _ in METRICS:

                git_values = metric_samples(git_rows, extract)

                oak_values = metric_samples(oak_rows, extract)

                git_median = median(git_values)

                oak_median = median(oak_values)

                if git_median is None or oak_median is None or git_median <= 0:

                    continue

                if oak_median <= git_median:

                    continue

                pct_gap = ((oak_median - git_median) / git_median) * 100.0

                weight = frequency_weight(operation)

                entries.append(

                    Opportunity(

                        section="losses",

                        scenario=scenario,

                        operation=operation,

                        metric=metric,

                        subject=oak_baseline,

                        value=oak_median,

                        baseline_value=git_median,

                        pct_gap=pct_gap,

                        score=pct_gap * weight,

                        n=min(len(git_values), len(oak_values)),

                        evidence_file=str(lane.path),

                        detail=f"weight {weight}",

                    )

                )

    entries.sort(key=lambda e: (-e.score, e.scenario, e.operation, e.metric))

    return entries

def build_quality_bar(lanes: list[Lane]) -> list[Opportunity]:

    """Section 2: quality violations on Oak subjects, independent of Git."""

    entries: list[Opportunity] = []

    for lane in lanes:

        if lane.missing:

            continue

        for (scenario, operation, subject), rows in sorted(lane.success.items()):

            if not is_oak_subject(subject, rows):

                continue

            recalls = [

                float(value)

                for value in (row.get("information_recall") for row in rows)

                if isinstance(value, (int, float)) and not isinstance(value, bool)

            ]

            precisions = [

                float(value)

                for value in (row.get("information_precision") for row in rows)

                if isinstance(value, (int, float)) and not isinstance(value, bool)

            ]

            pipes = [value for value in (row.get("pipe_compatible_unified") for row in rows) if isinstance(value, bool)]

            stables = [value for value in (row.get("output_stable") for row in rows) if isinstance(value, bool)]

            ansis = [

                int(value)

                for value in (row.get("ansi_escape_count_non_tty") for row in rows)

                if isinstance(value, int) and not isinstance(value, bool)

            ]

            def add(metric: str, value: float, n: int, detail: str) -> None:

                entries.append(

                    Opportunity(

                        section="quality",

                        scenario=scenario,

                        operation=operation,

                        metric=metric,

                        subject=subject,

                        value=value,

                        baseline_value=None,

                        pct_gap=None,

                        score=None,

                        n=n,

                        evidence_file=str(lane.path),

                        detail=detail,

                    )

                )

            if recalls and min(recalls) < 1.0:

                add(

                    "information_recall",

                    min(recalls),

                    len(recalls),

                    "output drops changed-path names; bytes saved by losing information are not a win",

                )

            if precisions and min(precisions) < 1.0:

                add(

                    "information_precision",

                    min(precisions),

                    len(precisions),

                    "output names paths outside the changed set; recall without precision can reward noisy dumps",

                )

            if pipes and not all(pipes):

                rate = sum(1.0 for value in pipes if value) / len(pipes)

                add(

                    "pipe_compatible_unified",

                    rate,

                    len(pipes),

                    "unified-diff structure broken for `patch`-style piped tooling",

                )

            if stables and not all(stables):

                rate = sum(1.0 for value in stables if value) / len(stables)

                add(

                    "output_stable",

                    rate,

                    len(stables),

                    "identical state produces different bytes; provider prompt caches miss",

                )

            if ansis and max(ansis) > 0:

                add(

                    "ansi_escape_count_non_tty",

                    float(max(ansis)),

                    len(ansis),

                    "ANSI escapes in piped output are pure token waste",

                )

    entries.sort(key=lambda e: (e.scenario, e.operation, e.metric, e.subject))

    return entries

def build_trends(lanes: list[Lane]) -> list[Opportunity]:

    """Section 3: Oak-only medians (remote.net.* and the mount lane)."""

    entries: list[Opportunity] = []

    for lane in lanes:

        if lane.missing:

            continue

        for (scenario, operation, subject), rows in sorted(lane.success.items()):

            if not is_oak_subject(subject, rows):

                continue

            if lane.name != "mount" and not operation.startswith("remote.net."):

                continue

            for metric, extract, _ in METRICS:

                values = metric_samples(rows, extract)

                value = median(values)

                if value is None:

                    continue

                entries.append(

                    Opportunity(

                        section="trends",

                        scenario=scenario,

                        operation=operation,

                        metric=metric,

                        subject=subject,

                        value=value,

                        baseline_value=None,

                        pct_gap=None,

                        score=None,

                        n=len(values),

                        evidence_file=str(lane.path),

                    )

                )

    entries.sort(key=lambda e: (e.scenario, e.operation, e.metric, e.subject))

    return entries

def build_unmeasured(lanes: list[Lane]) -> list[Opportunity]:

    """Section 4: skip rows grouped by skip_reason — coverage to unlock."""

    groups: dict[str, dict[str, Any]] = {}

    for lane in lanes:

        if lane.missing:

            continue

        for row in lane.skips:

            reason = str(row.get("skip_reason") or "unspecified")

            group = groups.setdefault(

                reason, {"count": 0, "lanes": set(), "ops": set(), "files": set()}

            )

            group["count"] += 1

            group["lanes"].add(lane.name)

            group["ops"].add(f"{row.get('scenario')}/{row.get('operation')}")

            group["files"].add(str(lane.path))

    entries: list[Opportunity] = []

    for reason in sorted(groups, key=lambda r: (-groups[r]["count"], r)):

        group = groups[reason]

        remedy = SKIP_REMEDIES.get(reason, DEFAULT_SKIP_REMEDY)

        detail = (

            f"lanes: {', '.join(sorted(group['lanes']))}; "

            f"operations: {', '.join(sorted(group['ops']))}; "

            f"unlock: {remedy}"

        )

        entries.append(

            Opportunity(

                section="unmeasured",

                scenario=None,

                operation=None,

                metric=f"skip:{reason}",

                subject=None,

                value=float(group["count"]),

                baseline_value=None,

                pct_gap=None,

                score=None,

                n=group["count"],

                evidence_file=", ".join(sorted(group["files"])),

                detail=detail,

            )

        )

    return entries

def fmt_value(metric: str, value: Optional[float]) -> str:

    if value is None:

        return "unmeasured"

    precision = METRIC_PRECISION.get(metric, 2)

    return f"{value:.{precision}f}"

def n_suffix(entry: Opportunity) -> str:

    return f"(n={entry.n}, low-n)" if entry.low_n else f"(n={entry.n})"

def loss_line(rank: int, entry: Opportunity) -> str:

    return (

        f"{rank}. `{entry.scenario}/{entry.operation}` — `{entry.metric}`: "

        f"{entry.subject} {fmt_value(entry.metric, entry.value)} vs git "

        f"{fmt_value(entry.metric, entry.baseline_value)} "

        f"({entry.pct_gap:+.1f}% worse, {entry.detail}, score {entry.score:.0f}) "

        f"{n_suffix(entry)} — `{entry.evidence_file}`"

    )

def quality_line(entry: Opportunity) -> str:

    return (

        f"- `{entry.scenario}/{entry.operation}` `{entry.subject}` — "

        f"`{entry.metric}` = {fmt_value(entry.metric, entry.value)}: {entry.detail} "

        f"{n_suffix(entry)} — `{entry.evidence_file}`"

    )

def trend_line(entry: Opportunity) -> str:

    return (

        f"- `{entry.scenario}/{entry.operation}` `{entry.subject}` — "

        f"median `{entry.metric}` = {fmt_value(entry.metric, entry.value)} "

        f"{n_suffix(entry)} — `{entry.evidence_file}`"

    )

def unmeasured_line(entry: Opportunity) -> str:

    reason = entry.metric[len("skip:"):]

    return f"- {entry.n}× `{reason}` — {entry.detail}"

def generate_report(

    results_dir: Path,

    git_subject: str = "git",

    oak_baseline_arg: Optional[str] = None,

    input_paths: Optional[list[Path]] = None,

) -> tuple[str, list[dict[str, Any]]]:

    lanes = load_lanes(results_dir, input_paths)

    oak_baseline = choose_oak_baseline(lanes, oak_baseline_arg)

    losses = build_losses(lanes, git_subject, oak_baseline)

    quality = build_quality_bar(lanes)

    trends = build_trends(lanes)

    unmeasured = build_unmeasured(lanes)

    low_confidence = [entry for entry in losses + quality + trends if entry.low_n]

    lines: list[str] = [

        "# Oak Opportunity Report",

        "",

        (

            "Ranked attack list for the Oak improvement swarm. Skip rows never enter "

            "medians or counts; null metric values are unmeasured, never zero (ADR-0002)."

        ),

        "",

        "## Inputs",

        "",

    ]

    for lane in lanes:

        if lane.missing:

            lines.append(f"- {lane.name}: `{lane.path}` — missing (lane not run)")

        else:

            lines.append(f"- {lane.name}: `{lane.path}` — read, {lane.row_count} rows")

    lines.append(f"- Git baseline: `{git_subject}`")

    lines.append(f"- Oak baseline: `{oak_baseline}`" if oak_baseline else "- Oak baseline: missing")

    lines.extend(

        [

            "",

            "## 1. Where Oak loses to Git",

            "",

            (

                "Successful rows only; medians per (scenario, operation). Score = pct worse "

                "x operation-family frequency weight (status/diff/commit/snapshot/proc.spawn=10, "

                "branch/log/show=5, init/clone/push/pull/fetch=2, else 1)."

            ),

            "",

        ]

    )

    if oak_baseline is None:

        lines.append("No Oak baseline subject present; comparison not possible.")

    elif not losses:

        lines.append("No (scenario, operation, metric) where the Oak baseline is worse than Git.")

    else:

        lines.extend(loss_line(rank, entry) for rank, entry in enumerate(losses, start=1))

    lines.extend(["", "## 2. Absolute quality bar", ""])

    if not quality:

        lines.append("No quality-bar violations on Oak subjects.")

    else:

        lines.extend(quality_line(entry) for entry in quality)

    lines.extend(["", "## 3. Oak-only trends (no Git comparator)", ""])

    mount_lane = next((lane for lane in lanes if lane.name == "mount"), None)

    if mount_lane is None:

        lines.append("- mount: lane not provided")

    elif mount_lane.missing:

        lines.append("- mount: lane not run")

    if not trends:

        lines.append("No remote.net.* or mount-lane measurements available.")

    else:

        lines.extend(trend_line(entry) for entry in trends)

    lines.extend(["", "## 4. Unmeasured (coverage to unlock)", ""])

    if not unmeasured:

        lines.append("No skip rows recorded.")

    else:

        lines.extend(unmeasured_line(entry) for entry in unmeasured)

    lines.extend(["", "## 5. Low confidence", ""])

    if not low_confidence:

        lines.append("No line above is based on fewer than 3 samples.")

    else:

        lines.append("Lines above based on fewer than 3 samples; re-run with more runs before trusting the gap.")

        lines.append("")

        for entry in low_confidence:

            lines.append(

                f"- [{entry.section}] `{entry.scenario}/{entry.operation}` "

                f"`{entry.metric}` `{entry.subject}` (n={entry.n})"

            )

    backlog_rank: dict[Opportunity, int] = {}

    for rank, entry in enumerate(losses + quality + trends + unmeasured, start=1):

        backlog_rank[entry] = rank

    json_entries: list[dict[str, Any]] = []

    for section_entries in (losses, quality, trends, unmeasured, low_confidence):

        for rank, entry in enumerate(section_entries, start=1):

            section = entry.section if section_entries is not low_confidence else "low_confidence"

            json_entries.append(

                {

                    "section": section,

                    "rank": rank,

                    "backlog_rank": backlog_rank.get(entry),

                    "scenario": entry.scenario,

                    "operation": entry.operation,

                    "metric": entry.metric,

                    "subject": entry.subject,

                    "value": entry.value,

                    "baseline_value": entry.baseline_value,

                    "pct_gap": entry.pct_gap,

                    "score": entry.score,

                    "n": entry.n,

                    "low_n": entry.low_n,

                    "evidence_file": entry.evidence_file,

                }

            )

    return "\n".join(lines) + "\n", json_entries

def main(argv: Optional[list[str]] = None) -> int:

    args = parse_args(argv)

    report, json_entries = generate_report(

        args.results,

        args.git_subject,

        args.oak_baseline,

        args.jsonl or None,

    )

    sys.stdout.write(report)

    if args.json:

        with args.json.open("w") as fh:

            json.dump(json_entries, fh, indent=2, sort_keys=True)

            fh.write("\n")

    return 0

if __name__ == "__main__":

    raise SystemExit(main())