"""Adversarial saved-baseline comparisons and claim eligibility controls."""

from dataclasses import replace

import contextlib

import io

import json

from pathlib import Path

import subprocess

import sys

import tempfile

import unittest

from unittest.mock import patch

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))

import benchmark_stats

from oakbench.baseline_book import Target, book_json, cells_from_rows, load_baseline_book

from oakbench.populations import comparison_identity

from oakbench.scorecard import evaluate_targets

def rows(subject="git", **overrides):

    result = []

    for index in range(30):

        row = dict(subject=subject, subject_kind="git" if subject == "git" else "oak",

                   bench_id="campaign", run=index, lane="core", profile="standard",

                   scenario="tiny", operation="status.clean", returncode=0,

                   elapsed_ms=10.0 if subject == "git" else 1.0,

                   benchmark_track="core-equivalent", command_semantics_version=1,

                   fixture_version=1, scenario_version=1, runner_class="mac-arm",

                   cache_state="warm", env_isolation_version="1", load_tier="idle",

                   binary_sha256="git-sha" if subject == "git" else "oak-sha")

        row.update(overrides)

        result.append(row)

    return result

TARGET = Target("t", "core", "standard", "tiny", "status.clean", "oak_local", "git",

                "elapsed_ms", "median", "ratio_at_most", .8, {})

def aa_rows():

    """Interleaved Git A/A control (same Git, second subject name) for per-cell floors."""

    return [dict(row, subject="git_aa", subject_kind="git", elapsed_ms=10.0 + (0.05 if row["run"] % 2 else -0.05))

            for row in rows()]

def baseline(source=None):

    return cells_from_rows((rows() + aa_rows()) if source is None else source, lane="core", metrics=("elapsed_ms",),

                           statistics_names=("median",), noise_floor_pct=1.0, bootstrap_samples=20)[0]

def evaluate(candidate=None, cell=None, mode="claim"):

    cell = baseline() if cell is None else cell

    return evaluate_targets(rows("oak_local") if candidate is None else candidate,

                            [TARGET], {cell.key: cell}, mode=mode, bootstrap_samples=20)[0]

class ClaimEligibilityTest(unittest.TestCase):

    def test_comparable_independent_trials_allow_different_vcs_binary_hashes(self):

        result = evaluate()

        self.assertEqual(result["verdict"], "GREEN")

        self.assertTrue(result["claim_eligible"])

    def test_saved_baseline_mismatch_is_invalid_even_in_diagnostic_mode(self):

        for field, value in [("runner_class", "linux"), ("fixture_version", 2),

                             ("benchmark_track", "agent-default"), ("cache_state", "cold"),

                             ("remote_transport", "network"), ("load_tier", "cpu50"),

                             ("output_oracle_version", "different")]:

            with self.subTest(field=field):

                for mode in ("claim", "diagnostic"):

                    result = evaluate(rows("oak_local", **{field: value}), mode=mode)

                    self.assertEqual(result["verdict"], "INVALID")

                    self.assertFalse(result["claim_eligible"])

    def test_missing_and_duplicate_trials_cannot_manufacture_precision(self):

        candidates = [rows("oak_local")[:1], rows("oak_local", run=0),

                      rows("oak_local") + [rows("oak_local")[0]] * 1000]

        missing = rows("oak_local")

        del missing[0]["bench_id"]

        candidates.append(missing)

        for candidate in candidates:

            with self.subTest(n=len(candidate)):

                result = evaluate(candidate)

                self.assertEqual(result["verdict"], "AMBER")

                self.assertFalse(result["claim_eligible"])

    def test_unknown_baseline_evidence_cannot_be_public_green(self):

        for changes in ({"noise_floor_pct": None}, {"noise_floor_pct": float("nan")},

                        {"noise_floor_pct": -1}, {"comparison_identity": None},

                        {"comparison_identity": {}}, {"independent_n": 1},

                        {"attempted_n": None}, {"failure_n": 1}, {"ci_low": None}):

            with self.subTest(changes=changes):

                result = evaluate(cell=replace(baseline(), **changes))

                self.assertNotEqual(result["verdict"], "GREEN")

                self.assertFalse(result["claim_eligible"])

    def test_low_or_unknown_confidence_cannot_qualify_as_claim_evidence(self):

        for confidence in (None, 0.5, float("nan"), 1.0):

            with self.subTest(baseline_confidence=confidence):

                self.assertFalse(evaluate(cell=replace(baseline(), confidence=confidence))["claim_eligible"])

        cell = cells_from_rows(rows(), lane="core", metrics=("elapsed_ms",), statistics_names=("median",),

                               noise_floor_pct=1.0, bootstrap_samples=20, confidence=.5)[0]

        self.assertFalse(evaluate(cell=cell)["claim_eligible"])

        cell = baseline()

        result = evaluate_targets(rows("oak_local"), [TARGET], {cell.key: cell}, mode="claim",

                                  confidence=.5, bootstrap_samples=20)[0]

        self.assertEqual(result["verdict"], "AMBER")

        self.assertEqual(result["confidence"], .5)

        self.assertEqual(result["bootstrap_samples"], 20)

    def test_invalid_interval_arguments_rejected_before_scoring_or_building(self):

        for kwargs in ({"bootstrap_samples": 0}, {"bootstrap_samples": -1}, {"bootstrap_samples": True},

                       {"bootstrap_samples": 1.5}, {"confidence": 0}, {"confidence": 1},

                       {"confidence": float("nan")}, {"confidence": float("inf")}):

            with self.subTest(kwargs=kwargs):

                with self.assertRaises(ValueError):

                    evaluate_targets([], [], {}, **kwargs)

                with self.assertRaises(ValueError):

                    cells_from_rows([], lane="core", **kwargs)

    def test_uniformly_unknown_cache_cannot_be_claim_evidence(self):

        result = evaluate(rows("oak_local", cache_state="unknown"), baseline(rows(cache_state="unknown")))

        self.assertEqual(result["verdict"], "AMBER")

        self.assertIn("cache_state", " ".join(result["messages"]))

    def test_legacy_diagnostic_is_explicitly_not_claim_eligible(self):

        cell = replace(baseline(), comparison_identity=None, noise_floor_pct=None)

        result = evaluate(rows("oak_local")[:1], cell, mode="diagnostic")

        self.assertEqual(result["verdict"], "GREEN")

        self.assertFalse(result["claim_eligible"])

        self.assertIn("diagnostic only", " ".join(result["messages"]))

    def test_baseline_builder_preserves_failures_and_unmeasured_denominators(self):

        source = rows() + [dict(rows()[0], returncode=1), dict(rows()[0], elapsed_ms=None)]

        cell = baseline(source)

        self.assertEqual(cell.attempted_n, 32)

        self.assertEqual(cell.failure_n, 1)

        self.assertFalse(evaluate(cell=cell)["claim_eligible"])

    def test_global_operator_floor_is_not_claim_eligible_for_latency(self):

        # ADR-0011 / QA P1-5: no single floor for every cell on the claim path.

        cell = baseline(rows())

        self.assertEqual(cell.noise_floor_source, "operator_global")

        result = evaluate(cell=cell)

        self.assertFalse(result["claim_eligible"])

        self.assertTrue(any("per-cell A/A noise floor required" in m for m in result["claim_ineligibility_reasons"]))

        self.assertEqual(baseline().noise_floor_source, "per_cell_aa")

    def test_claim_verdicts_are_fdr_controlled_across_targets(self):

        from oakbench.scorecard import apply_claim_fdr

        results = [{"verdict": "GREEN", "p_value": 0.001, "claim_eligible": True},

                   {"verdict": "GREEN", "p_value": 0.2, "claim_eligible": True},

                   {"verdict": "AMBER", "p_value": None}]

        apply_claim_fdr(results, fdr=0.05)

        self.assertEqual([r["verdict"] for r in results], ["GREEN", "AMBER", "AMBER"])

        self.assertFalse(results[1]["claim_eligible"])

        self.assertEqual(results[0]["fdr_family_size"], 2)

    def test_baseline_builder_rejects_mixed_populations(self):

        with self.assertRaisesRegex(ValueError, "incompatible baseline population"):

            baseline(rows() + rows(runner_class="linux"))

    def test_book_v2_roundtrip_and_duplicate_keys_rejected(self):

        book = book_json([baseline()])

        self.assertEqual(book["schema_version"], 2)

        with tempfile.TemporaryDirectory() as tmp:

            path = Path(tmp) / "book.json"

            path.write_text(json.dumps(book))

            cell = next(iter(load_baseline_book(path).values()))

            self.assertTrue(evaluate(cell=cell)["claim_eligible"])

            book["cells"].append(book["cells"][0])

            path.write_text(json.dumps(book))

            with self.assertRaisesRegex(ValueError, "duplicate baseline cell"):

                load_baseline_book(path)

    def test_unknown_identity_version_rejected(self):

        book = book_json([baseline()])

        book["cells"][0]["comparison_identity"]["schema_version"] = 99

        with tempfile.TemporaryDirectory() as tmp:

            path = Path(tmp) / "book.json"

            path.write_text(json.dumps(book))

            with self.assertRaisesRegex(ValueError, "unsupported baseline comparison_identity"):

                load_baseline_book(path)

    def test_versioned_patch_guard_requires_actual_success(self):

        target = replace(TARGET, guards={"patch_apply_ok": True})

        cell = baseline(rows(patch_oracle_version="apply-tree-v1") + [dict(r, patch_oracle_version="apply-tree-v1") for r in aa_rows()])

        for evidence in (None, False, True):

            candidate = rows("oak_local", patch_oracle_version="apply-tree-v1", patch_apply_ok=evidence)

            result = evaluate_targets(candidate, [target], {cell.key: cell}, mode="claim", bootstrap_samples=20)[0]

            self.assertEqual(result["verdict"], "GREEN" if evidence is True else "INVALID")

    def test_failure_only_recall_guard_fails_without_crashing(self):

        target = replace(TARGET, guards={"information_recall_at_least": 1.0})

        cell = baseline()

        result = evaluate_targets(rows("oak_local", returncode=1), [target], {cell.key: cell})[0]

        self.assertEqual(result["verdict"], "INVALID")

    def test_nested_fixture_version_normalizes_but_subject_prompt_does_not_compare(self):

        row = rows("oak_local")[0]

        del row["fixture_version"]

        row["fixture"] = {"fixture_version": 1}

        row["task_prompt_id"] = "oak-specific-label"

        self.assertEqual(comparison_identity(row), comparison_identity(rows()[0]))

    def test_cli_defaults_to_claim_and_refuses_legacy_evidence(self):

        target = {"schema_version": 1, "targets": [dict(id="t", lane="core", profile="standard",

                  scenario="tiny", operation="status.clean", subject="oak_local", comparator="git",

                  metric="elapsed_ms", goal=dict(type="ratio_at_most", value=.8))]}

        with tempfile.TemporaryDirectory() as tmp:

            root = Path(tmp)

            (root / "rows.jsonl").write_text("\n".join(json.dumps(row) for row in rows("oak_local")))

            (root / "book.json").write_text(json.dumps(book_json([replace(baseline(), comparison_identity=None)])))

            (root / "targets.json").write_text(json.dumps(target))

            command = [sys.executable, "scripts/scorecard.py", str(root / "rows.jsonl"),

                       "--baseline-book", str(root / "book.json"), "--targets", str(root / "targets.json"), "--json"]

            proc = subprocess.run(command, capture_output=True, text=True)

            self.assertEqual(proc.returncode, 1)

            self.assertEqual(json.loads(proc.stdout)[0]["mode"], "claim")

class OutcomeReportingTest(unittest.TestCase):

    def test_failure_only_cells_and_unknown_outcomes_stay_visible(self):

        source = [dict(rows()[0], returncode=1, elapsed_ms=999),

                  dict(rows()[0], returncode=None), dict(rows()[0], returncode=77)]

        with tempfile.TemporaryDirectory() as tmp:

            path = Path(tmp) / "rows.jsonl"

            path.write_text("\n".join(json.dumps(row) for row in source))

            output = io.StringIO()

            with patch.object(sys, "argv", ["benchmark_stats.py", str(path)]), contextlib.redirect_stdout(output):

                self.assertEqual(benchmark_stats.main(), 0)

            self.assertIn("| `git` | `tiny/status.clean` | 2 | 0 | 1 | 1 | 1 | 0 |", output.getvalue())

            self.assertIn("| 0 | unmeasured |", output.getvalue())

            self.assertNotIn("999", output.getvalue())

class AgentProducerIdentityTest(unittest.TestCase):

    def test_producer_records_runner_and_explicit_remaining_claim_blockers(self):

        from types import SimpleNamespace

        import agent_workflow as aw

        import workflow_ab

        from oakbench.populations import missing_claim_identity

        with tempfile.TemporaryDirectory() as tmp:

            root = Path(tmp)

            stdout, stderr, prompt = [root / name for name in ("stdout", "stderr", "prompt")]

            for path in (stdout, stderr, prompt): path.write_text("")

            execution = aw.AgentExecution(0, False, 1, stdout, stderr, prompt)

            oracle = SimpleNamespace(elapsed_ms=1, vcs_ms=1, test_ms=1, success=True, checks={}, failure_reason=None,

                changed_files=[], test_returncode=0, test_output_bytes=0, dirty_files_at_end=0, commit_delta=1,

                bytes_written=0, artifact_paths=[])

            with patch.object(aw, "runner_fields", return_value={"runner_class": "dedicated-test", "runner_id": "observed", "runner_profile": {}}):

                row = aw.make_row("synthetic", "2026-09-11T00:00:00Z", "test",

                    aw.AgentConfig("test", "codex_exec_json", "provider", Path(sys.executable), "version", "model"),

                    workflow_ab.Subject("git", "git", "Git", Path("/git")), workflow_ab.WORKFLOWS["bugfix_test_loop"],

                    "agent-default", 0, root, root, 0, 0, execution, oracle, "task", {}, {}, {}, "minimal", "full-docs")

            self.assertEqual(aw.validate_row(row), [])

            identity = comparison_identity(row)

            self.assertEqual(identity["runner_class"], "dedicated-test")

            self.assertEqual(identity["oracle_version"], aw.ORACLE_VERSION)

            self.assertEqual(row["cache_state_reason"], "cache_state_unset")

            self.assertIsNone(row["calibration_version"])

            self.assertTrue(row["calibration_unmeasured_reason"])

            self.assertEqual(missing_claim_identity(identity, "agent", "elapsed_ms", "agent.workflow"), ["cache_state"])

            self.assertEqual(missing_claim_identity(identity, "agent", "tokens", "agent.workflow"), ["calibration_version"])

if __name__ == "__main__":

    unittest.main()