from __future__ import annotations

import contextlib

import io

import json

from pathlib import Path

import sys

import tempfile

import unittest

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))

import publish_gate

from oakbench.public_output import output_evidence_failures, workflow_evidence_failures

from oakbench.output_semantics import OUTPUT_ORACLE_VERSION, PATCH_ORACLE_VERSION

def exact_output():

    return {

        "information_recall": 1.0, "information_precision": 1.0,

        "information_recall_source": "exact_metadata_path_set",

        "information_precision_source": "exact_metadata_path_set",

        "output_oracle_version": OUTPUT_ORACLE_VERSION,

        "pipe_compatible_unified": True, "patch_application_required": True,

        "patch_apply_ok": True, "patch_tree_matches": True, "patch_apply_returncode": 0,

        "patch_oracle_version": PATCH_ORACLE_VERSION,

        "patch_apply_source": "git_apply_exact_tree",

        "patch_instrument_sha256": "a" * 64,

        "patch_instrument_version": "git version 2.50.1",

    }

class PublicOutputTest(unittest.TestCase):

    def test_absent_measurements_are_not_fabricated(self):

        self.assertEqual(output_evidence_failures({}), [])

        self.assertEqual(output_evidence_failures({"information_recall": None}), [])

    def test_exact_evidence_accepts_valid_patch_and_measured_low_recall(self):

        row = exact_output()

        self.assertEqual(output_evidence_failures(row), [])

        row["information_recall"] = 0.0

        self.assertEqual(output_evidence_failures(row), [])

    def test_legacy_substrings_and_structural_headers_are_not_proof(self):

        for change in (

            {"information_recall_source": "ground_truth_path_substring_match"},

            {"information_precision_source": "path_token_precision_against_changed_paths"},

            {"output_oracle_version": None},

            {"patch_apply_ok": None}, {"patch_tree_matches": False},

            {"patch_apply_returncode": 1}, {"patch_apply_returncode": False},

            {"patch_oracle_version": None}, {"patch_apply_source": "header_regex"},

            {"patch_skip_reason": "pristine_tree_over_cap"},

            {"patch_failure_reason": "patch_result_tree_mismatch"},

            {"information_recall": float("nan")}, {"information_precision": 1.1},

            {"information_recall": True}, {"patch_instrument_sha256": None},

            {"patch_instrument_version": ""},

            {"probe_output_truncated": True}, {"output_truncated": True},

        ):

            with self.subTest(change=change):

                self.assertTrue(output_evidence_failures({**exact_output(), **change}))

    def test_dirty_structure_is_diagnostic_not_patch_applicability(self):

        row = {key: value for key, value in exact_output().items() if not key.startswith("patch_")}

        row["operation"] = "diff.dirty.inforecall"

        self.assertEqual(output_evidence_failures(row), [])

        row["patch_application_required"] = True

        self.assertTrue(output_evidence_failures(row))

    def test_malformed_flags_cannot_bypass_evidence_checks(self):

        for flag in ("pipe_compatible_unified", "patch_apply_ok", "patch_tree_matches"):

            self.assertEqual(output_evidence_failures({flag: None}), [])

            for value in (1, 0, "true", [], {}):

                with self.subTest(flag=flag, value=value):

                    self.assertTrue(output_evidence_failures({flag: value}))

    def test_cross_subject_workflow_evidence_cannot_hide_asymmetry(self):

        rows = [{"subject": subject, "scenario": "task", "operation": "agent.workflow",

                 "workflow_integrity_source": source, "workflow_integrity_passed": True}

                for subject, source in (("git", "trusted_git_head"), ("oak", "metadata_only"))]

        self.assertTrue(workflow_evidence_failures(rows))

        rows[1]["workflow_integrity_source"] = "trusted_git_head"

        self.assertEqual(workflow_evidence_failures(rows), [])

        rows[1]["workflow_integrity_passed"] = None

        self.assertTrue(workflow_evidence_failures(rows))

        rows[1].pop("workflow_integrity_source")

        self.assertTrue(workflow_evidence_failures(rows))

    def test_skipped_workflow_instrument_cannot_disappear_from_public_comparison(self):

        rows = [{"subject": subject, "scenario": "task", "operation": "agent.workflow",

                 "workflow_integrity_source": "trusted_task_payload;known_good_git_head_tree",

                 "workflow_integrity_passed": True,

                 "agent": {"name": "agent", "model": "model", "provider": "provider", "environment": "minimal"}}

                for subject in ("git", "oak")]

        rows[0].update(returncode=77, skipped=True, skip_reason="git_head_instrument_unavailable",

                       workflow_integrity_passed=None)

        self.assertTrue(workflow_evidence_failures(rows))

        with tempfile.TemporaryDirectory() as temporary:

            path = Path(temporary) / "rows.jsonl"

            path.write_text("\n".join(json.dumps(row) for row in rows))

            output = io.StringIO()

            with contextlib.redirect_stdout(output):

                self.assertEqual(publish_gate.main([str(path)]), 1)

            self.assertIn("FAIL   workflow-evidence", output.getvalue())

        rows[1]["agent"] = {**rows[1]["agent"], "model": "different"}

        self.assertEqual(workflow_evidence_failures(rows), [])

    def test_publication_entrypoint_enforces_evidence(self):

        rows = []

        for subject in ("git", "git_untracked_cache", "git_fsmonitor"):

            rows.append({

                "profile": "standard", "subject": subject, "scenario": "tiny_text",

                "operation": "diff.full.inforecall", "returncode": 0,

                "env_isolation_version": "1", "randomized_subject_order": True,

                "subject_order": ["git", "git_untracked_cache", "git_fsmonitor"],

                "benchmark_track": "core-equivalent", "bench_id": "gate-test", "run": 0,

                **exact_output(),

            })

        measured = [

            {key: value for key, value in row.items() if key not in exact_output()}

            for row in rows

        ]

        rows.extend({**row, "operation": "diff.dirty.inforecall",

                     **{key: value for key, value in exact_output().items() if not key.startswith("patch_")}}

                    for row in measured)

        rows.extend(

            {**row, "operation": "status.clean", "run": run}

            for row in measured for run in range(30)

        )

        with tempfile.TemporaryDirectory() as temporary:

            path = Path(temporary) / "rows.jsonl"

            path.write_text("\n".join(json.dumps(row) for row in rows))

            with contextlib.redirect_stdout(io.StringIO()):

                self.assertEqual(publish_gate.main([str(path)]), 0)

            rows[0]["patch_apply_ok"] = None

            rows[0]["patch_skip_reason"] = "git_unavailable"

            path.write_text("\n".join(json.dumps(row) for row in rows))

            output = io.StringIO()

            with contextlib.redirect_stdout(output):

                self.assertEqual(publish_gate.main([str(path)]), 1)

            self.assertIn("FAIL   output-evidence", output.getvalue())

if __name__ == "__main__":

    unittest.main()