from __future__ import annotations

import contextlib

import io

import json

from pathlib import Path

import sys

import tempfile

import unittest

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))

import publish_gate

from oakbench.public_output import output_evidence_failures, workflow_evidence_failures

from oakbench.output_semantics import OUTPUT_ORACLE_VERSION, PATCH_ORACLE_VERSION

from oakbench.workflow_oracles import VERSION as WORKFLOW_ORACLE_VERSION

def exact_output():

    return {

        "information_recall": 1.0, "information_precision": 1.0,

        "information_recall_source": "exact_metadata_path_set",

        "information_precision_source": "exact_metadata_path_set",

        "output_oracle_version": OUTPUT_ORACLE_VERSION,

        "pipe_compatible_unified": True, "patch_application_required": True,

        "patch_apply_ok": True, "patch_tree_matches": True, "patch_apply_returncode": 0,

        "patch_oracle_version": PATCH_ORACLE_VERSION,

        "patch_apply_source": "git_apply_exact_tree",

        "patch_instrument_sha256": "a" * 64,

        "patch_instrument_version": "git version 2.50.1",

    }

class PublicOutputTest(unittest.TestCase):

    def test_absent_measurements_are_not_fabricated(self):

        self.assertEqual(output_evidence_failures({}), [])

        self.assertEqual(output_evidence_failures({"information_recall": None}), [])

    def test_exact_evidence_accepts_valid_patch_and_measured_low_recall(self):

        row = exact_output()

        self.assertEqual(output_evidence_failures(row), [])

        row["information_recall"] = 0.0

        self.assertEqual(output_evidence_failures(row), [])

    def test_legacy_substrings_and_structural_headers_are_not_proof(self):

        for change in (

            {"information_recall_source": "ground_truth_path_substring_match"},

            {"information_precision_source": "path_token_precision_against_changed_paths"},

            {"output_oracle_version": None},

            {"patch_apply_ok": None}, {"patch_tree_matches": False},

            {"patch_apply_returncode": 1}, {"patch_apply_returncode": False},

            {"patch_oracle_version": None}, {"patch_apply_source": "header_regex"},

            {"patch_skip_reason": "pristine_tree_over_cap"},

            {"patch_failure_reason": "patch_result_tree_mismatch"},

            {"information_recall": float("nan")}, {"information_precision": 1.1},

            {"information_recall": True}, {"patch_instrument_sha256": None},

            {"patch_instrument_version": ""},

            {"probe_output_truncated": True}, {"output_truncated": True},

        ):

            with self.subTest(change=change):

                self.assertTrue(output_evidence_failures({**exact_output(), **change}))

    def test_dirty_structure_is_diagnostic_not_patch_applicability(self):

        row = {key: value for key, value in exact_output().items() if not key.startswith("patch_")}

        row["operation"] = "diff.dirty.inforecall"

        self.assertEqual(output_evidence_failures(row), [])

        row["patch_application_required"] = True

        self.assertTrue(output_evidence_failures(row))

    def test_known_partial_exact_metrics_cannot_publish(self):

        for flag in ("output_truncated", "probe_output_truncated"):

            for metric in ("information_recall", "information_precision"):

                row = {metric: 1.0, metric + "_source": "exact_metadata_path_set",

                       "output_oracle_version": OUTPUT_ORACLE_VERSION, flag: True}

                self.assertTrue(output_evidence_failures(row))

                row[metric] = None

                self.assertEqual(output_evidence_failures(row), [])

    def test_malformed_flags_cannot_bypass_evidence_checks(self):

        for flag in ("pipe_compatible_unified", "patch_apply_ok", "patch_tree_matches"):

            self.assertEqual(output_evidence_failures({flag: None}), [])

            for value in (1, 0, "true", [], {}):

                with self.subTest(flag=flag, value=value):

                    self.assertTrue(output_evidence_failures({flag: value}))

    def test_real_bugfix_summary_failures_block_publication(self):

        import workflow_ab

        subject = workflow_ab.Subject("oak", "oak", "Oak", Path("/oak"))

        workflow = workflow_ab.WORKFLOWS["bugfix_test_loop"]

        metadata = {"bench_id": "gate", "profile": "workflow", "source": {"oak_hash": "reviewed", "oak_status": "clean"}}

        for passed, reason in ((True, None), (False, "wrong_result"), (None, "missing_instrument")):

            summary = workflow_ab.sum_rows([], metadata, subject, workflow, 0, "workflow", "workflow.total", {

                "workflow_integrity_passed": passed, "workflow_integrity_failure_reason": reason if passed is False else None,

                "workflow_integrity_unmeasured_reason": reason if passed is None else None})

            with tempfile.TemporaryDirectory() as tmp:

                path = Path(tmp) / "rows.jsonl"

                path.write_text(json.dumps(summary)+"\n")

                with contextlib.redirect_stdout(io.StringIO()):

                    code = publish_gate.main([str(path)])

                self.assertEqual(code, 1)  # An unqualified singleton is never a public study.

    def test_real_bugfix_payload_oracle_failure_blocks_pair(self):

        import workflow_ab

        with tempfile.TemporaryDirectory() as tmp:

            root = Path(tmp)

            repo = root / "repo"

            workflow_ab.make_bugfix_fixture(repo)

            workflow = workflow_ab.WORKFLOWS["bugfix_test_loop"]

            subject = workflow_ab.Subject("oak", "oak", "Oak", Path("/oak"))

            metadata = {"bench_id": "producer", "profile": "workflow", "source": {"oak_hash": "reviewed", "oak_status": "clean"}}

            for repaired in (False, True):

                if repaired:

                    pricing = repo / "app/pricing.py"

                    pricing.write_text(pricing.read_text().replace("return cents - percent", "return cents - ((cents * percent) // 100)"))

                fields = workflow_ab.workflow_integrity_fields(subject, workflow, repo, [])

                actual = workflow_ab.sum_rows([], metadata, subject, workflow, 0, "workflow", "workflow.total", fields)

                peer = {**actual, "subject": "git", "returncode": 0, "workflow_integrity_passed": True}

                path = root / "rows.jsonl"

                path.write_text("\n".join(map(json.dumps, [actual, peer])))

                with contextlib.redirect_stdout(io.StringIO()): code = publish_gate.main([str(path)])

                self.assertEqual(code, 1)  # Repairing worktree alone supplies neither HEAD proof nor study qualification.

    def test_real_load_tier_skip_cannot_remove_trusted_subject(self):

        import workflow_ab

        workflow = workflow_ab.WORKFLOWS["wide_config_refactor"]

        subject = workflow_ab.Subject("oak_local", "oak", "Oak", Path("/oak"))

        metadata = {"profile": "workflow", "source": {"oak_hash": "reviewed", "oak_status": "clean"}}

        skipped = workflow_ab.load_tier_skip_rows([subject], [workflow], 1, metadata, "load unavailable", "busy")[0]

        passing = {**metadata, "subject": "git", "scenario": workflow.name,

                   "operation": "workflow.total", "returncode": 0, "workflow_integrity_passed": True}

        failures = workflow_evidence_failures([passing, skipped])

        self.assertTrue(any("oak_local" in failure for failure in failures))

        self.assertTrue(any("independent committed-tree" in failure for failure in failures))

        with tempfile.TemporaryDirectory() as tmp:

            path = Path(tmp) / "rows.jsonl"

            path.write_text("\n".join(map(json.dumps, [passing, skipped])))

            with contextlib.redirect_stdout(io.StringIO()):

                self.assertEqual(publish_gate.main([str(path)]), 1)

    def test_real_measured_agent_failure_remains_publishable(self):

        import agent_workflow as aw

        import workflow_ab

        from types import SimpleNamespace

        with tempfile.TemporaryDirectory() as tmp:

            root = Path(tmp)

            stdout, stderr, prompt = [root / name for name in ("stdout", "stderr", "prompt")]

            for path in (stdout, stderr, prompt): path.write_text("")

            rows = []

            for run, passed in enumerate((True, True, False)):

                oracle = SimpleNamespace(elapsed_ms=1, vcs_ms=1, test_ms=1, success=passed,

                    checks={"required_checks_passed": passed, "final_vcs_clean": True},

                    failure_reason=None if passed else "required_checks_passed", changed_files=[],

                    test_returncode=0 if passed else 1, test_output_bytes=0, dirty_files_at_end=0,

                    commit_delta=1, bytes_written=0, artifact_paths=[])

                oracle.evidence = {"oracle_version": WORKFLOW_ORACLE_VERSION,

                    "workflow_integrity_passed": passed, "workflow_integrity_failure_reason": None if passed else "required_checks_passed",

                    "workflow_integrity_source": "trusted_task_worktree;independent_git_reader",

                    "workflow_integrity_evidence_class": "independent_committed_payload" if passed else "worktree_payload",

                    "workflow_integrity_unmeasured_reason": None,

                    "git_head_instrument_sha256": "a" * 64, "workflow_subject_binary_sha256": "b" * 64}

                rows.append(aw.make_row("gate", "2026-09-11T00:00:00Z", "test",

                    aw.AgentConfig("synthetic", "codex_exec_json", "test", Path(sys.executable), "test", "test"),

                    workflow_ab.Subject("git", "git", "Git", Path("/git")),

                    workflow_ab.WORKFLOWS["bugfix_test_loop"], "agent-default", run, root, root,

                    0, 0, aw.AgentExecution(0, False, 1, stdout, stderr, prompt), oracle,

                    "task", {}, {}, {}, "minimal", "full-docs"))

            self.assertEqual(rows[-1]["outcome"], "fail")

            self.assertIs(rows[-1]["workflow_integrity_passed"], False)

            self.assertEqual(workflow_evidence_failures(rows), [])

            path = root / "rows.jsonl"

            path.write_text("\n".join(map(json.dumps, rows)))

            with contextlib.redirect_stdout(io.StringIO()):

                self.assertEqual(publish_gate.main([str(path)]), 1)  # n<30 and randomization unset remain unqualified.

            for change in ({"workflow_integrity_passed": None}, {"workflow_integrity_passed": 0},

                           {"outcome": "unknown"}, {"returncode": "77"}, {"skipped": True}):

                self.assertTrue(workflow_evidence_failures([{**rows[-1], **change}]), change)

    def test_trusted_pre_head_agent_failure_is_measured_not_persisted_success(self):

        import agent_workflow as aw

        import workflow_ab

        from oakbench.workflow_oracles import task_evidence

        from types import SimpleNamespace

        with tempfile.TemporaryDirectory() as tmp:

            root = Path(tmp)

            repo = root / "repo"

            workflow_ab.make_wide_config_fixture(repo)

            evidence = task_evidence("wide_config_refactor", repo, "git", "/git")

            self.assertIs(evidence["workflow_integrity_passed"], False)

            self.assertEqual(evidence["workflow_integrity_evidence_class"], "worktree_payload")

            stdout, stderr, prompt = [root / name for name in ("stdout", "stderr", "prompt")]

            for path in (stdout, stderr, prompt): path.write_text("")

            oracle = SimpleNamespace(elapsed_ms=1, vcs_ms=1, test_ms=1, success=False,

                checks={"required_checks_passed": False}, failure_reason=evidence["workflow_integrity_failure_reason"],

                changed_files=[], test_returncode=1, test_output_bytes=0, dirty_files_at_end=0,

                commit_delta=0, bytes_written=0, artifact_paths=[], evidence=evidence)

            rows = [aw.make_row("gate", "2026-09-11T00:00:00Z", "test",

                aw.AgentConfig("synthetic", "codex_exec_json", "test", Path(sys.executable), "test", "test"),

                workflow_ab.Subject(name, kind, name, Path("/" + kind)),

                workflow_ab.WORKFLOWS["wide_config_refactor"], "agent-default", 0, repo, root,

                0, 0, aw.AgentExecution(0, False, 1, stdout, stderr, prompt), oracle,

                "task", {}, {"oak_hash": "reviewed", "oak_status": "clean"}, {}, "minimal", "full-docs")

                for name, kind in (("git", "git"), ("oak", "oak"))]

            self.assertEqual(workflow_evidence_failures(rows), [])

            path = root / "rows.jsonl"

            path.write_text("\n".join(map(json.dumps, rows)))

            with contextlib.redirect_stdout(io.StringIO()):

                self.assertEqual(publish_gate.main([str(path)]), 1)  # n<30 and randomization unset remain unqualified.

            for change in ({"oracle_version": "retired"}, {"workflow_integrity_source": None},

                           {"workflow_integrity_passed": True}, {"success": True}, {"returncode": 0},

                           {"workflow_integrity_evidence_class": "independent_committed_tree"}):

                self.assertTrue(workflow_evidence_failures([rows[0], {**rows[1], **change}]), change)

    def test_cross_subject_workflow_evidence_cannot_hide_asymmetry(self):

        rows = [{"subject": subject, "scenario": "wide_config_refactor", "operation": "agent.workflow",

                 "workflow_integrity_source": source, "workflow_integrity_passed": True, "returncode": 0,

                 "workflow_integrity_evidence_class": evidence_class, "oracle_version": WORKFLOW_ORACLE_VERSION,

                 "workflow_subject_binary_sha256": "b" * 64, "git_head_instrument_sha256": "a" * 64}

                for subject, source, evidence_class in (("git", "trusted_exact_task_worktree;independent_committed_tree", "independent_committed_tree"),

                    ("oak", "trusted_exact_task_worktree;subject_produced_oak_head_export", "subject_produced_committed_tree"))]

        self.assertTrue(workflow_evidence_failures(rows))

        rows[1]["workflow_integrity_source"] = "another independently validated reader"

        rows[1]["workflow_integrity_evidence_class"] = "independent_committed_tree"

        self.assertEqual(workflow_evidence_failures(rows), [])

        rows[1]["workflow_integrity_passed"] = None

        self.assertTrue(workflow_evidence_failures(rows))

        rows[1].pop("workflow_integrity_source")

        self.assertTrue(workflow_evidence_failures(rows))

    def test_skipped_workflow_instrument_cannot_disappear_from_public_comparison(self):

        rows = [{"subject": subject, "scenario": "wide_config_refactor", "operation": "agent.workflow",

                 "workflow_integrity_source": "trusted_exact_task_worktree;independent_committed_tree",

                 "workflow_integrity_evidence_class": "independent_committed_tree", "oracle_version": WORKFLOW_ORACLE_VERSION,

                 "workflow_integrity_passed": True, "returncode": 0,

                 "agent": {"name": "agent", "model": "model", "provider": "provider", "environment": "minimal"}}

                for subject in ("git", "oak")]

        rows[0].update(returncode=77, skipped=True, skip_reason="git_head_instrument_unavailable",

                       workflow_integrity_passed=None)

        self.assertTrue(workflow_evidence_failures(rows))

        with tempfile.TemporaryDirectory() as temporary:

            path = Path(temporary) / "rows.jsonl"

            path.write_text("\n".join(json.dumps(row) for row in rows))

            output = io.StringIO()

            with contextlib.redirect_stdout(output):

                self.assertEqual(publish_gate.main([str(path)]), 1)

            self.assertIn("FAIL   workflow-evidence", output.getvalue())

        rows[1]["agent"] = {**rows[1]["agent"], "model": "different"}

        self.assertTrue(workflow_evidence_failures(rows))

    def test_bugfix_unequal_producer_labels_cannot_publish(self):

        rows = [{"subject": subject, "subject_kind": kind, "scenario": "bugfix_test_loop",

                 "operation": "workflow.total", "profile": "workflow", "returncode": 0,

                 "workflow_integrity_passed": True, "workflow_integrity_source": source,

                 "oracle_version": WORKFLOW_ORACLE_VERSION,

                 "source": {"oak_hash": "reviewed-revision", "oak_status": "clean"}}

                for subject, kind, source in (

                    ("git", "git", "worktree_payload_oracle;known_good_git_head_tree"),

                    ("oak", "oak", "worktree_payload_oracle;direct_repo_metadata_fingerprint"))]

        self.assertTrue(workflow_evidence_failures(rows))

        with tempfile.TemporaryDirectory() as temporary:

            path = Path(temporary) / "rows.jsonl"

            path.write_text("\n".join(json.dumps(row) for row in rows))

            output = io.StringIO()

            with contextlib.redirect_stdout(output):

                code = publish_gate.main([str(path)])

            self.assertEqual(code, 1, output.getvalue())

            self.assertIn("FAIL   workflow-evidence", output.getvalue())

    def test_trusted_workflows_require_measured_independent_same_version_evidence(self):

        for scenario in ("wide_config_refactor", "large_asset_manifest"):

            rows = [{"subject": subject, "scenario": scenario, "operation": "workflow.total",

                     "oracle_version": WORKFLOW_ORACLE_VERSION, "workflow_integrity_passed": True, "returncode": 0,

                     "workflow_integrity_evidence_class": "independent_committed_tree",

                     "workflow_integrity_source": source, "workflow_subject_binary_sha256": "b" * 64, "git_head_instrument_sha256": "a" * 64}

                    for subject, source in (("git", "trusted_exact_task_worktree;independent_committed_tree"),

                                            ("git_candidate", "trusted_exact_task_worktree;reviewed_external_reader"))]

            self.assertEqual(workflow_evidence_failures(rows), [])

            for change in ({"git_head_instrument_sha256": "b" * 64}, {"workflow_subject_binary_sha256": None}, {"oracle_version": "other"}, {"oracle_version": None},

                           {"workflow_integrity_evidence_class": None},

                           {"workflow_integrity_source": "trusted_exact_task_worktree;subject_produced_oak_head_export"},

                           {"workflow_integrity_evidence_class": "subject_produced_committed_tree"},

                           {"workflow_integrity_passed": None}, {"workflow_integrity_passed": False}):

                with self.subTest(scenario=scenario, change=change):

                    self.assertTrue(workflow_evidence_failures([rows[0], {**rows[1], **change}]))

            rows[1].update(subject="oak", workflow_integrity_source="trusted_exact_task_worktree;subject_produced_oak_head_export",

                           workflow_integrity_evidence_class="subject_produced_committed_tree")

            with tempfile.TemporaryDirectory() as temporary:

                path = Path(temporary) / "rows.jsonl"

                path.write_text("\n".join(json.dumps(row) for row in rows))

                output = io.StringIO()

                with contextlib.redirect_stdout(output):

                    self.assertEqual(publish_gate.main([str(path)]), 1)

                self.assertIn("FAIL   workflow-evidence", output.getvalue())

    def test_publication_entrypoint_enforces_evidence(self):

        rows = []

        for subject in ("git", "git_untracked_cache", "git_fsmonitor"):

            rows.append({

                "profile": "standard", "subject": subject, "scenario": "tiny_text",

                "operation": "diff.full.inforecall", "returncode": 0,

                "env_isolation_version": "1", "randomized_subject_order": True,

                "subject_order": ["git", "git_untracked_cache", "git_fsmonitor"],

                "benchmark_track": "core-equivalent", "bench_id": "gate-test", "run": 0,

                **exact_output(),

            })

        measured = [

            {key: value for key, value in row.items() if key not in exact_output()}

            for row in rows

        ]

        rows.extend({**row, "operation": "diff.dirty.inforecall",

                     **{key: value for key, value in exact_output().items() if not key.startswith("patch_")}}

                    for row in measured)

        rows.extend(

            {**row, "operation": "status.clean", "run": run}

            for row in measured for run in range(30)

        )

        with tempfile.TemporaryDirectory() as temporary:

            path = Path(temporary) / "rows.jsonl"

            path.write_text("\n".join(json.dumps(row) for row in rows))

            with contextlib.redirect_stdout(io.StringIO()):

                self.assertEqual(publish_gate.main([str(path)]), 0)

            rows[0]["patch_apply_ok"] = None

            rows[0]["patch_skip_reason"] = "git_unavailable"

            path.write_text("\n".join(json.dumps(row) for row in rows))

            output = io.StringIO()

            with contextlib.redirect_stdout(output):

                self.assertEqual(publish_gate.main([str(path)]), 1)

            self.assertIn("FAIL   output-evidence", output.getvalue())

if __name__ == "__main__":

    unittest.main()