"""Negative task-oracle tests: fixture validators are not trusted evidence."""

from __future__ import annotations

import json

import subprocess

from pathlib import Path

import shutil

import sys

import tempfile

import unittest

from unittest import mock

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))

import workflow_ab

import agent_workflow

from oakbench import workflow_oracles

class ExistingOracleFalsePassTest(unittest.TestCase):

    def test_oracle_version_is_pinned(self):

        self.assertEqual(workflow_oracles.VERSION, "2026-09-11.2")

    def test_deleting_rewritten_configs_fails(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            for path in (repo / "services").glob("*/config.toml"):

                path.write_text(path.read_text().replace("timeout_ms = 2500", "timeout_ms = 3000"))

            (repo / "services/service-0003/config.toml").unlink()

            self.assertIsNotNone(workflow_ab.workflow_worktree_payload_failure(

                workflow_ab.WORKFLOWS["wide_config_refactor"], repo

            ))

    def test_empty_manifest_fails(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            (repo / "manifest.json").write_text('{"assets": []}')

            self.assertIsNotNone(workflow_ab.workflow_worktree_payload_failure(

                workflow_ab.WORKFLOWS["large_asset_manifest"], repo

            ))

def finish_configs(repo):

    for path in (repo / "services").glob("*/config.toml"):

        path.write_text(path.read_text().replace("timeout_ms = 2500", "timeout_ms = 3000"))

def finish_assets(repo):

    path = repo / "manifest.json"

    data = json.loads(path.read_text())

    for entry in data["assets"]:

        entry["reviewed"] = True

    path.write_text(json.dumps(data))

class TrustedWorkflowOracleTest(unittest.TestCase):

    def assert_both_lanes_reject(self, name, repo, reason):

        workflow = workflow_ab.WORKFLOWS[name]

        failure = workflow_ab.workflow_worktree_payload_failure(workflow, repo)

        self.assertIsNotNone(failure)

        self.assertIn(reason, failure)

        # This is the exact external command run by real-agent run_oracle.

        proc = agent_workflow.run_command(agent_workflow.test_command_for(workflow), repo)

        self.assertEqual(proc.returncode, 1, proc.stderr)

        self.assertIn(reason, proc.stdout)

    def test_exact_configs_reject_loss_extras_wrong_values_and_types(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            finish_configs(repo)

            self.assertIsNone(workflow_oracles.validate("wide_config_refactor", repo))

            path = repo / "services/service-0003/config.toml"

            original = path.read_bytes()

            for wrong in (b"", original.replace(b"3000", b"1800"), original + b"retries = 99\n"):

                with self.subTest(wrong=wrong):

                    path.write_bytes(wrong)

                    self.assert_both_lanes_reject("wide_config_refactor", repo, "config_payload_mismatch")

            path.unlink()

            self.assert_both_lanes_reject("wide_config_refactor", repo, "config_path_set_mismatch")

            path.symlink_to("../../service-0000/config.toml")

            self.assert_both_lanes_reject("wide_config_refactor", repo, "payload_path_type")

            path.unlink()

            path.write_bytes(original)

            extra = repo / "services/extra"

            extra.write_bytes(b"unexpected")

            self.assert_both_lanes_reject("wide_config_refactor", repo, "config_path_set_mismatch")

            extra.unlink()

            untouched = repo / "services/service-0001/config.toml"

            untouched.write_bytes(untouched.read_bytes().replace(b"1800", b"3000"))

            self.assert_both_lanes_reject("wide_config_refactor", repo, "config_payload_mismatch")

    def test_assets_reject_manifest_lies_and_content_corruption(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_large_asset_fixture(repo)

            finish_assets(repo)

            self.assertIsNone(workflow_oracles.validate("large_asset_manifest", repo))

            path = repo / "manifest.json"

            valid = json.loads(path.read_text())

            variants = [

                ({"assets": []}, "manifest_asset_count"),

                ({"assets": valid["assets"][:3]}, "manifest_asset_count"),

                ({"assets": valid["assets"] + [valid["assets"][0]]}, "manifest_asset_count"),

                ({"assets": [valid["assets"][0]] * 4}, "manifest_duplicate_asset"),

                ({"assets": [{**item, "bytes": True} for item in valid["assets"]]}, "manifest_asset_fields"),

                ({"assets": [{**item, "reviewed": 1} for item in valid["assets"]]}, "manifest_asset_fields"),

                ({"assets": [{**item, "path": "../outside"} for item in valid["assets"]]}, "manifest_asset_identity"),

                ({"assets": None}, "manifest_asset_count"),

                ({}, "manifest_object_shape"),

            ]

            for data, reason in variants:

                with self.subTest(data=data):

                    path.write_text(json.dumps(data))

                    self.assert_both_lanes_reject("large_asset_manifest", repo, reason)

            path.write_text('{"assets": [], "assets": []}')

            self.assert_both_lanes_reject("large_asset_manifest", repo, "duplicate_json_key")

            path.write_text(json.dumps(valid))

            extra = repo / "assets/asset-04.bin"

            extra.write_bytes(b"extra")

            self.assert_both_lanes_reject("large_asset_manifest", repo, "asset_path_set_mismatch")

            extra.unlink()

            asset = repo / "assets/asset-03.bin"

            with asset.open("r+b") as stream:

                stream.write(b"X")

            self.assert_both_lanes_reject("large_asset_manifest", repo, "asset_content_mismatch")

            asset.unlink()

            self.assert_both_lanes_reject("large_asset_manifest", repo, "asset_path_set_mismatch")

    def test_deleted_or_noop_fixture_validator_cannot_bypass_oracle(self):

        for name, filename, maker, finish in (

            ("wide_config_refactor", "validate_configs.py", workflow_ab.make_wide_config_fixture, finish_configs),

            ("large_asset_manifest", "validate_manifest.py", workflow_ab.make_large_asset_fixture, finish_assets),

        ):

            with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:

                repo = Path(tmp) / "repo"

                repo.mkdir()

                artifacts = Path(tmp) / "artifacts"

                artifacts.mkdir()

                maker(repo)

                before = agent_workflow.workspace_manifest(repo)

                finish(repo)

                validator = repo / filename

                for replacement in (b"raise SystemExit(0)\n", None):

                    if replacement is None:

                        validator.unlink()

                    else:

                        validator.write_bytes(replacement)

                    self.assert_both_lanes_reject(name, repo, "fixture_validator_modified_or_missing")

                    subject = workflow_ab.Subject(name="git", kind="git", label="git", bin=Path("/git"))

                    with mock.patch.object(agent_workflow, "vcs_status", return_value=("", 0, 0.0)), \

                         mock.patch.object(agent_workflow, "commit_count", return_value=2):

                        result = agent_workflow.run_oracle(

                            subject, workflow_ab.WORKFLOWS[name], repo, before, 1, artifacts,

                        )

                    self.assertFalse(result.success)

                    self.assertFalse(result.checks["required_checks_passed"])

                    self.assertIn("fixture_validator_modified_or_missing", (artifacts / "oracle.test.log").read_text())

    def test_invalid_trusted_oracle_envelope_is_unmeasured(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            before = agent_workflow.workspace_manifest(repo)

            finish_configs(repo)

            artifacts = repo / "logs"

            artifacts.mkdir()

            subject = workflow_ab.Subject("git", "git", "Git", Path("/git"))

            for output, code in (("", 1), ("{}", 0), ("not json", 0),

                                 (json.dumps({"oracle_version": workflow_oracles.VERSION,

                                              "workflow_integrity_source": "trusted", "workflow_integrity_passed": None}), 77)):

                with mock.patch.object(agent_workflow, "run_command", return_value=subprocess.CompletedProcess([], code, output, "")), \

                     mock.patch.object(agent_workflow, "vcs_status", return_value=("", 0, 0)), \

                     mock.patch.object(agent_workflow, "commit_count", return_value=2):

                    result = agent_workflow.run_oracle(subject, workflow_ab.WORKFLOWS["wide_config_refactor"], repo, before, 1, artifacts)

                self.assertIsNone(result.evidence["workflow_integrity_passed"])

                self.assertEqual(result.evidence["workflow_integrity_unmeasured_reason"], "trusted_oracle_evidence_invalid_or_missing")

    def test_unavailable_required_check_does_not_hide_measured_failure(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            before = agent_workflow.workspace_manifest(repo)

            finish_configs(repo)

            evidence = {"oracle_version": workflow_oracles.VERSION, "workflow_integrity_source": "trusted",

                "workflow_integrity_passed": None, "workflow_integrity_unmeasured_reason": "missing"}

            with mock.patch.object(agent_workflow, "run_command", return_value=subprocess.CompletedProcess([], 77, json.dumps(evidence), "")), \

                 mock.patch.object(agent_workflow, "vcs_status", return_value=("dirty", 1, 0)), \

                 mock.patch.object(agent_workflow, "expected_change_ok", return_value=True), \

                 mock.patch.object(agent_workflow, "commit_count", return_value=2):

                result = agent_workflow.run_oracle(workflow_ab.Subject("git", "git", "Git", Path("/git")),

                    workflow_ab.WORKFLOWS["wide_config_refactor"], repo, before, 1, repo)

            self.assertIsNone(result.checks["required_checks_passed"])

            self.assertEqual(result.failure_reason, "final_vcs_clean")

    def test_nontrusted_check_return77_is_unmeasured(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            with mock.patch.object(agent_workflow, "run_command", return_value=subprocess.CompletedProcess([], 77, "", "missing tool")), \

                 mock.patch.object(agent_workflow, "vcs_status", return_value=("", 0, 0)), \

                 mock.patch.object(agent_workflow, "expected_change_ok", return_value=True), \

                 mock.patch.object(agent_workflow, "commit_count", return_value=2):

                result = agent_workflow.run_oracle(workflow_ab.Subject("git", "git", "Git", Path("/git")),

                    workflow_ab.WORKFLOWS["bugfix_test_loop"], repo, {}, 1, repo)

            self.assertIsNone(result.checks["required_checks_passed"])

            self.assertIsNone(result.failure_reason)

            self.assertIsNone(result.evidence["workflow_integrity_passed"])

            self.assertEqual(result.evidence["workflow_integrity_unmeasured_reason"], "required_check_instrument_unavailable")

    def test_fixture_module_shadowing_is_not_executed(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            finish_configs(repo)

            (repo / "json.py").write_text("raise RuntimeError('untrusted import executed')")

            proc = agent_workflow.run_command(

                agent_workflow.test_command_for(workflow_ab.WORKFLOWS["wide_config_refactor"]), repo,

            )

            self.assertEqual(proc.returncode, 0, proc.stderr)

    def test_scripted_unavailable_instrument_row_persists(self):

        from oakbench.rows import validate_rows

        from oakbench.results import ResultsStore

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            finish_configs(repo)

            subject = workflow_ab.Subject("git", "git", "Git", Path("/subject/not-reader"))

            with mock.patch.object(workflow_oracles, "resolve_git_instrument", return_value=(None, {})):

                evidence = workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS["wide_config_refactor"], repo, [])

            row = workflow_ab.sum_rows([], {"bench_id": "test", "profile": "test"}, subject,

                workflow_ab.WORKFLOWS["wide_config_refactor"], 0, "workflow", "workflow.total", evidence)

            self.assertEqual(row["returncode"], 77)

            self.assertEqual(validate_rows([row], "workflow"), [])

            ResultsStore(repo / "results", "workflow").write("test", [row])

    def test_workflow_summary_fails_when_trusted_payload_fails(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            finish_configs(repo)

            (repo / "services/service-0003/config.toml").unlink()

            subject = workflow_ab.Subject(name="oak", kind="oak", label="oak", bin=Path("/oak"))

            fields = workflow_ab.workflow_integrity_fields(

                subject, workflow_ab.WORKFLOWS["wide_config_refactor"], repo, [],

            )

            self.assertFalse(fields["correctness_passed"])

            self.assertIn("config_path_set_mismatch", fields["correctness_failure_reason"])

            self.assertEqual(fields["oracle_version"], workflow_oracles.VERSION)

@unittest.skipIf(shutil.which("git") is None, "git not installed")

class CommittedWorkflowOracleTest(unittest.TestCase):

    def test_subject_binary_never_becomes_committed_tree_reader(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            finish_configs(repo)

            subject = workflow_ab.Subject("git", "git", "Git candidate", Path("/candidate/git"))

            with mock.patch.object(workflow_oracles, "resolve_git_instrument", return_value=(None, {})) as resolve:

                workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS["wide_config_refactor"], repo, [])

                resolve.assert_called_once_with(None)

            command = agent_workflow.test_command_for(workflow_ab.WORKFLOWS["wide_config_refactor"], subject)

            self.assertEqual(command[command.index("--binary") + 1], "/candidate/git")

            self.assertNotIn("--git-instrument", command)

    def test_nontrusted_git_records_actual_scoped_head_evidence(self):

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_bugfix_fixture(repo)

            path = repo / "app/pricing.py"

            path.write_text(path.read_text().replace("return cents - percent", "return cents - ((cents * percent) // 100)"))

            self.git(repo, "init"); self.git(repo, "add", "."); self.git(repo, "commit", "-m", "fixed")

            fields = workflow_ab.workflow_integrity_fields(workflow_ab.Subject("git", "git", "Git", Path(shutil.which("git"))),

                workflow_ab.WORKFLOWS["bugfix_test_loop"], repo, [])

            self.assertTrue(fields["workflow_integrity_passed"])

            self.assertEqual(fields["workflow_integrity_evidence_class"], "subject_binary_committed_payload")

            self.assertEqual(fields["workflow_subject_binary_sha256"], fields["git_head_instrument_sha256"])

    def test_killed_git_instrument_is_unmeasured(self):

        with mock.patch.object(workflow_oracles, "resolve_git_instrument", return_value=("/git", {})), \

             mock.patch.object(workflow_oracles.subprocess, "run", return_value=subprocess.CompletedProcess([], -9, b"", b"")):

            self.assertEqual(workflow_oracles.validate_git_head("wide_config_refactor", Path("/tmp")), "git_head_oracle_unavailable")

            self.assertEqual(workflow_oracles.git_head_text(Path("/tmp"), "x"), (None, "git_head_oracle_unavailable"))

    def test_path_instrument_and_missing_instrument_are_not_subject_failures(self):

        real_git = shutil.which("git")

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            finish_configs(repo)

            with mock.patch.object(workflow_oracles.shutil, "which", return_value=None) as lookup:

                evidence = workflow_oracles.task_evidence("wide_config_refactor", repo, "git")

                lookup.assert_called_with("git")

                self.assertIsNone(evidence["workflow_integrity_passed"])

                self.assertEqual(evidence["workflow_integrity_unmeasured_reason"], "git_head_oracle_unavailable")

                self.assertIsNone(evidence["git_head_instrument_sha256"])

                self.assertEqual(evidence["workflow_integrity_evidence_class"], "worktree_payload")

            binary, metadata = workflow_oracles.resolve_git_instrument(real_git)

            self.assertTrue(Path(binary).is_absolute())

            self.assertEqual(len(metadata["git_head_instrument_sha256"]), 64)

            self.assertTrue(metadata["git_head_instrument_version"])

    @unittest.skipUnless(shutil.which("oak"), "Oak installed binary required")

    def test_correct_oak_worktree_cannot_hide_partial_checkpoint(self):

        binary = shutil.which("oak")

        with tempfile.TemporaryDirectory() as tmp:

            repo = Path(tmp)

            workflow_ab.make_wide_config_fixture(repo)

            def oak(*args):

                process = subprocess.run([binary, *args], cwd=repo, env=workflow_ab.base_env(), capture_output=True, timeout=60)

                self.assertEqual(process.returncode, 0, process.stderr)

            oak("init"); oak("commit", "--no-verify")

            finish_configs(repo)

            oak("commit", "--no-verify", "services/service-0000/config.toml")

            self.assertIsNone(workflow_oracles.validate("wide_config_refactor", repo))

            evidence = workflow_oracles.task_evidence("wide_config_refactor", repo, "oak", binary)

            self.assertIs(evidence["workflow_integrity_passed"], False, evidence)

            self.assertEqual(evidence["workflow_integrity_evidence_class"], "worktree_payload")

            self.assertIn("git_head_config_payload_mismatch", evidence["workflow_integrity_failure_reason"])

            oak("commit", "--no-verify")

            verified = workflow_oracles.task_evidence("wide_config_refactor", repo, "oak", binary)

            self.assertIs(verified["workflow_integrity_passed"], True, verified)

            self.assertNotEqual(evidence["oak_export_head"], verified["oak_export_head"])

    def git(self, repo, *args):

        result = subprocess.run(

            [shutil.which("git"), *args], cwd=repo, env=workflow_ab.base_env(),

            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,

        )

        self.assertEqual(result.returncode, 0, result.stderr)

    def test_correct_worktree_cannot_hide_partial_commit(self):

        for name, maker, finish, staged in (

            ("wide_config_refactor", workflow_ab.make_wide_config_fixture, finish_configs,

             "services/service-0000/config.toml"),

            ("large_asset_manifest", workflow_ab.make_large_asset_fixture, finish_assets, "manifest.json"),

        ):

            with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:

                repo = Path(tmp)

                maker(repo)

                self.git(repo, "init")

                self.git(repo, "add", ".")

                if name == "large_asset_manifest":

                    self.git(repo, "rm", "--cached", "assets/asset-03.bin")

                self.git(repo, "commit", "-m", "initial")

                finish(repo)

                self.git(repo, "add", staged)

                self.git(repo, "commit", "-m", "partial task")

                self.assertIsNone(workflow_oracles.validate(name, repo))

                self.assertIsNotNone(workflow_oracles.validate_git_head(name, repo))

                subject = workflow_ab.Subject(name="git", kind="git", label="Git", bin=Path(shutil.which("git")))

                fields = workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS[name], repo, [])

                self.assertFalse(fields["correctness_passed"])

                proc = agent_workflow.run_command(agent_workflow.test_command_for(workflow_ab.WORKFLOWS[name], subject), repo)

                self.assertEqual(proc.returncode, 1, proc.stderr)

                self.assertIn("git_head_", proc.stdout)

                self.git(repo, "add", ".")

                self.git(repo, "commit", "-m", "complete task")

                self.assertIsNone(workflow_oracles.validate_git_head(name, repo))

                proc = agent_workflow.run_command(agent_workflow.test_command_for(workflow_ab.WORKFLOWS[name], subject), repo)

                self.assertEqual(proc.returncode, 0, proc.stderr)

                observed = json.loads(proc.stdout)

                self.assertEqual(observed["workflow_integrity_evidence_class"], "subject_binary_committed_tree")

                self.assertEqual(observed["workflow_subject_binary_sha256"], observed["git_head_instrument_sha256"])

class RealSubjectWorkflowOracleTest(unittest.TestCase):

    def test_mock_agent_config_and_asset_tasks_on_real_vcs(self):

        for kind in ("git", "oak"):

            binary = shutil.which(kind)

            if binary is None:

                with self.subTest(kind=kind):

                    self.skipTest(f"{kind} not installed")

                continue

            for name in ("wide_config_refactor", "large_asset_manifest"):

                with self.subTest(kind=kind, name=name), tempfile.TemporaryDirectory() as tmp:

                    root = Path(tmp)

                    subject = workflow_ab.Subject(name=kind, kind=kind, label=kind, bin=Path(binary))

                    workflow = workflow_ab.WORKFLOWS[name]

                    fixture = workflow_ab.prepare_fixture(root, workflow)

                    row = agent_workflow.run_one(

                        agent=agent_workflow.AgentConfig(name="mock", adapter="deterministic_mock", provider="local",

                                                        bin=None, model=None, version="test"),

                        subject=subject, workflow=workflow, fixture=fixture, run_index=0,

                        run_root=root / "runs", artifacts_root=root / "artifacts", timeout_seconds=300,

                        track="agent-default", agent_environment="minimal", instruction_level="full-docs",

                        bench_id="oracle-integration", timestamp_utc="2026-09-10T00:00:00Z", source={}, host_meta={},

                    )

                    self.assertTrue(row["success"], row.get("failure_reason"))

                    self.assertEqual(agent_workflow.validate_row(row), [])

                    self.assertIn(workflow_oracles.VERSION, row["scenario_version"])

                    self.assertEqual(row["oracle_version"], workflow_oracles.VERSION)

                    self.assertTrue(row["workflow_integrity_passed"])

                    self.assertIn("git_head_instrument_sha256", row)

                    from oakbench.populations import comparison_identity

                    identity = comparison_identity(row)

                    self.assertEqual(identity["oracle_version"], workflow_oracles.VERSION)

                    self.assertEqual(identity["env_isolation_version"], agent_workflow.ENV_ISOLATION_VERSION)