"""Evidence requirements for publishing output-usefulness measurements.

Legacy structural markers remain diagnostic data. They cannot certify that

standard patch tooling reconstructs the intended repository tree.

"""

from __future__ import annotations

from typing import Any

import math

import re

import json

from oakbench.rows import row_returncode

from oakbench.populations import comparison_identity

from oakbench.output_semantics import OUTPUT_ORACLE_VERSION, PATCH_ORACLE_VERSION

from oakbench.workflow_oracles import SCENARIOS as TRUSTED_WORKFLOWS, VERSION as WORKFLOW_ORACLE_VERSION

def output_evidence_failures(row: dict[str, Any]) -> list[str]:

    failures: list[str] = []

    for flag in ("pipe_compatible_unified", "patch_apply_ok", "patch_tree_matches", "patch_application_required"):

        if row.get(flag) is not None and type(row[flag]) is not bool:

            failures.append(f"{flag} is not a nullable boolean")

    for metric, source in (

        ("information_recall", "information_recall_source"),

        ("information_precision", "information_precision_source"),

    ):

        value = row.get(metric)

        if value is not None and (

            type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 1

        ):

            failures.append(f"{metric} is not a finite fraction")

        if row.get(metric) is not None and (row.get("output_truncated") is True or row.get("probe_output_truncated") is True):

            failures.append(f"{metric} is measured from incomplete output")

        if row.get(metric) is not None and (

            row.get("output_oracle_version") != OUTPUT_ORACLE_VERSION

            or row.get(source) != "exact_metadata_path_set"

        ):

            failures.append(f"{metric} lacks versioned exact-path evidence")

    claims_applicability = row.get("patch_application_required") is True or row.get("patch_apply_ok") is True

    if claims_applicability and not (

        row.get("patch_apply_ok") is True

        and row.get("patch_tree_matches") is True

        and row.get("patch_apply_returncode") == 0

        and type(row.get("patch_apply_returncode")) is int

        and row.get("patch_oracle_version") == PATCH_ORACLE_VERSION

        and row.get("patch_apply_source") == "git_apply_exact_tree"

        and isinstance(row.get("patch_instrument_sha256"), str)

        and re.fullmatch(r"[0-9a-f]{64}", row["patch_instrument_sha256"])

        and isinstance(row.get("patch_instrument_version"), str)

        and row["patch_instrument_version"].strip()

        and not row.get("patch_skip_reason")

        and not row.get("patch_failure_reason")

        and not row.get("probe_output_truncated")

        and not row.get("output_truncated")

    ):

        failures.append("structural/applicability success lacks a successful exact-tree patch application")

    return failures

def workflow_evidence_failures(rows: list[dict[str, Any]]) -> list[str]:

    """Require independent persisted-state evidence for trusted payload tasks.

    Source labels describe how observations were obtained; they are not

    equivalence classes. Scripted summaries must pass; measured agent failures

    remain evidence for success-rate accounting.

    Successful comparisons require the shared committed-content contract.

    """

    groups = {}

    failures = []

    for row in rows:

        operation = row.get("operation")

        if operation not in {"agent.workflow", "workflow.total", "workflow.skipped"}:

            continue

        code = row_returncode(row)

        integrity = row.get("workflow_integrity_passed")

        measured_agent_failure = (

            operation == "agent.workflow" and type(integrity) is bool

            and row.get("outcome") in {"fail", "timeout", "error"}

            and row.get("success") is False and code not in {0, 77}

            and not row.get("skipped")

            and row.get("outcome_attribution") == "agent" and row.get("agent_launched") is True

        )

        if not measured_agent_failure and (

            operation == "workflow.skipped" or integrity is not True or code != 0

            or row.get("skipped") or row.get("success") is False

            or row.get("correctness_passed") is False

            or (operation == "agent.workflow" and row.get("outcome") not in {None, "pass"})

        ):

            failures.append(f"{row.get('subject')}/{row.get('scenario')}: workflow correctness is failed, skipped or unmeasured")

        if row.get("operation") in {"agent.workflow", "workflow.total"} and row.get("workflow_integrity_evidence_class") in {"independent_committed_tree", "independent_committed_payload"}:

            if row.get("workflow_integrity_source") == "trusted_task_worktree;independent_oak_sqlite_reader":

                from oakbench.oak_sqlite_reader import READER_VERSION

                valid = row.get("oak_reader_version") == READER_VERSION

                valid = valid and all(isinstance(row.get(field), str) and re.fullmatch(r"[0-9a-f]{64}", row[field])

                    for field in ("oak_reader_sha256", "oak_reader_head", "oak_reader_python_sha256"))

                valid = valid and all(isinstance(row.get(field), str) and row[field].strip()

                    for field in ("oak_reader_blake3_version", "oak_reader_zstandard_version"))

                if not valid:

                    failures.append("independent Oak reader lacks versioned instrument/object evidence")

            else:

                instrument = row.get("git_head_instrument_sha256")

                subject = row.get("workflow_subject_binary_sha256")

                if (not all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) for value in (instrument, subject))

                        or instrument == subject or "subject_produced" in str(row.get("workflow_integrity_source", ""))):

                    failures.append("independent reader lacks distinct observed subject/instrument identities or uses subject-produced evidence")

        agent = row.get("agent") if isinstance(row.get("agent"), dict) else {}

        key = (row.get("scenario"), "workflow.total" if operation == "workflow.skipped" else operation)

        groups.setdefault(key, []).append(row)

    for key, items in groups.items():

        if len({item.get("subject") for item in items}) < 2:

            continue

        conditions = {}

        for item in items:

            agent = item.get("agent") if isinstance(item.get("agent"), dict) else {}

            condition = comparison_identity(item)

            condition.pop("workflow_integrity_evidence_class", None)  # failed pre-HEAD observations remain task failures

            condition["agent.name"] = agent.get("name")

            conditions.setdefault(item.get("subject"), set()).add(json.dumps(condition, sort_keys=True))

        if len({tuple(sorted(values)) for values in conditions.values()}) != 1:

            failures.append(f"{key}: subjects have unmatched experimental conditions")

        verified = [item for item in items if item.get("workflow_integrity_passed") is True]

        if len({item.get("workflow_integrity_evidence_class") for item in verified}) > 1:

            failures.append(f"{key}: successful subjects have unequal correctness evidence classes")

        if any(item.get("oracle_version") != WORKFLOW_ORACLE_VERSION for item in items):

            failures.append(f"{key}: trusted task validator version is incompatible or missing")

        # Failed task validation can finish before any HEAD reader runs. It

        # measures failure, not persisted completion; keep those trials in

        # success-rate evidence without inventing an independent reader.

        if any(item.get("workflow_integrity_evidence_class") not in {"independent_committed_tree", "independent_committed_payload"} for item in items

               if not (item.get("operation") == "agent.workflow" and item.get("workflow_integrity_passed") is False)):

            failures.append(f"{key}: trusted persisted-state comparison requires independent committed-tree readers; subject-produced exports and metadata are insufficient")

        if any(not isinstance(item.get("workflow_integrity_source"), str) or not item["workflow_integrity_source"].strip() for item in items):

            failures.append(f"{key}: correctness evidence source is missing")

        if any("subject_produced" in str(item.get("workflow_integrity_source", ""))

               and item.get("workflow_integrity_evidence_class") in {"independent_committed_tree", "independent_committed_payload"} for item in items):

            failures.append(f"{key}: subject-produced evidence cannot declare an independent reader class")

        if any(type(item.get("workflow_integrity_passed")) is not bool for item in items):

            failures.append(f"{key}: correctness evidence is failed or unmeasured")

    return failures

def workflow_qualification_failures(rows: list[dict[str, Any]], min_samples: int = 30) -> list[str]:

    """Necessary publication hygiene, not a power calculation or study certification."""

    selected = [row for row in rows if row.get("operation") in {"workflow.total", "workflow.skipped", "agent.workflow", "agent.setup"}]

    failures = []

    if not selected:

        return failures

    values = [row.get("env_isolation_version") for row in selected]

    versions = {value for value in values if isinstance(value, str)}

    if len(versions) != 1 or any(not isinstance(value, str) or not value.strip() or value in {"unknown", "unmeasured"} for value in values):

        failures.append("workflow env-isolation is missing or inconsistent")

    groups = {}

    for row in selected:

        failures.extend(output_evidence_failures(row))

        if row.get("operation") == "agent.setup":

            failures.append("agent setup infrastructure failure cannot qualify a study")

        randomized = next((row[name] for name in ("randomized_subject_order", "randomize_subject_order", "subject_order_randomized") if name in row), None)

        order = row.get("subject_order", row.get("run_subject_order"))

        if randomized is not True or not isinstance(order, list) or row.get("subject") not in order:

            failures.append("workflow subject randomization/order is unmeasured")

        agent = row.get("agent") if isinstance(row.get("agent"), dict) else {}

        key = (row.get("scenario"), row.get("operation"), row.get("subject"), row.get("benchmark_track"),

               row.get("instruction_level", agent.get("instruction_level")), json.dumps(agent, sort_keys=True))

        identity = (row.get("bench_id"), row.get("run"))

        if not isinstance(identity[0], str) or not identity[0] or type(identity[1]) is not int:

            failures.append("workflow independent trial identity missing")

        groups.setdefault(key, []).append(json.dumps(identity, sort_keys=True))

        if row.get("operation") == "agent.workflow" and (row.get("agent_launched") is not True or row.get("outcome_attribution") != "agent"):

            failures.append("workflow outcome is not an observed agent trial")

    for key, identities in groups.items():

        if len(set(identities)) != len(identities):

            failures.append(f"{key[:3]}: duplicate workflow trial identities")

        if len(set(identities)) < min_samples:

            failures.append(f"{key[:3]}: workflow samples below n={min_samples}")

    return list(dict.fromkeys(failures))