import copy

from pathlib import Path

import sys

import unittest

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))

from oakbench.public_output import workflow_evidence_failures, workflow_qualification_failures

from oakbench.populations import comparison_identity, comparison_conflicts

from oakbench.workflow_oracles import VERSION

def rows():

    return [dict(subject=subject, scenario='bugfix_test_loop', operation='workflow.total', run=run,

        bench_id='study', returncode=0, workflow_integrity_passed=True, oracle_version=VERSION,

        workflow_integrity_evidence_class='independent_committed_payload', workflow_integrity_source='git_committed_payload',

        git_head_instrument_sha256='a'*64, workflow_subject_binary_sha256='b'*64,

        env_isolation_version='3', randomized_subject_order=True, subject_order=['git','oak'], benchmark_track='agent-default')

        for subject in ('git','oak') for run in range(30)]

class WorkflowPublicationQualificationTest(unittest.TestCase):

    def test_qualified_observations_are_allowed(self):

        self.assertEqual(workflow_qualification_failures(rows()), [])

        self.assertEqual(workflow_evidence_failures(rows()), [])

    def test_thin_duplicate_missing_and_mixed_environment_block(self):

        for data in (rows()[:1], rows()+[rows()[0]], [{**row,'env_isolation_version': None} for row in rows()],

                     [{**row,'randomized_subject_order':False} for row in rows()],

                     [rows()[0], {**rows()[1], 'env_isolation_version':'2'}]):

            self.assertTrue(workflow_qualification_failures(data))

    def test_different_conditions_cannot_disappear_into_singletons(self):

        for changes in ({'benchmark_track':'core-equivalent'}, {'instruction_level':'zero-shot'},

                        {'agent':{'name':'x','model':'different'}},

                        {'workflow_integrity_evidence_class':'subject_produced_committed_payload'}):

            data=rows()

            for row in data:

                if row['subject']=='oak': row.update(changes)

            self.assertTrue(workflow_evidence_failures(data),changes)

    def test_comparison_identity_includes_correctness_class(self):

        left=rows()[0]

        right={**left,'workflow_integrity_evidence_class':'subject_produced_committed_payload'}

        self.assertIn('workflow_integrity_evidence_class',comparison_conflicts(comparison_identity(left),comparison_identity(right)))

    def test_failed_agent_trials_keep_failure_denominator(self):

        data=rows()

        for row in data:

            row.update(operation='agent.workflow', agent_launched=True,outcome_attribution='agent',success=True,outcome='pass')

        data[-1].update(workflow_integrity_passed=False,workflow_integrity_evidence_class='worktree_payload',

                        success=False,outcome='fail',returncode=1)

        self.assertEqual(workflow_evidence_failures(data),[])

        self.assertEqual(workflow_qualification_failures(data),[])

        data[-1]['outcome_attribution']='harness'

        self.assertTrue(workflow_evidence_failures(data))

class RealMixedOutcomePublicationTest(unittest.TestCase):

    def test_real_producers_preserve_failed_arm_in_publication_and_pair(self):

        import contextlib

        import io

        import json

        import os

        import shutil

        import subprocess

        import tempfile

        from unittest import mock

        import agent_workflow as aw

        import workflow_ab

        import publish_gate

        from oakbench.campaign_report import report_campaign

        if not os.environ.get('OAK_BENCH_OAK_READER_PYTHON'):

            self.skipTest('configure independent reader Python to exercise real decoder')

        with tempfile.TemporaryDirectory() as tmp:

            root=Path(tmp); fixture=root/'fixture'

            workflow_ab.make_bugfix_fixture(fixture)

            agent=aw.AgentConfig('mock','deterministic_mock','local',None,None,'1')

            original=aw.run_mock_agent

            def wrong_agent(subject, workflow, repo, *args, **kwargs):

                result=original(subject, workflow, repo, *args, **kwargs)

                (repo/'app/pricing.py').write_text('def apply_discount(cents, percent):\n    return cents\n')

                for command in ([str(subject.bin),'add','.'],[str(subject.bin),'commit','-m','incorrect answer']):

                    subprocess.run(command,cwd=repo,env=workflow_ab.base_env(),check=True,capture_output=True)

                return result

            observations=[]

            for kind in ('git','oak'):

                subject=workflow_ab.Subject(kind,kind,kind,Path(shutil.which(kind)))

                with mock.patch.object(aw,'run_mock_agent',side_effect=wrong_agent if kind=='git' else original):

                    row=aw.run_one(agent,subject,workflow_ab.WORKFLOWS['bugfix_test_loop'],fixture,0,

                        root/'work',root/'artifacts',60,'agent-default','minimal','full-docs','study','2026-09-18T00:00:00Z',

                        {'oak_hash':'test-binary-provenance','oak_status':'clean'},

                        {'randomized_subject_order':True,'subject_order':['git','oak']})

                observations.append(row)

            self.assertEqual([r['success'] for r in observations],[False,True])

            self.assertEqual(workflow_evidence_failures(observations),[])

            plan={'schema_version':1,'campaign_id':'study','plan_sha256':'bound','manifest':{},'schedule':[]}

            events=[]

            for index,row in enumerate(observations):

                trial_id=str(index); row['task_id']=trial_id

                plan['schedule'].append(dict(trial_id=trial_id,block_id='pair',ordinal=index,repetition=0,

                    workflow='bugfix_test_loop',instruction_level='full-docs',agent='mock',model=None,subject=row['subject']))

                events.extend([dict(event='started',trial_id=trial_id,plan_sha256='bound'),

                               dict(event='finished',trial_id=trial_id,plan_sha256='bound',outcome=row['outcome'],row=row)])

            report=report_campaign(plan,events,bootstrap_samples=20)

            comparison=report['groups'][0]['paired_comparisons'][0]

            self.assertEqual(comparison['complete_pairs'],1)

            self.assertEqual(comparison['success_delta_percentage_points'],100.0)

            # Repeated IDs below test the gate's mechanics only: not 30 measured trials.

            gate_rows=[{**row,'run':rep,'task_id':f'{rep}-{row["subject"]}'} for rep in range(30) for row in observations]

            path=root/'rows.jsonl';path.write_text('\n'.join(map(json.dumps,gate_rows)))

            output=io.StringIO()

            with contextlib.redirect_stdout(output): result=publish_gate.main([str(path)])

            self.assertEqual(result,0,output.getvalue())

            candidate=Path('/Applications/Xcode.app/Contents/Developer/usr/bin/git')

            instrument=Path('/Library/Developer/CommandLineTools/usr/bin/git')

            if candidate.exists() and instrument.exists():

                # Distinct installed vendor builds test configured identity plumbing,

                # not independent Git implementation or statistical power.

                with mock.patch.dict(os.environ, {'OAK_BENCH_GIT_INSTRUMENT': str(instrument)}):

                    passed_git=aw.run_one(agent,workflow_ab.Subject('git','git','Git candidate',candidate),

                        workflow_ab.WORKFLOWS['bugfix_test_loop'],fixture,0,root/'success-work',root/'success-artifacts',

                        60,'agent-default','minimal','full-docs','study','2026-09-18T00:00:00Z',

                        {'oak_hash':'test-binary-provenance','oak_status':'clean'},

                        {'randomized_subject_order':True,'subject_order':['git','oak']})

                self.assertTrue(passed_git['success'],passed_git.get('failure_reason'))

                self.assertEqual(passed_git['workflow_integrity_evidence_class'],'independent_committed_payload')

                self.assertNotEqual(passed_git['workflow_subject_binary_sha256'],passed_git['git_head_instrument_sha256'])

                successful_pair=[passed_git, observations[1]]

                self.assertEqual(workflow_evidence_failures(successful_pair),[])

                path.write_text('\n'.join(json.dumps({**row,'run':rep,'task_id':f'{rep}-{row["subject"]}'})

                                          for rep in range(30) for row in successful_pair))

                output=io.StringIO()

                with contextlib.redirect_stdout(output): result=publish_gate.main([str(path)])

                self.assertEqual(result,0,output.getvalue())