Viewing files at commit b76ab8b42dfdCommit detailsView latest

Browse files
tests/test_core_oracle_sequence.py 174 lines · 11.2 KB · python
1
"""Production-order controls for isolated output correctness diagnostics."""
2
from pathlib import Path
3
import shutil
4
import sys
5
import tempfile
6
import unittest
7
from unittest import mock
8
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
9
import bench
10
from oakbench import conformance
11
from oakbench.conformance import tree_manifest
12
from oakbench.output_semantics import information_recall_fields
13
14
@unittest.skipUnless(shutil.which('git'), 'Git control required')
15
class CoreOracleSequenceTest(unittest.TestCase):
16
    def test_diagnostic_io_follows_timed_snapshot_chain(self):
17
        with tempfile.TemporaryDirectory() as temporary:
18
            root = Path(temporary)
19
            scenario = bench.Scenario('oracle_order', 2, 80, 2, False, 1)
20
            fixture = bench.ensure_fixture(root / 'fixtures', scenario)
21
            subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
22
            events = []
23
            command = bench.run_command
24
            apply = bench.oakbench_semantics.patch_application_fields
25
            def recorded_command(*args, **kwargs):
26
                events.append(args[2])
27
                return command(*args, **kwargs)
28
            def recorded_apply(*args, **kwargs):
29
                events.append('patch-oracle')
30
                return apply(*args, **kwargs)
31
            def recorded_manifest(*args, **kwargs):
32
                events.append('manifest-oracle')
33
                return tree_manifest(*args, **kwargs)
34
            with mock.patch.object(bench, 'run_command', side_effect=recorded_command), mock.patch.object(bench.oakbench_semantics, 'patch_application_fields', side_effect=recorded_apply), mock.patch.object(conformance, 'tree_manifest', side_effect=recorded_manifest):
35
                rows = bench.run_subject_scenario(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', skip_remote=True)
36
            self.assertGreater(events.index('patch-oracle'), events.index('task.snapshot.commit'))
37
            self.assertGreater(events.index('manifest-oracle'), events.index('task.snapshot.commit'))
38
            self.assertGreater(events.index('diff.dirty'), events.index('task.snapshot.commit'))
39
            probe = next(row for row in rows if row['operation'] == 'diff.full.inforecall')
40
            self.assertIsNone(probe['elapsed_ms'])
41
            self.assertEqual(probe['measurement_phase'], 'output_diagnostic')
42
            timed_diff = next(row for row in rows if row['operation'] == 'diff.dirty')
43
            self.assertEqual(timed_diff['cache_state'], 'hash_warmed_isolated_dirty_replay')
44
            self.assertTrue(probe['patch_apply_ok'])
45
46
    def test_failed_diff_mutation_never_reaches_later_same_repo_command(self):
47
        for returncode in (0, 3):
48
            with self.subTest(returncode=returncode), tempfile.TemporaryDirectory() as temporary:
49
                root = Path(temporary)
50
                scenario = bench.Scenario('mutating_diff', 2, 80, 2, False, 1)
51
                fixture = bench.ensure_fixture(root / 'fixtures', scenario)
52
                wrapper = root / 'mutating-git'
53
                wrapper.write_text('#!' + sys.executable + '\n' +
54
                    'import pathlib, subprocess, sys\n' +
55
                    'if sys.argv[1] == "diff":\n' +
56
                    '    pathlib.Path("collateral").write_text("unexpected")\n' +
57
                    f'    sys.exit({returncode})\n' +
58
                    f'sys.exit(subprocess.call([{shutil.which("git")!r}, *sys.argv[1:]]))\n')
59
                wrapper.chmod(0o755)
60
                subject = bench.Subject('git', 'git', 'Git', wrapper)
61
                original = bench.run_command
62
                poisoned = set()
63
                def command(*args, **kwargs):
64
                    cwd = args[3]
65
                    self.assertNotIn(cwd, poisoned, 'measurement executed after diff corrupted this repository')
66
                    row = original(*args, **kwargs)
67
                    if args[2] == 'diff.dirty':
68
                        self.assertTrue((cwd / 'collateral').exists())
69
                        poisoned.add(cwd)
70
                    return row
71
                with mock.patch.object(bench, 'run_command', side_effect=command):
72
                    rows = bench.run_subject_scenario(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', skip_remote=True)
73
                diff = next(row for row in rows if row['operation'] == 'diff.dirty')
74
                self.assertEqual(diff['patch_failure_reason'], 'diff_mutated_subject_tree')
75
                self.assertEqual(diff['returncode'], 1)
76
                self.assertFalse(diff.get('skipped', False))
77
                self.assertTrue(any(row['operation'] == 'task.snapshot' for row in rows))
78
                self.assertFalse(any(row['operation'] == 'diff.full.inforecall' and not row.get('skipped') for row in rows))
79
80
    def test_failed_information_diff_is_measured_mutation_not_skip(self):
81
        with tempfile.TemporaryDirectory() as temporary:
82
            root = Path(temporary)
83
            (root / 'a').write_text('original')
84
            subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
85
            command = [sys.executable, '-c', 'import pathlib,sys; pathlib.Path("a").write_text("corrupt"); sys.exit(3)']
86
            row = bench.run_information_probe(subject, bench.Scenario('mutating_info', 1, 8, 1, False, 1),
87
                    'diff.full', root, command, 0, {}, ['a'], root)
88
            self.assertEqual(row['process_returncode'], 3)
89
            self.assertEqual(row['patch_failure_reason'], 'diff_mutated_subject_tree')
90
            self.assertFalse(row['patch_apply_ok'])
91
            self.assertNotIn('patch_skip_reason', row)
92
            self.assertIsNone(row['elapsed_ms'])
93
94
    def test_truncated_path_metrics_are_unmeasured(self):
95
        fields = information_recall_fields(' M a\n', ['a', 'b'], 100, True, 'status')
96
        for name in ('information_recall', 'information_precision', 'bytes_per_changed_file_named', 'changed_files_recovered', 'changed_files_reported', 'changed_files_false_positive'):
97
            self.assertIsNone(fields[name], name)
98
99
    def test_main_prerequisite_failure_stops_all_following_operations(self):
100
        for failed_op in ('repo.init', 'snapshot.initial.add', 'snapshot.initial.commit',
101
                          'status.clean', 'status.dirty', 'snapshot.dirty.commit', 'branch.create', 'task.snapshot.commit'):
102
            with self.subTest(operation=failed_op), tempfile.TemporaryDirectory() as temporary:
103
                root = Path(temporary)
104
                scenario = bench.Scenario('prerequisite', 2, 80, 2, False, 1)
105
                fixture = bench.ensure_fixture(root / 'fixtures', scenario)
106
                subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
107
                called = []
108
                original = bench.run_command
109
                def command(*args, **kwargs):
110
                    self.assertNotIn(failed_op, called, 'command after a failed prerequisite')
111
                    called.append(args[2])
112
                    row = original(*args, **kwargs)
113
                    if args[2] == failed_op:
114
                        row['returncode'] = 3
115
                    return row
116
                with mock.patch.object(bench, 'run_command', side_effect=command):
117
                    rows = bench.run_subject_scenario(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', skip_remote=True)
118
                self.assertIn(failed_op, called)
119
                self.assertTrue(any(row['returncode'] != 0 for row in rows))
120
                # Abandoned operations are recorded skips, never missing rows.
121
                failed_main = failed_op.rsplit('.', 1)[0] if failed_op.endswith(('.add', '.commit')) else failed_op
122
                planned = bench.planned_operations(subject, skip_remote=True, skip_diff=False, skip_determinism_probe=False)
123
                abandoned = planned[planned.index(failed_main) + 1:]
124
                skipped = {row['operation']: row for row in rows if row.get('skipped')}
125
                for op in abandoned:
126
                    self.assertIn(op, skipped, (failed_op, op))
127
                    self.assertEqual(skipped[op]['returncode'], 77)
128
                    self.assertEqual(skipped[op]['skip_reason'], 'aborted_after_failed_prerequisite:' + failed_main)
129
130
    def test_branch_failed_attestation_stops_even_with_zero_exit(self):
131
        with tempfile.TemporaryDirectory() as temporary:
132
            root = Path(temporary)
133
            scenario = bench.Scenario('branch_attestation', 2, 80, 2, False, 1)
134
            fixture = bench.ensure_fixture(root / 'fixtures', scenario)
135
            subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
136
            original = bench.apply_state_change_attestation
137
            def attest(row, repo, before, operation, **kwargs):
138
                if operation == 'branch.create':
139
                    row.update(returncode=0, integrity_check_passed=False)
140
                    return row
141
                return original(row, repo, before, operation, **kwargs)
142
            with mock.patch.object(bench, 'apply_state_change_attestation', side_effect=attest):
143
                rows = bench.run_subject_scenario(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', skip_remote=True)
144
            measured = [row for row in rows if not row.get('skipped')]
145
            self.assertEqual(measured[-1]['operation'], 'branch.create')
146
            tail = rows[rows.index(measured[-1]) + 1:]
147
            self.assertEqual([row['operation'] for row in tail][:1], ['task.snapshot'])
148
            self.assertTrue(all(row['skip_reason'] == 'aborted_after_failed_prerequisite:branch.create' for row in tail))
149
150
    def test_replay_setup_failure_still_stops_fsmonitor(self):
151
        import subprocess
152
        with tempfile.TemporaryDirectory() as temporary:
153
            root = Path(temporary)
154
            scenario = bench.Scenario('setup_cleanup', 1, 80, 1, False, 1)
155
            fixture = bench.ensure_fixture(root / 'fixtures', scenario)
156
            subject = bench.Subject('git_fsmonitor', 'git', 'Git', Path('/git'), git_mode='fsmonitor')
157
            calls = []
158
            def command(args, **kwargs):
159
                calls.append(args)
160
                return subprocess.CompletedProcess(args, 3 if 'add' in args else 0, b'', b'failed')
161
            with mock.patch.object(bench.subprocess, 'run', side_effect=command):
162
                rows = bench.run_isolated_dirty_phase(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', False)
163
            self.assertEqual(rows[0]['returncode'], 77)
164
            self.assertEqual(calls[-1], ['/git', 'fsmonitor--daemon', 'stop'])
165
166
    def test_determinism_first_failure_cannot_be_overwritten_by_second_success(self):
167
        with tempfile.TemporaryDirectory() as temporary:
168
            root = Path(temporary)
169
            script = 'import pathlib,sys; p=pathlib.Path("attempt"); first=not p.exists(); p.touch(); sys.exit(3 if first else 0)'
170
            subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
171
            row = bench.run_determinism_probe(subject, bench.Scenario('repeat', 1, 8, 1, False, 1),
172
                'status.dirty', root, [sys.executable, '-c', script], 0, {})
173
            self.assertEqual(row['returncode'], 3)
174
            self.assertIsNone(row['output_stable'])