| 1 | """Production-order controls for isolated output correctness diagnostics."""
|
| 2 | from pathlib import Path
|
| 3 | import shutil
|
| 4 | import sys
|
| 5 | import tempfile
|
| 6 | import unittest
|
| 7 | from unittest import mock
|
| 8 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
| 9 | import bench
|
| 10 | from oakbench import conformance
|
| 11 | from oakbench.conformance import tree_manifest
|
| 12 | from oakbench.output_semantics import information_recall_fields
|
| 13 |
|
| 14 | @unittest.skipUnless(shutil.which('git'), 'Git control required')
|
| 15 | class CoreOracleSequenceTest(unittest.TestCase):
|
| 16 | def test_diagnostic_io_follows_timed_snapshot_chain(self):
|
| 17 | with tempfile.TemporaryDirectory() as temporary:
|
| 18 | root = Path(temporary)
|
| 19 | scenario = bench.Scenario('oracle_order', 2, 80, 2, False, 1)
|
| 20 | fixture = bench.ensure_fixture(root / 'fixtures', scenario)
|
| 21 | subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
|
| 22 | events = []
|
| 23 | command = bench.run_command
|
| 24 | apply = bench.oakbench_semantics.patch_application_fields
|
| 25 | def recorded_command(*args, **kwargs):
|
| 26 | events.append(args[2])
|
| 27 | return command(*args, **kwargs)
|
| 28 | def recorded_apply(*args, **kwargs):
|
| 29 | events.append('patch-oracle')
|
| 30 | return apply(*args, **kwargs)
|
| 31 | def recorded_manifest(*args, **kwargs):
|
| 32 | events.append('manifest-oracle')
|
| 33 | return tree_manifest(*args, **kwargs)
|
| 34 | with mock.patch.object(bench, 'run_command', side_effect=recorded_command), mock.patch.object(bench.oakbench_semantics, 'patch_application_fields', side_effect=recorded_apply), mock.patch.object(conformance, 'tree_manifest', side_effect=recorded_manifest):
|
| 35 | rows = bench.run_subject_scenario(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', skip_remote=True)
|
| 36 | self.assertGreater(events.index('patch-oracle'), events.index('task.snapshot.commit'))
|
| 37 | self.assertGreater(events.index('manifest-oracle'), events.index('task.snapshot.commit'))
|
| 38 | self.assertGreater(events.index('diff.dirty'), events.index('task.snapshot.commit'))
|
| 39 | probe = next(row for row in rows if row['operation'] == 'diff.full.inforecall')
|
| 40 | self.assertIsNone(probe['elapsed_ms'])
|
| 41 | self.assertEqual(probe['measurement_phase'], 'output_diagnostic')
|
| 42 | timed_diff = next(row for row in rows if row['operation'] == 'diff.dirty')
|
| 43 | self.assertEqual(timed_diff['cache_state'], 'hash_warmed_isolated_dirty_replay')
|
| 44 | self.assertTrue(probe['patch_apply_ok'])
|
| 45 |
|
| 46 | def test_failed_diff_mutation_never_reaches_later_same_repo_command(self):
|
| 47 | for returncode in (0, 3):
|
| 48 | with self.subTest(returncode=returncode), tempfile.TemporaryDirectory() as temporary:
|
| 49 | root = Path(temporary)
|
| 50 | scenario = bench.Scenario('mutating_diff', 2, 80, 2, False, 1)
|
| 51 | fixture = bench.ensure_fixture(root / 'fixtures', scenario)
|
| 52 | wrapper = root / 'mutating-git'
|
| 53 | wrapper.write_text('#!' + sys.executable + '\n' +
|
| 54 | 'import pathlib, subprocess, sys\n' +
|
| 55 | 'if sys.argv[1] == "diff":\n' +
|
| 56 | ' pathlib.Path("collateral").write_text("unexpected")\n' +
|
| 57 | f' sys.exit({returncode})\n' +
|
| 58 | f'sys.exit(subprocess.call([{shutil.which("git")!r}, *sys.argv[1:]]))\n')
|
| 59 | wrapper.chmod(0o755)
|
| 60 | subject = bench.Subject('git', 'git', 'Git', wrapper)
|
| 61 | original = bench.run_command
|
| 62 | poisoned = set()
|
| 63 | def command(*args, **kwargs):
|
| 64 | cwd = args[3]
|
| 65 | self.assertNotIn(cwd, poisoned, 'measurement executed after diff corrupted this repository')
|
| 66 | row = original(*args, **kwargs)
|
| 67 | if args[2] == 'diff.dirty':
|
| 68 | self.assertTrue((cwd / 'collateral').exists())
|
| 69 | poisoned.add(cwd)
|
| 70 | return row
|
| 71 | with mock.patch.object(bench, 'run_command', side_effect=command):
|
| 72 | rows = bench.run_subject_scenario(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', skip_remote=True)
|
| 73 | diff = next(row for row in rows if row['operation'] == 'diff.dirty')
|
| 74 | self.assertEqual(diff['patch_failure_reason'], 'diff_mutated_subject_tree')
|
| 75 | self.assertEqual(diff['returncode'], 1)
|
| 76 | self.assertFalse(diff.get('skipped', False))
|
| 77 | self.assertTrue(any(row['operation'] == 'task.snapshot' for row in rows))
|
| 78 | self.assertFalse(any(row['operation'] == 'diff.full.inforecall' and not row.get('skipped') for row in rows))
|
| 79 |
|
| 80 | def test_failed_information_diff_is_measured_mutation_not_skip(self):
|
| 81 | with tempfile.TemporaryDirectory() as temporary:
|
| 82 | root = Path(temporary)
|
| 83 | (root / 'a').write_text('original')
|
| 84 | subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
|
| 85 | command = [sys.executable, '-c', 'import pathlib,sys; pathlib.Path("a").write_text("corrupt"); sys.exit(3)']
|
| 86 | row = bench.run_information_probe(subject, bench.Scenario('mutating_info', 1, 8, 1, False, 1),
|
| 87 | 'diff.full', root, command, 0, {}, ['a'], root)
|
| 88 | self.assertEqual(row['process_returncode'], 3)
|
| 89 | self.assertEqual(row['patch_failure_reason'], 'diff_mutated_subject_tree')
|
| 90 | self.assertFalse(row['patch_apply_ok'])
|
| 91 | self.assertNotIn('patch_skip_reason', row)
|
| 92 | self.assertIsNone(row['elapsed_ms'])
|
| 93 |
|
| 94 | def test_truncated_path_metrics_are_unmeasured(self):
|
| 95 | fields = information_recall_fields(' M a\n', ['a', 'b'], 100, True, 'status')
|
| 96 | for name in ('information_recall', 'information_precision', 'bytes_per_changed_file_named', 'changed_files_recovered', 'changed_files_reported', 'changed_files_false_positive'):
|
| 97 | self.assertIsNone(fields[name], name)
|
| 98 |
|
| 99 | def test_main_prerequisite_failure_stops_all_following_operations(self):
|
| 100 | for failed_op in ('repo.init', 'snapshot.initial.add', 'snapshot.initial.commit',
|
| 101 | 'status.clean', 'status.dirty', 'snapshot.dirty.commit', 'branch.create', 'task.snapshot.commit'):
|
| 102 | with self.subTest(operation=failed_op), tempfile.TemporaryDirectory() as temporary:
|
| 103 | root = Path(temporary)
|
| 104 | scenario = bench.Scenario('prerequisite', 2, 80, 2, False, 1)
|
| 105 | fixture = bench.ensure_fixture(root / 'fixtures', scenario)
|
| 106 | subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
|
| 107 | called = []
|
| 108 | original = bench.run_command
|
| 109 | def command(*args, **kwargs):
|
| 110 | self.assertNotIn(failed_op, called, 'command after a failed prerequisite')
|
| 111 | called.append(args[2])
|
| 112 | row = original(*args, **kwargs)
|
| 113 | if args[2] == failed_op:
|
| 114 | row['returncode'] = 3
|
| 115 | return row
|
| 116 | with mock.patch.object(bench, 'run_command', side_effect=command):
|
| 117 | rows = bench.run_subject_scenario(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', skip_remote=True)
|
| 118 | self.assertIn(failed_op, called)
|
| 119 | self.assertTrue(any(row['returncode'] != 0 for row in rows))
|
| 120 | # Abandoned operations are recorded skips, never missing rows.
|
| 121 | failed_main = failed_op.rsplit('.', 1)[0] if failed_op.endswith(('.add', '.commit')) else failed_op
|
| 122 | planned = bench.planned_operations(subject, skip_remote=True, skip_diff=False, skip_determinism_probe=False)
|
| 123 | abandoned = planned[planned.index(failed_main) + 1:]
|
| 124 | skipped = {row['operation']: row for row in rows if row.get('skipped')}
|
| 125 | for op in abandoned:
|
| 126 | self.assertIn(op, skipped, (failed_op, op))
|
| 127 | self.assertEqual(skipped[op]['returncode'], 77)
|
| 128 | self.assertEqual(skipped[op]['skip_reason'], 'aborted_after_failed_prerequisite:' + failed_main)
|
| 129 |
|
| 130 | def test_branch_failed_attestation_stops_even_with_zero_exit(self):
|
| 131 | with tempfile.TemporaryDirectory() as temporary:
|
| 132 | root = Path(temporary)
|
| 133 | scenario = bench.Scenario('branch_attestation', 2, 80, 2, False, 1)
|
| 134 | fixture = bench.ensure_fixture(root / 'fixtures', scenario)
|
| 135 | subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
|
| 136 | original = bench.apply_state_change_attestation
|
| 137 | def attest(row, repo, before, operation, **kwargs):
|
| 138 | if operation == 'branch.create':
|
| 139 | row.update(returncode=0, integrity_check_passed=False)
|
| 140 | return row
|
| 141 | return original(row, repo, before, operation, **kwargs)
|
| 142 | with mock.patch.object(bench, 'apply_state_change_attestation', side_effect=attest):
|
| 143 | rows = bench.run_subject_scenario(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', skip_remote=True)
|
| 144 | measured = [row for row in rows if not row.get('skipped')]
|
| 145 | self.assertEqual(measured[-1]['operation'], 'branch.create')
|
| 146 | tail = rows[rows.index(measured[-1]) + 1:]
|
| 147 | self.assertEqual([row['operation'] for row in tail][:1], ['task.snapshot'])
|
| 148 | self.assertTrue(all(row['skip_reason'] == 'aborted_after_failed_prerequisite:branch.create' for row in tail))
|
| 149 |
|
| 150 | def test_replay_setup_failure_still_stops_fsmonitor(self):
|
| 151 | import subprocess
|
| 152 | with tempfile.TemporaryDirectory() as temporary:
|
| 153 | root = Path(temporary)
|
| 154 | scenario = bench.Scenario('setup_cleanup', 1, 80, 1, False, 1)
|
| 155 | fixture = bench.ensure_fixture(root / 'fixtures', scenario)
|
| 156 | subject = bench.Subject('git_fsmonitor', 'git', 'Git', Path('/git'), git_mode='fsmonitor')
|
| 157 | calls = []
|
| 158 | def command(args, **kwargs):
|
| 159 | calls.append(args)
|
| 160 | return subprocess.CompletedProcess(args, 3 if 'add' in args else 0, b'', b'failed')
|
| 161 | with mock.patch.object(bench.subprocess, 'run', side_effect=command):
|
| 162 | rows = bench.run_isolated_dirty_phase(subject, scenario, fixture, 0, root / 'runs', {}, False, 10000, 'core-equivalent', False)
|
| 163 | self.assertEqual(rows[0]['returncode'], 77)
|
| 164 | self.assertEqual(calls[-1], ['/git', 'fsmonitor--daemon', 'stop'])
|
| 165 |
|
| 166 | def test_determinism_first_failure_cannot_be_overwritten_by_second_success(self):
|
| 167 | with tempfile.TemporaryDirectory() as temporary:
|
| 168 | root = Path(temporary)
|
| 169 | script = 'import pathlib,sys; p=pathlib.Path("attempt"); first=not p.exists(); p.touch(); sys.exit(3 if first else 0)'
|
| 170 | subject = bench.Subject('git', 'git', 'Git', Path(shutil.which('git')))
|
| 171 | row = bench.run_determinism_probe(subject, bench.Scenario('repeat', 1, 8, 1, False, 1),
|
| 172 | 'status.dirty', root, [sys.executable, '-c', script], 0, {})
|
| 173 | self.assertEqual(row['returncode'], 3)
|
| 174 | self.assertIsNone(row['output_stable'])
|