| 1 | """Same persisted payload is required of both subjects for every workflow."""
|
| 2 | import json
|
| 3 | from pathlib import Path
|
| 4 | import shutil
|
| 5 | import subprocess
|
| 6 | import sys
|
| 7 | import tempfile
|
| 8 | import unittest
|
| 9 |
|
| 10 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
| 11 | import workflow_ab
|
| 12 | from oakbench import workflow_oracles as oracle
|
| 13 |
|
| 14 |
|
| 15 | class WorkflowParityTest(unittest.TestCase):
|
| 16 | def command(self, binary, repo, *args):
|
| 17 | result = subprocess.run([binary, *args], cwd=repo, env=workflow_ab.base_env(), capture_output=True, timeout=60)
|
| 18 | self.assertEqual(result.returncode, 0, result.stderr)
|
| 19 |
|
| 20 | def test_uncommitted_then_committed_payload_matrix(self):
|
| 21 | from test_workflow_oracles import finish_configs, finish_assets
|
| 22 | for kind in ('git', 'oak'):
|
| 23 | binary = shutil.which(kind)
|
| 24 | if binary is None:
|
| 25 | self.fail('Required local subject unavailable: ' + kind)
|
| 26 | for scenario in sorted(oracle.SCENARIOS):
|
| 27 | with self.subTest(kind=kind, scenario=scenario), tempfile.TemporaryDirectory() as tmp:
|
| 28 | repo = Path(tmp)
|
| 29 | if scenario in oracle.EXACT_SCENARIOS:
|
| 30 | workflow_ab.WORKFLOWS[scenario].make_fixture(repo)
|
| 31 | else:
|
| 32 | for relative in oracle.COMMITTED_PAYLOADS.get(scenario, {'history.txt': 'seed'}):
|
| 33 | path = repo / relative
|
| 34 | path.parent.mkdir(parents=True, exist_ok=True)
|
| 35 | path.write_text('initial\n')
|
| 36 | self.command(binary, repo, 'init')
|
| 37 | if kind == 'git':
|
| 38 | self.command(binary, repo, 'add', '.')
|
| 39 | self.command(binary, repo, 'commit', '-m', 'seed')
|
| 40 | else:
|
| 41 | self.command(binary, repo, 'commit', '--no-verify')
|
| 42 | if scenario == 'wide_config_refactor': finish_configs(repo)
|
| 43 | elif scenario == 'large_asset_manifest': finish_assets(repo)
|
| 44 | elif scenario != 'history_archaeology':
|
| 45 | for relative, expected in oracle.COMMITTED_PAYLOADS[scenario].items():
|
| 46 | (repo / relative).write_text(expected + '\n')
|
| 47 | if scenario == 'sync_pull_dirty':
|
| 48 | (repo / 'work.txt').write_text('dirty local edit\n')
|
| 49 | subject = workflow_ab.Subject(kind, kind, kind, Path(binary))
|
| 50 | evidence = workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS[scenario], repo, [])
|
| 51 | if scenario == 'history_archaeology':
|
| 52 | self.assertIsNone(evidence['workflow_integrity_passed'])
|
| 53 | self.assertEqual(evidence['workflow_integrity_unmeasured_reason'], 'history_answer_oracle_unavailable')
|
| 54 | continue
|
| 55 | self.assertIs(evidence['workflow_integrity_passed'], False, evidence)
|
| 56 | if kind == 'git':
|
| 57 | self.command(binary, repo, 'add', '.')
|
| 58 | self.command(binary, repo, 'commit', '-m', 'task')
|
| 59 | else:
|
| 60 | self.command(binary, repo, 'commit', '--no-verify')
|
| 61 | evidence = workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS[scenario], repo, [])
|
| 62 | self.assertIs(evidence['workflow_integrity_passed'], True, evidence)
|
| 63 | command = [sys.executable, '-I', oracle.__file__, scenario, str(repo), '--evidence-json', '--'+kind+'-head', '--binary', binary]
|
| 64 | result = subprocess.run(command, capture_output=True, text=True, timeout=120)
|
| 65 | self.assertEqual(result.returncode, 0, result.stderr)
|
| 66 | self.assertIs(json.loads(result.stdout)['workflow_integrity_passed'], True)
|
| 67 |
|
| 68 | def test_scoped_git_reader_does_not_splice_moving_head(self):
|
| 69 | from unittest.mock import patch
|
| 70 | binary = shutil.which('git')
|
| 71 | with tempfile.TemporaryDirectory() as tmp:
|
| 72 | repo = Path(tmp)
|
| 73 | expected = oracle.COMMITTED_PAYLOADS['sync_push_divergence']
|
| 74 | self.command(binary, repo, 'init')
|
| 75 | (repo/'work.txt').write_text(expected['work.txt'])
|
| 76 | (repo/'notes.txt').write_text('wrong')
|
| 77 | self.command(binary, repo, 'add', '.')
|
| 78 | self.command(binary, repo, 'commit', '-m', 'partial A')
|
| 79 | original_run = subprocess.run
|
| 80 | def head():
|
| 81 | return original_run([binary, 'rev-parse', 'HEAD'], cwd=repo, capture_output=True, text=True).stdout.strip()
|
| 82 | first = head()
|
| 83 | (repo/'work.txt').write_text('wrong')
|
| 84 | (repo/'notes.txt').write_text(expected['notes.txt'])
|
| 85 | self.command(binary, repo, 'add', '.')
|
| 86 | self.command(binary, repo, 'commit', '-m', 'partial B')
|
| 87 | second = head()
|
| 88 | self.command(binary, repo, 'update-ref', 'HEAD', first)
|
| 89 | for relative, marker in expected.items():
|
| 90 | (repo/relative).write_text(marker)
|
| 91 | commands = []
|
| 92 | switched = False
|
| 93 | def moving(command, **kwargs):
|
| 94 | nonlocal switched
|
| 95 | result = original_run(command, **kwargs)
|
| 96 | commands.append(command)
|
| 97 | if not switched and ('blob' in command or 'show' in command):
|
| 98 | original_run([binary, '-C', str(repo), 'update-ref', 'HEAD', second], check=True)
|
| 99 | switched = True
|
| 100 | return result
|
| 101 | subject = workflow_ab.Subject('git', 'git', 'git', Path(binary))
|
| 102 | with patch.object(oracle.subprocess, 'run', side_effect=moving):
|
| 103 | evidence = workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS['sync_push_divergence'], repo, [])
|
| 104 | self.assertIs(evidence['workflow_integrity_passed'], False, evidence)
|
| 105 | self.assertTrue(switched)
|
| 106 | sizes = [c[-1] for c in commands if 'cat-file' in c and '-s' in c]
|
| 107 | reads = [c[-1] for c in commands if 'cat-file' in c and 'blob' in c]
|
| 108 | self.assertEqual(sizes, reads)
|
| 109 | self.assertTrue(all(len(oid) in (40,64) and 'HEAD' not in oid for oid in sizes))
|
| 110 | self.assertEqual(sum('ls-tree' in c for c in commands), 1)
|
| 111 |
|
| 112 | def test_scoped_invalid_committed_utf8_is_measured_failure_both_subjects(self):
|
| 113 | import os
|
| 114 | if not os.environ.get('OAK_BENCH_OAK_READER_PYTHON'):
|
| 115 | self.skipTest('optional independent Oak reader is not configured')
|
| 116 | for kind in ('git', 'oak'):
|
| 117 | with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
| 118 | repo = Path(tmp)
|
| 119 | (repo/'app').mkdir()
|
| 120 | path = repo/'app/pricing.py'
|
| 121 | marker = oracle.COMMITTED_PAYLOADS['bugfix_test_loop']['app/pricing.py']
|
| 122 | path.write_bytes(marker.encode() + b'\xff')
|
| 123 | binary = shutil.which(kind)
|
| 124 | self.command(binary, repo, 'init')
|
| 125 | if kind == 'git':
|
| 126 | self.command(binary, repo, 'add', '.')
|
| 127 | self.command(binary, repo, 'commit', '-m', 'invalid UTF8')
|
| 128 | else:
|
| 129 | self.command(binary, repo, 'commit', '--no-verify')
|
| 130 | path.write_text(marker)
|
| 131 | subject = workflow_ab.Subject(kind, kind, kind, Path(binary))
|
| 132 | evidence = workflow_ab.workflow_integrity_fields(subject, workflow_ab.WORKFLOWS['bugfix_test_loop'], repo, [])
|
| 133 | self.assertIs(evidence['workflow_integrity_passed'], False, evidence)
|
| 134 | self.assertIn('invalid_utf8', evidence['workflow_integrity_failure_reason'])
|
| 135 |
|
| 136 | def test_scripted_cli_stamps_actual_randomized_subject_order(self):
|
| 137 | with tempfile.TemporaryDirectory() as tmp:
|
| 138 | root=Path(tmp)
|
| 139 | config=root/'subjects.toml'
|
| 140 | config.write_text('\n'.join(f'[subjects.{kind}]\nenabled = true\nkind = "{kind}"\nlabel = "{kind}"\nbin = {json.dumps(shutil.which(kind))}\n' for kind in ('git','oak')))
|
| 141 | command=[sys.executable,str(Path(workflow_ab.__file__)), '--workflows','bugfix_test_loop',
|
| 142 | '--subjects','git,oak','--config',str(config),'--runs','1', '--randomize-subject-order',
|
| 143 | '--workdir',str(root/'work'),'--results',str(root/'results')]
|
| 144 | result=subprocess.run(command,capture_output=True,text=True,timeout=120)
|
| 145 | self.assertEqual(result.returncode,0,result.stdout+result.stderr)
|
| 146 | rows=[json.loads(line) for line in (root/'results/latest.jsonl').read_text().splitlines()]
|
| 147 | summaries=[row for row in rows if row['operation']=='workflow.total']
|
| 148 | self.assertEqual(len(summaries),2)
|
| 149 | for row in summaries:
|
| 150 | self.assertIs(row['randomized_subject_order'],True)
|
| 151 | self.assertEqual(set(row['subject_order']),{'git','oak'})
|
| 152 | self.assertEqual([row['subject'] for row in summaries],summaries[0]['subject_order'])
|
| 153 |
|
| 154 | def test_independent_reader_exact_modes_match_git(self):
|
| 155 | import os
|
| 156 | from test_workflow_oracles import finish_configs
|
| 157 | if not os.environ.get('OAK_BENCH_OAK_READER_PYTHON'):
|
| 158 | self.skipTest('configure independent Oak reader')
|
| 159 | for kind in ('git','oak'):
|
| 160 | with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
| 161 | repo=Path(tmp); binary=shutil.which(kind)
|
| 162 | workflow_ab.make_wide_config_fixture(repo); finish_configs(repo)
|
| 163 | (repo/'services/service-0000/config.toml').chmod(0o755)
|
| 164 | self.command(binary,repo,'init')
|
| 165 | if kind=='git':
|
| 166 | self.command(binary,repo,'add','.');self.command(binary,repo,'commit','-m','wrong mode')
|
| 167 | else: self.command(binary,repo,'commit','--no-verify')
|
| 168 | (repo/'services/service-0000/config.toml').chmod(0o644) # correct worktree cannot hide bad committed mode
|
| 169 | evidence=oracle.task_evidence('wide_config_refactor',repo,kind,binary)
|
| 170 | self.assertIs(evidence['workflow_integrity_passed'],False,evidence)
|
| 171 |
|
| 172 | def test_scoped_committed_modes_match_worktree_contract(self):
|
| 173 | import os
|
| 174 | if not os.environ.get('OAK_BENCH_OAK_READER_PYTHON'):
|
| 175 | self.skipTest('configure independent Oak reader')
|
| 176 | for kind in ('git','oak'):
|
| 177 | with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
| 178 | repo=Path(tmp); binary=shutil.which(kind)
|
| 179 | workflow_ab.make_bugfix_fixture(repo)
|
| 180 | path=repo/'app/pricing.py'
|
| 181 | path.write_text(path.read_text().replace('return cents - percent','return cents - ((cents * percent) // 100)'))
|
| 182 | path.chmod(0o755)
|
| 183 | self.command(binary,repo,'init')
|
| 184 | if kind=='git':
|
| 185 | self.command(binary,repo,'add','.');self.command(binary,repo,'commit','-m','wrong mode')
|
| 186 | else: self.command(binary,repo,'commit','--no-verify')
|
| 187 | path.chmod(0o644)
|
| 188 | evidence=oracle.task_evidence('bugfix_test_loop',repo,kind,binary)
|
| 189 | self.assertIs(evidence['workflow_integrity_passed'],False,evidence)
|