Viewing files at commit bd39a563b7f5Commit detailsView latest

Browse files
tests/test_oracle_parity_matrix.py 206 lines · 8.6 KB · python
1
"""Git/Oak committed-evidence parity matrix on real binaries (QA P1 A1/A4).
2
3
Every row states the same repository situation for each arm; the arms must
4
agree on measured pass/fail. "Unmeasured" (None) is allowed only for a
5
missing instrument, never for subject-controlled state or a failing subject
6
binary. Ported from the reviewer's stageA/parity.py.
7
"""
8
from __future__ import annotations
9
10
import importlib.util
11
import os
12
import shutil
13
import subprocess
14
import sys
15
import tempfile
16
import unittest
17
from pathlib import Path
18
from unittest import mock
19
20
ROOT = Path(__file__).resolve().parents[1]
21
sys.path.insert(0, str(ROOT / "scripts"))
22
23
from oakbench import workflow_oracles as O  # noqa: E402
24
from oakbench.subjects import which_measured_git  # noqa: E402
25
26
GIT = which_measured_git()
27
OAK = shutil.which("oak")
28
29
30
def reader_python() -> str | None:
31
    # The independent reader is stdlib-only: the harness interpreter always qualifies.
32
    return os.environ.get("OAK_BENCH_OAK_READER_PYTHON") or sys.executable
33
34
35
def run(*command, cwd):
36
    result = subprocess.run(list(command), cwd=cwd, capture_output=True, text=True,
37
                            env={**os.environ, "OAK_NO_UPDATE_CHECK": "1"})
38
    if result.returncode:
39
        raise RuntimeError(f"{command}: {result.stderr}")
40
41
42
def commit(kind, repo):
43
    if kind == "git":
44
        run(GIT, "add", "-A", cwd=repo)
45
        run(GIT, "-c", "user.email=a@b", "-c", "user.name=a", "commit", "-qm", "x", "--allow-empty", cwd=repo)
46
    else:
47
        run(OAK, "commit", "--no-verify", cwd=repo)
48
49
50
def init(kind, repo):
51
    (repo / "seed.txt").write_text("seed\n")
52
    run(*([GIT, "init", "-q"] if kind == "git" else [OAK, "init"]), cwd=repo)
53
    commit(kind, repo)
54
55
56
def write_payload(repo, scenario):
57
    expected = dict(O.COMMITTED_PAYLOADS[scenario])
58
    if scenario == "sync_pull_dirty":
59
        expected["work.txt"] = "dirty local edit"
60
    for relative, marker in expected.items():
61
        path = repo / relative
62
        path.parent.mkdir(parents=True, exist_ok=True)
63
        if path.is_symlink() or path.exists():
64
            path.unlink()
65
        path.write_text(marker + "\n")
66
        os.chmod(path, 0o644)
67
68
69
READER = reader_python()  # captured before setUp isolates the environment
70
71
72
@unittest.skipUnless(GIT and OAK, "real Git and Oak required")
73
class OracleParityMatrixTest(unittest.TestCase):
74
    def setUp(self):
75
        patcher = mock.patch.dict(os.environ)
76
        patcher.start()
77
        self.addCleanup(patcher.stop)
78
        os.environ.pop("OAK_BENCH_OAK_READER_PYTHON", None)
79
        os.environ.pop("OAK_BENCH_GIT_INSTRUMENT", None)
80
        # The "oak-export" arm is the explicit subject-produced opt-in; the
81
        # "oak-reader" arm passes the reader interpreter (the default arm).
82
        os.environ["OAK_BENCH_OAK_COMMITTED_READER"] = "export"
83
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-parity-")).resolve()
84
        self.addCleanup(shutil.rmtree, self.tmp, True)
85
        self.stubs = self.tmp / "stubs"
86
        self.stubs.mkdir()
87
88
    def stub(self, name, command, code):
89
        path = self.stubs / name
90
        path.write_text(f'#!/bin/sh\n[ "$1" = {command} ] && {{ echo "simulated oak {command} bug" >&2; exit {code}; }}\n'
91
                        f'exec {OAK} "$@"\n')
92
        path.chmod(0o755)
93
        return str(path)
94
95
    def arms(self):
96
        yield "git", GIT, None
97
        yield "oak-export", OAK, None
98
        if READER:
99
            yield "oak-reader", OAK, READER
100
101
    def evaluate(self, scenario, situation, *, oak_binary=None):
102
        results = {}
103
        for label, binary, reader in self.arms():
104
            kind = "git" if label == "git" else "oak"
105
            repo = Path(tempfile.mkdtemp(dir=self.tmp)).resolve()
106
            init(kind, repo)
107
            situation(kind, repo)
108
            evidence = O.task_evidence(scenario, repo, kind=kind,
109
                                       binary=(oak_binary or binary) if kind == "oak" else binary,
110
                                       oak_reader_python=reader)
111
            results[label] = (evidence["workflow_integrity_passed"],
112
                              evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"])
113
        return results
114
115
    def assert_all(self, results, passed):
116
        for label, (value, reason) in results.items():
117
            self.assertIs(value, passed, f"{label}: {reason} (all arms: {results})")
118
119
    def test_payload_situations_agree_across_arms(self):
120
        scenario = "sync_push_divergence"
121
122
        def correct(kind, repo):
123
            write_payload(repo, scenario)
124
            commit(kind, repo)
125
126
        def wrong_committed(kind, repo):
127
            write_payload(repo, scenario)
128
            (repo / "notes.txt").write_text("wrong\n")
129
            commit(kind, repo)
130
            write_payload(repo, scenario)
131
132
        def exec_committed(kind, repo):
133
            write_payload(repo, scenario)
134
            os.chmod(repo / "notes.txt", 0o755)
135
            commit(kind, repo)
136
            os.chmod(repo / "notes.txt", 0o644)
137
138
        def oversized_committed(kind, repo):  # A4: task limit (1 MiB) fails first, as in Git
139
            write_payload(repo, scenario)
140
            (repo / "notes.txt").write_bytes(b"upstream change\n" + b"a" * (2 * 1024 * 1024))
141
            commit(kind, repo)
142
            write_payload(repo, scenario)
143
144
        def never_committed(kind, repo):
145
            write_payload(repo, scenario)
146
147
        def metadata_destroyed(kind, repo):  # A1 case 1
148
            write_payload(repo, scenario)
149
            commit(kind, repo)
150
            shutil.rmtree(repo / (".git" if kind == "git" else ".oak"))
151
152
        self.assert_all(self.evaluate(scenario, correct), True)
153
        for situation in (wrong_committed, exec_committed, oversized_committed, never_committed, metadata_destroyed):
154
            with self.subTest(situation=situation.__name__):
155
                self.assert_all(self.evaluate(scenario, situation), False)
156
157
    def test_failing_subject_oak_is_a_measured_failure_in_every_payload_workflow(self):
158
        # A1 case 2: a released Oak whose export or status exits nonzero must
159
        # fail the Oak arm, never turn it into an unmeasured skip.
160
        failing = {"export": self.stub("oak-export-fails", "export", 1),
161
                   "status": self.stub("oak-status-fails", "status", 1)}
162
        for scenario in sorted(O.COMMITTED_PAYLOADS):
163
            def correct(kind, repo, scenario=scenario):
164
                write_payload(repo, scenario)
165
                commit(kind, repo)
166
            for command, binary in failing.items():
167
                with self.subTest(scenario=scenario, command=command):
168
                    repo = Path(tempfile.mkdtemp(dir=self.tmp)).resolve()
169
                    init("oak", repo)
170
                    correct("oak", repo)
171
                    evidence = O.task_evidence(scenario, repo, kind="oak", binary=binary)
172
                    self.assertIs(evidence["workflow_integrity_passed"], False, evidence)
173
                    self.assertIsNone(evidence["workflow_integrity_unmeasured_reason"])
174
175
    def test_oracle_cli_exit_is_failure_not_sentinel_for_failing_export(self):
176
        scenario = "sync_pull_upstream"
177
        repo = Path(tempfile.mkdtemp(dir=self.tmp)).resolve()
178
        init("oak", repo)
179
        write_payload(repo, scenario)
180
        commit("oak", repo)
181
        binary = self.stub("oak-export-fails", "export", 1)
182
        result = subprocess.run([sys.executable, "-I", str(ROOT / "scripts/oakbench/workflow_oracles.py"), scenario,
183
                                 str(repo), "--oak-head", "--binary", binary], capture_output=True, text=True,
184
                                env={k: v for k, v in os.environ.items() if k != "OAK_BENCH_OAK_READER_PYTHON"})
185
        self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
186
        self.assertIn("oak_head_export_failed", result.stdout)
187
188
    def test_reader_arm_is_exercised_when_configured(self):
189
        if not READER:
190
            self.skipTest("optional reader dependencies not configured")
191
        self.assertIn("oak-reader", [label for label, _, _ in self.arms()])
192
193
    def test_missing_instrument_is_still_unmeasured(self):
194
        scenario = "sync_pull_upstream"
195
        repo = Path(tempfile.mkdtemp(dir=self.tmp)).resolve()
196
        init("oak", repo)
197
        write_payload(repo, scenario)
198
        commit("oak", repo)
199
        evidence = O.task_evidence(scenario, repo, kind="oak", binary=str(self.tmp / "no-such-oak"))
200
        self.assertIsNone(evidence["workflow_integrity_passed"])
201
        evidence = O.task_evidence(scenario, repo, kind="oak", oak_reader_python=str(self.tmp / "no-python"))
202
        self.assertIsNone(evidence["workflow_integrity_passed"])
203
204
205
if __name__ == "__main__":
206
    unittest.main()