Viewing files at commit 290f11640be1Commit detailsView latest

Browse files
tests/test_oracle_parity_matrix.py 207 lines · 8.4 KB · python
1
"""Git/Oak committed-evidence parity matrix on real binaries (QA P1 A1/A4).
2
3
Every row states the same repository situation for each arm; the arms must
4
agree on measured pass/fail. "Unmeasured" (None) is allowed only for a
5
missing instrument, never for subject-controlled state or a failing subject
6
binary. Ported from the reviewer's stageA/parity.py.
7
"""
8
from __future__ import annotations
9
10
import importlib.util
11
import os
12
import shutil
13
import subprocess
14
import sys
15
import tempfile
16
import unittest
17
from pathlib import Path
18
from unittest import mock
19
20
ROOT = Path(__file__).resolve().parents[1]
21
sys.path.insert(0, str(ROOT / "scripts"))
22
23
from oakbench import workflow_oracles as O  # noqa: E402
24
from oakbench.subjects import which_measured_git  # noqa: E402
25
26
GIT = which_measured_git()
27
OAK = shutil.which("oak")
28
29
30
def reader_python() -> str | None:
31
    configured = os.environ.get("OAK_BENCH_OAK_READER_PYTHON")
32
    if configured:
33
        return configured
34
    if importlib.util.find_spec("blake3") and importlib.util.find_spec("zstandard"):
35
        return sys.executable
36
    return None
37
38
39
def run(*command, cwd):
40
    result = subprocess.run(list(command), cwd=cwd, capture_output=True, text=True,
41
                            env={**os.environ, "OAK_NO_UPDATE_CHECK": "1"})
42
    if result.returncode:
43
        raise RuntimeError(f"{command}: {result.stderr}")
44
45
46
def commit(kind, repo):
47
    if kind == "git":
48
        run(GIT, "add", "-A", cwd=repo)
49
        run(GIT, "-c", "user.email=a@b", "-c", "user.name=a", "commit", "-qm", "x", "--allow-empty", cwd=repo)
50
    else:
51
        run(OAK, "commit", "--no-verify", cwd=repo)
52
53
54
def init(kind, repo):
55
    (repo / "seed.txt").write_text("seed\n")
56
    run(*([GIT, "init", "-q"] if kind == "git" else [OAK, "init"]), cwd=repo)
57
    commit(kind, repo)
58
59
60
def write_payload(repo, scenario):
61
    expected = dict(O.COMMITTED_PAYLOADS[scenario])
62
    if scenario == "sync_pull_dirty":
63
        expected["work.txt"] = "dirty local edit"
64
    for relative, marker in expected.items():
65
        path = repo / relative
66
        path.parent.mkdir(parents=True, exist_ok=True)
67
        if path.is_symlink() or path.exists():
68
            path.unlink()
69
        path.write_text(marker + "\n")
70
        os.chmod(path, 0o644)
71
72
73
READER = reader_python()  # captured before setUp isolates the environment
74
75
76
@unittest.skipUnless(GIT and OAK, "real Git and Oak required")
77
class OracleParityMatrixTest(unittest.TestCase):
78
    def setUp(self):
79
        patcher = mock.patch.dict(os.environ)
80
        patcher.start()
81
        self.addCleanup(patcher.stop)
82
        os.environ.pop("OAK_BENCH_OAK_READER_PYTHON", None)
83
        os.environ.pop("OAK_BENCH_GIT_INSTRUMENT", None)
84
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-parity-")).resolve()
85
        self.addCleanup(shutil.rmtree, self.tmp, True)
86
        self.stubs = self.tmp / "stubs"
87
        self.stubs.mkdir()
88
89
    def stub(self, name, command, code):
90
        path = self.stubs / name
91
        path.write_text(f'#!/bin/sh\n[ "$1" = {command} ] && {{ echo "simulated oak {command} bug" >&2; exit {code}; }}\n'
92
                        f'exec {OAK} "$@"\n')
93
        path.chmod(0o755)
94
        return str(path)
95
96
    def arms(self):
97
        yield "git", GIT, None
98
        yield "oak-export", OAK, None
99
        if READER:
100
            yield "oak-reader", OAK, READER
101
102
    def evaluate(self, scenario, situation, *, oak_binary=None):
103
        results = {}
104
        for label, binary, reader in self.arms():
105
            kind = "git" if label == "git" else "oak"
106
            repo = Path(tempfile.mkdtemp(dir=self.tmp)).resolve()
107
            init(kind, repo)
108
            situation(kind, repo)
109
            evidence = O.task_evidence(scenario, repo, kind=kind,
110
                                       binary=(oak_binary or binary) if kind == "oak" else binary,
111
                                       oak_reader_python=reader)
112
            results[label] = (evidence["workflow_integrity_passed"],
113
                              evidence["workflow_integrity_failure_reason"] or evidence["workflow_integrity_unmeasured_reason"])
114
        return results
115
116
    def assert_all(self, results, passed):
117
        for label, (value, reason) in results.items():
118
            self.assertIs(value, passed, f"{label}: {reason} (all arms: {results})")
119
120
    def test_payload_situations_agree_across_arms(self):
121
        scenario = "sync_push_divergence"
122
123
        def correct(kind, repo):
124
            write_payload(repo, scenario)
125
            commit(kind, repo)
126
127
        def wrong_committed(kind, repo):
128
            write_payload(repo, scenario)
129
            (repo / "notes.txt").write_text("wrong\n")
130
            commit(kind, repo)
131
            write_payload(repo, scenario)
132
133
        def exec_committed(kind, repo):
134
            write_payload(repo, scenario)
135
            os.chmod(repo / "notes.txt", 0o755)
136
            commit(kind, repo)
137
            os.chmod(repo / "notes.txt", 0o644)
138
139
        def oversized_committed(kind, repo):  # A4: task limit (1 MiB) fails first, as in Git
140
            write_payload(repo, scenario)
141
            (repo / "notes.txt").write_bytes(b"upstream change\n" + b"a" * (2 * 1024 * 1024))
142
            commit(kind, repo)
143
            write_payload(repo, scenario)
144
145
        def never_committed(kind, repo):
146
            write_payload(repo, scenario)
147
148
        def metadata_destroyed(kind, repo):  # A1 case 1
149
            write_payload(repo, scenario)
150
            commit(kind, repo)
151
            shutil.rmtree(repo / (".git" if kind == "git" else ".oak"))
152
153
        self.assert_all(self.evaluate(scenario, correct), True)
154
        for situation in (wrong_committed, exec_committed, oversized_committed, never_committed, metadata_destroyed):
155
            with self.subTest(situation=situation.__name__):
156
                self.assert_all(self.evaluate(scenario, situation), False)
157
158
    def test_failing_subject_oak_is_a_measured_failure_in_every_payload_workflow(self):
159
        # A1 case 2: a released Oak whose export or status exits nonzero must
160
        # fail the Oak arm, never turn it into an unmeasured skip.
161
        failing = {"export": self.stub("oak-export-fails", "export", 1),
162
                   "status": self.stub("oak-status-fails", "status", 1)}
163
        for scenario in sorted(O.COMMITTED_PAYLOADS):
164
            def correct(kind, repo, scenario=scenario):
165
                write_payload(repo, scenario)
166
                commit(kind, repo)
167
            for command, binary in failing.items():
168
                with self.subTest(scenario=scenario, command=command):
169
                    repo = Path(tempfile.mkdtemp(dir=self.tmp)).resolve()
170
                    init("oak", repo)
171
                    correct("oak", repo)
172
                    evidence = O.task_evidence(scenario, repo, kind="oak", binary=binary)
173
                    self.assertIs(evidence["workflow_integrity_passed"], False, evidence)
174
                    self.assertIsNone(evidence["workflow_integrity_unmeasured_reason"])
175
176
    def test_oracle_cli_exit_is_failure_not_sentinel_for_failing_export(self):
177
        scenario = "sync_pull_upstream"
178
        repo = Path(tempfile.mkdtemp(dir=self.tmp)).resolve()
179
        init("oak", repo)
180
        write_payload(repo, scenario)
181
        commit("oak", repo)
182
        binary = self.stub("oak-export-fails", "export", 1)
183
        result = subprocess.run([sys.executable, "-I", str(ROOT / "scripts/oakbench/workflow_oracles.py"), scenario,
184
                                 str(repo), "--oak-head", "--binary", binary], capture_output=True, text=True,
185
                                env={k: v for k, v in os.environ.items() if k != "OAK_BENCH_OAK_READER_PYTHON"})
186
        self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
187
        self.assertIn("oak_head_export_failed", result.stdout)
188
189
    def test_reader_arm_is_exercised_when_configured(self):
190
        if not READER:
191
            self.skipTest("optional reader dependencies not configured")
192
        self.assertIn("oak-reader", [label for label, _, _ in self.arms()])
193
194
    def test_missing_instrument_is_still_unmeasured(self):
195
        scenario = "sync_pull_upstream"
196
        repo = Path(tempfile.mkdtemp(dir=self.tmp)).resolve()
197
        init("oak", repo)
198
        write_payload(repo, scenario)
199
        commit("oak", repo)
200
        evidence = O.task_evidence(scenario, repo, kind="oak", binary=str(self.tmp / "no-such-oak"))
201
        self.assertIsNone(evidence["workflow_integrity_passed"])
202
        evidence = O.task_evidence(scenario, repo, kind="oak", oak_reader_python=str(self.tmp / "no-python"))
203
        self.assertIsNone(evidence["workflow_integrity_passed"])
204
205
206
if __name__ == "__main__":
207
    unittest.main()