Viewing files at commit 21316fe59317Commit detailsView latest

Browse files
tests/test_conformance_gate.py 148 lines · 7.2 KB · python
1
"""Conformance gate semantics and its wiring into devloop and publish_gate."""
2
import contextlib
3
import io
4
import json
5
from pathlib import Path
6
import sys
7
import tempfile
8
import unittest
9
10
ROOT = Path(__file__).resolve().parents[1]
11
sys.path.insert(0, str(ROOT / "scripts"))
12
import devloop  # noqa: E402
13
import publish_gate  # noqa: E402
14
from oakbench import conformance_gate as gate  # noqa: E402
15
from oakbench.conformance import SCENARIOS  # noqa: E402
16
from oakbench.conformance_matrix import BEHAVIOURS  # noqa: E402
17
18
ALL = list(SCENARIOS) + [b.id for b in BEHAVIOURS]
19
20
21
def conformance_row(behaviour, code=0, legs=None, subject="oak", **extra):
22
    row = {"evidence_class": "diagnostic_conformance", "subject": subject, "scenario": behaviour,
23
           "returncode": code, "binary_sha256": "b" * 64, **extra}
24
    if legs is not None:
25
        row["failed_legs"] = legs
26
    if code == 1:
27
        row["failure"] = f"{(legs or ['behaviour'])[0]}: injected"
28
    if code == 77:
29
        row.setdefault("skip_reason", "filesystem capability unavailable: case_sensitive_fs")
30
    return row
31
32
33
def healthy_rows(expected, subject="oak"):
34
    """Rows matching today's expected failures exactly: the gate's PASS state."""
35
    rows = []
36
    for behaviour in ALL:
37
        entry = expected["entries"].get(behaviour)
38
        if entry:
39
            legs = None if behaviour in SCENARIOS else entry["failed_legs"]
40
            rows.append(conformance_row(behaviour, 1, legs, subject))
41
        else:
42
            rows.append(conformance_row(behaviour, 0, None, subject))
43
    return rows
44
45
46
class EvaluateTests(unittest.TestCase):
47
    def setUp(self):
48
        self.expected = gate.load_expected()
49
50
    def test_matching_expected_failures_pass(self):
51
        result = gate.evaluate(healthy_rows(self.expected), self.expected, ALL)
52
        self.assertEqual(result["verdict"], "PASS", result)
53
        self.assertEqual(len(result["expected_failures"]), len(self.expected["entries"]))
54
55
    def test_new_failure_is_a_regression(self):
56
        rows = healthy_rows(self.expected)
57
        rows = [r for r in rows if r["scenario"] != "eol_crlf_v1"] + [conformance_row("eol_crlf_v1", 1, ["clone_fs"])]
58
        result = gate.evaluate(rows, self.expected, ALL)
59
        self.assertEqual(result["verdict"], "REGRESSED")
60
        self.assertIn("eol_crlf_v1", result["regressions"][0])
61
62
    def test_known_failure_changing_shape_is_a_regression(self):
63
        rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "path_tab_v1"]
64
        rows.append(conformance_row("path_tab_v1", 1, ["clone_fs", "clone_read"]))
65
        result = gate.evaluate(rows, self.expected, ALL)
66
        self.assertEqual(result["verdict"], "REGRESSED")
67
        self.assertIn("fb-531", result["regressions"][0])
68
69
    def test_fix_flips_to_fixed_without_regression(self):
70
        rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "path_tab_v1"]
71
        rows.append(conformance_row("path_tab_v1", 0))
72
        result = gate.evaluate(rows, self.expected, ALL)
73
        self.assertEqual(result["verdict"], "PASS")
74
        self.assertIn("fb-531", result["fixed"][0])
75
76
    def test_missing_rows_and_harness_errors_are_unmeasured(self):
77
        rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "eol_lf_v1"]
78
        self.assertEqual(gate.evaluate(rows, self.expected, ALL)["verdict"], "UNMEASURED")
79
        rows.append(conformance_row("eol_lf_v1", 2, failure="harness error"))
80
        self.assertEqual(gate.evaluate(rows, self.expected, ALL)["verdict"], "UNMEASURED")
81
        self.assertEqual(gate.evaluate([], self.expected)["verdict"], "UNMEASURED")
82
83
    def test_capability_skips_are_recorded_not_unmeasured_but_reader_skips_are(self):
84
        rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "path_case_distinct_v1"]
85
        rows.append(conformance_row("path_case_distinct_v1", 77))
86
        result = gate.evaluate(rows, self.expected, ALL)
87
        self.assertEqual(result["verdict"], "PASS")
88
        self.assertEqual(len(result["skipped"]), 1)
89
        rows[-1] = conformance_row("path_case_distinct_v1", 77, skip_reason="leg unmeasured: reader bound")
90
        self.assertEqual(gate.evaluate(rows, self.expected, ALL)["verdict"], "UNMEASURED")
91
92
    def test_git_rows_never_gate(self):
93
        rows = healthy_rows(self.expected) + [conformance_row("eol_lf_v1", 1, ["clone_fs"], subject="git")]
94
        result = gate.evaluate(rows, self.expected, ALL)
95
        self.assertEqual(result["verdict"], "PASS")
96
        self.assertEqual(len(result["git_reference_failures"]), 1)
97
98
    def test_list_validation_requires_ticket_and_legs(self):
99
        with tempfile.TemporaryDirectory() as tmp:
100
            path = Path(tmp) / "x.json"
101
            for entry in ({"behaviour": "a", "failed_legs": ["clone_fs"]},
102
                          {"behaviour": "a", "ticket": "TODO", "failed_legs": ["clone_fs"]},
103
                          {"behaviour": "a", "ticket": "fb-1"}):
104
                path.write_text(json.dumps({"schema_version": 1, "entries": [entry]}))
105
                with self.assertRaises(ValueError):
106
                    gate.load_expected(path)
107
108
109
class WiringTests(unittest.TestCase):
110
    def setUp(self):
111
        self.expected = gate.load_expected()
112
113
    def test_devloop_lane_is_default_and_maps_verdicts(self):
114
        self.assertIn("conformance", devloop.LANES)
115
        regressions, notes = devloop.conformance_checks(healthy_rows(self.expected))
116
        self.assertEqual(regressions, [])
117
        self.assertFalse(any(n.startswith("UNMEASURED") for n in notes))
118
        rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "eol_lf_v1"]
119
        regressions, notes = devloop.conformance_checks(rows + [conformance_row("eol_lf_v1", 1, ["pull_fs"])])
120
        self.assertTrue(regressions and regressions[0].startswith("[conformance] CONFORMANCE eol_lf_v1"))
121
        regressions, notes = devloop.conformance_checks(rows)
122
        self.assertTrue(any(n.startswith("UNMEASURED conformance") for n in notes))
123
124
    def gate_code(self, rows):
125
        with tempfile.TemporaryDirectory() as tmp:
126
            path = Path(tmp) / "rows.jsonl"
127
            path.write_text("".join(json.dumps(r) + "\n" for r in rows))
128
            with contextlib.redirect_stdout(io.StringIO()) as out:
129
                code = publish_gate.main([str(path)])
130
            return code, out.getvalue()
131
132
    def test_publish_gate_requires_and_checks_conformance_for_oak_rows(self):
133
        oak_row = {"subject": "oak_installed", "binary_sha256": "b" * 64, "returncode": 0,
134
                   "source": {"oak_hash": "abc", "oak_status": "clean"}}
135
        code, out = self.gate_code([oak_row])
136
        self.assertIn("FAIL   conformance: Oak rows published without", out)
137
        code, out = self.gate_code([oak_row, *healthy_rows(self.expected)])
138
        self.assertIn("PASS   conformance (oak): PASS", out)
139
        stale = [r for r in healthy_rows(self.expected) if r["scenario"] != "path_tab_v1"]
140
        code, out = self.gate_code([oak_row, *stale, conformance_row("path_tab_v1", 0)])
141
        self.assertIn("expected-failures list is stale", out)
142
        other = dict(oak_row, binary_sha256="c" * 64)
143
        code, out = self.gate_code([other, *healthy_rows(self.expected)])
144
        self.assertIn("published Oak binaries lack conformance rows", out)
145
146
147
if __name__ == "__main__":
148
    unittest.main()