| 1 | """Conformance gate semantics and its wiring into devloop and publish_gate."""
|
| 2 | import contextlib
|
| 3 | import io
|
| 4 | import json
|
| 5 | from pathlib import Path
|
| 6 | import sys
|
| 7 | import tempfile
|
| 8 | import unittest
|
| 9 |
|
| 10 | ROOT = Path(__file__).resolve().parents[1]
|
| 11 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 12 | import devloop # noqa: E402
|
| 13 | import publish_gate # noqa: E402
|
| 14 | from oakbench import conformance_gate as gate # noqa: E402
|
| 15 | from oakbench.conformance import SCENARIOS # noqa: E402
|
| 16 | from oakbench.conformance_matrix import BEHAVIOURS # noqa: E402
|
| 17 |
|
| 18 | ALL = list(SCENARIOS) + [b.id for b in BEHAVIOURS]
|
| 19 |
|
| 20 |
|
| 21 | def conformance_row(behaviour, code=0, legs=None, subject="oak", **extra):
|
| 22 | row = {"evidence_class": "diagnostic_conformance", "subject": subject, "scenario": behaviour,
|
| 23 | "returncode": code, "binary_sha256": "b" * 64, **extra}
|
| 24 | if legs is not None:
|
| 25 | row["failed_legs"] = legs
|
| 26 | if code == 1:
|
| 27 | row["failure"] = f"{(legs or ['behaviour'])[0]}: injected"
|
| 28 | if code == 77:
|
| 29 | row.setdefault("skip_reason", "filesystem capability unavailable: case_sensitive_fs")
|
| 30 | return row
|
| 31 |
|
| 32 |
|
| 33 | def healthy_rows(expected, subject="oak"):
|
| 34 | """Rows matching today's expected failures exactly: the gate's PASS state."""
|
| 35 | rows = []
|
| 36 | for behaviour in ALL:
|
| 37 | entry = expected["entries"].get(behaviour)
|
| 38 | if entry:
|
| 39 | legs = None if behaviour in SCENARIOS else entry["failed_legs"]
|
| 40 | rows.append(conformance_row(behaviour, 1, legs, subject))
|
| 41 | else:
|
| 42 | rows.append(conformance_row(behaviour, 0, None, subject))
|
| 43 | return rows
|
| 44 |
|
| 45 |
|
| 46 | class EvaluateTests(unittest.TestCase):
|
| 47 | def setUp(self):
|
| 48 | self.expected = gate.load_expected()
|
| 49 |
|
| 50 | def test_matching_expected_failures_pass(self):
|
| 51 | result = gate.evaluate(healthy_rows(self.expected), self.expected, ALL)
|
| 52 | self.assertEqual(result["verdict"], "PASS", result)
|
| 53 | self.assertEqual(len(result["expected_failures"]), len(self.expected["entries"]))
|
| 54 |
|
| 55 | def test_new_failure_is_a_regression(self):
|
| 56 | rows = healthy_rows(self.expected)
|
| 57 | rows = [r for r in rows if r["scenario"] != "eol_crlf_v1"] + [conformance_row("eol_crlf_v1", 1, ["clone_fs"])]
|
| 58 | result = gate.evaluate(rows, self.expected, ALL)
|
| 59 | self.assertEqual(result["verdict"], "REGRESSED")
|
| 60 | self.assertIn("eol_crlf_v1", result["regressions"][0])
|
| 61 |
|
| 62 | def test_known_failure_changing_shape_is_a_regression(self):
|
| 63 | rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "path_tab_v1"]
|
| 64 | rows.append(conformance_row("path_tab_v1", 1, ["clone_fs", "clone_read"]))
|
| 65 | result = gate.evaluate(rows, self.expected, ALL)
|
| 66 | self.assertEqual(result["verdict"], "REGRESSED")
|
| 67 | self.assertIn("fb-531", result["regressions"][0])
|
| 68 |
|
| 69 | def test_fix_flips_to_fixed_without_regression(self):
|
| 70 | rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "path_tab_v1"]
|
| 71 | rows.append(conformance_row("path_tab_v1", 0))
|
| 72 | result = gate.evaluate(rows, self.expected, ALL)
|
| 73 | self.assertEqual(result["verdict"], "PASS")
|
| 74 | self.assertIn("fb-531", result["fixed"][0])
|
| 75 |
|
| 76 | def test_missing_rows_and_harness_errors_are_unmeasured(self):
|
| 77 | rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "eol_lf_v1"]
|
| 78 | self.assertEqual(gate.evaluate(rows, self.expected, ALL)["verdict"], "UNMEASURED")
|
| 79 | rows.append(conformance_row("eol_lf_v1", 2, failure="harness error"))
|
| 80 | self.assertEqual(gate.evaluate(rows, self.expected, ALL)["verdict"], "UNMEASURED")
|
| 81 | self.assertEqual(gate.evaluate([], self.expected)["verdict"], "UNMEASURED")
|
| 82 |
|
| 83 | def test_capability_skips_are_recorded_not_unmeasured_but_reader_skips_are(self):
|
| 84 | rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "path_case_distinct_v1"]
|
| 85 | rows.append(conformance_row("path_case_distinct_v1", 77))
|
| 86 | result = gate.evaluate(rows, self.expected, ALL)
|
| 87 | self.assertEqual(result["verdict"], "PASS")
|
| 88 | self.assertEqual(len(result["skipped"]), 1)
|
| 89 | rows[-1] = conformance_row("path_case_distinct_v1", 77, skip_reason="leg unmeasured: reader bound")
|
| 90 | self.assertEqual(gate.evaluate(rows, self.expected, ALL)["verdict"], "UNMEASURED")
|
| 91 |
|
| 92 | def test_git_rows_never_gate(self):
|
| 93 | rows = healthy_rows(self.expected) + [conformance_row("eol_lf_v1", 1, ["clone_fs"], subject="git")]
|
| 94 | result = gate.evaluate(rows, self.expected, ALL)
|
| 95 | self.assertEqual(result["verdict"], "PASS")
|
| 96 | self.assertEqual(len(result["git_reference_failures"]), 1)
|
| 97 |
|
| 98 | def test_list_validation_requires_ticket_and_legs(self):
|
| 99 | with tempfile.TemporaryDirectory() as tmp:
|
| 100 | path = Path(tmp) / "x.json"
|
| 101 | for entry in ({"behaviour": "a", "failed_legs": ["clone_fs"]},
|
| 102 | {"behaviour": "a", "ticket": "TODO", "failed_legs": ["clone_fs"]},
|
| 103 | {"behaviour": "a", "ticket": "fb-1"}):
|
| 104 | path.write_text(json.dumps({"schema_version": 1, "entries": [entry]}))
|
| 105 | with self.assertRaises(ValueError):
|
| 106 | gate.load_expected(path)
|
| 107 |
|
| 108 |
|
| 109 | class WiringTests(unittest.TestCase):
|
| 110 | def setUp(self):
|
| 111 | self.expected = gate.load_expected()
|
| 112 |
|
| 113 | def test_devloop_lane_is_default_and_maps_verdicts(self):
|
| 114 | self.assertIn("conformance", devloop.LANES)
|
| 115 | regressions, notes = devloop.conformance_checks(healthy_rows(self.expected))
|
| 116 | self.assertEqual(regressions, [])
|
| 117 | self.assertFalse(any(n.startswith("UNMEASURED") for n in notes))
|
| 118 | rows = [r for r in healthy_rows(self.expected) if r["scenario"] != "eol_lf_v1"]
|
| 119 | regressions, notes = devloop.conformance_checks(rows + [conformance_row("eol_lf_v1", 1, ["pull_fs"])])
|
| 120 | self.assertTrue(regressions and regressions[0].startswith("[conformance] CONFORMANCE eol_lf_v1"))
|
| 121 | regressions, notes = devloop.conformance_checks(rows)
|
| 122 | self.assertTrue(any(n.startswith("UNMEASURED conformance") for n in notes))
|
| 123 |
|
| 124 | def gate_code(self, rows):
|
| 125 | with tempfile.TemporaryDirectory() as tmp:
|
| 126 | path = Path(tmp) / "rows.jsonl"
|
| 127 | path.write_text("".join(json.dumps(r) + "\n" for r in rows))
|
| 128 | with contextlib.redirect_stdout(io.StringIO()) as out:
|
| 129 | code = publish_gate.main([str(path)])
|
| 130 | return code, out.getvalue()
|
| 131 |
|
| 132 | def test_publish_gate_requires_and_checks_conformance_for_oak_rows(self):
|
| 133 | oak_row = {"subject": "oak_installed", "binary_sha256": "b" * 64, "returncode": 0,
|
| 134 | "source": {"oak_hash": "abc", "oak_status": "clean"}}
|
| 135 | code, out = self.gate_code([oak_row])
|
| 136 | self.assertIn("FAIL conformance: Oak rows published without", out)
|
| 137 | code, out = self.gate_code([oak_row, *healthy_rows(self.expected)])
|
| 138 | self.assertIn("PASS conformance (oak): PASS", out)
|
| 139 | stale = [r for r in healthy_rows(self.expected) if r["scenario"] != "path_tab_v1"]
|
| 140 | code, out = self.gate_code([oak_row, *stale, conformance_row("path_tab_v1", 0)])
|
| 141 | self.assertIn("expected-failures list is stale", out)
|
| 142 | other = dict(oak_row, binary_sha256="c" * 64)
|
| 143 | code, out = self.gate_code([other, *healthy_rows(self.expected)])
|
| 144 | self.assertIn("published Oak binaries lack conformance rows", out)
|
| 145 |
|
| 146 |
|
| 147 | if __name__ == "__main__":
|
| 148 | unittest.main()
|