Viewing files at commit 3cd2661fdbe9Commit detailsView latest

Browse files
tests/test_lane_contract.py 315 lines · 13.9 KB · python
1
"""Lane-contract conformance: every lane, five checks, positive and negative.
2
3
Parameterized (subTest) over lane_contract.LANE_TABLE. For each lane the
4
valid synthetic rows (mirroring the lane's real emitters) must pass all five
5
checks, and a crafted violation must fail each check that applies — negative
6
tests prove the checks have teeth. The meta-test guarantees a new lane cannot
7
ship without a LANE_TABLE entry.
8
"""
9
10
from __future__ import annotations
11
12
import copy
13
import sys
14
from pathlib import Path
15
16
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
17
18
import unittest
19
20
from oakbench import lane_contract, remotes, rows as oakbench_rows
21
from oakbench.lane_contract import (
22
    DE_FACTO_LANES,
23
    LANE_TABLE,
24
    catalog_operations,
25
    check_catalog_membership,
26
    check_measurement_source,
27
    check_row_contract,
28
    check_skip_coherence,
29
    check_transport_purity,
30
    expected_lanes,
31
)
32
33
34
def sample_rows(lane: str) -> list[dict]:
35
    """Fresh deep copy so one subTest's mutation never leaks into another."""
36
    return copy.deepcopy(LANE_TABLE[lane].sample_rows_builder())
37
38
39
def catalog_errors(contract: lane_contract.LaneContract, rows: list[dict]) -> list[str]:
40
    catalog = catalog_operations(contract.catalog_path) if contract.catalog_path else None
41
    return check_catalog_membership(
42
        rows,
43
        catalog,
44
        exempt_ops=contract.catalog_exempt_ops,
45
        exempt_prefixes=contract.catalog_exempt_prefixes,
46
        exempt_suffixes=contract.catalog_exempt_suffixes,
47
    )
48
49
50
class LaneTableMetaTest(unittest.TestCase):
51
    """Adding a lane without a contract entry fails CI with a clear message."""
52
53
    def test_every_lane_has_a_contract_entry(self) -> None:
54
        missing = sorted(expected_lanes() - set(LANE_TABLE))
55
        self.assertEqual(
56
            missing,
57
            [],
58
            "New lane(s) without a LANE_TABLE entry in scripts/oakbench/lane_contract.py: "
59
            f"{missing}. Add one LaneContract per lane (see docs/new-lane-template.md) "
60
            "so row contracts, skip coherence, transport purity, measurement_source, "
61
            "and catalog membership stay machine-checked.",
62
        )
63
64
    def test_no_orphan_contract_entries(self) -> None:
65
        orphans = sorted(set(LANE_TABLE) - expected_lanes())
66
        self.assertEqual(
67
            orphans,
68
            [],
69
            f"LANE_TABLE entries for unknown lanes: {orphans}. Either add the lane to "
70
            "rows.LANES, list it in lane_contract.DE_FACTO_LANES, or delete the entry.",
71
        )
72
73
    def test_rows_lanes_all_validatable(self) -> None:
74
        # Every rows.py lane delegates to validate_rows under its own key.
75
        for lane in oakbench_rows.LANES:
76
            with self.subTest(lane=lane):
77
                self.assertEqual(LANE_TABLE[lane].validate_lane, lane)
78
79
    def test_de_facto_lanes_documented(self) -> None:
80
        self.assertEqual(set(DE_FACTO_LANES), {"netshape", "external"})
81
        # netshape opts into the core key contract (netshape_bench.write_rows).
82
        self.assertEqual(LANE_TABLE["netshape"].validate_lane, "core")
83
        # external (tperf) rows are not bench rows; universal keys only.
84
        self.assertIsNone(LANE_TABLE["external"].validate_lane)
85
        self.assertTrue(LANE_TABLE["external"].universal_keys)
86
87
88
class ValidRowsPassAllChecksTest(unittest.TestCase):
89
    def test_all_lanes(self) -> None:
90
        for lane, contract in LANE_TABLE.items():
91
            rows = sample_rows(lane)
92
            with self.subTest(lane=lane, check="row_contract"):
93
                self.assertEqual(check_row_contract(rows, contract), [])
94
            with self.subTest(lane=lane, check="skip_coherence"):
95
                self.assertEqual(check_skip_coherence(rows), [])
96
            with self.subTest(lane=lane, check="transport_purity"):
97
                self.assertEqual(check_transport_purity(rows), [])
98
            with self.subTest(lane=lane, check="measurement_source"):
99
                self.assertEqual(
100
                    check_measurement_source(
101
                        rows, contract.allowed_sources, required=contract.source_required
102
                    ),
103
                    [],
104
                )
105
            with self.subTest(lane=lane, check="catalog_membership"):
106
                self.assertEqual(catalog_errors(contract, rows), [])
107
108
109
class RowContractNegativeTest(unittest.TestCase):
110
    def test_missing_required_key_fails(self) -> None:
111
        for lane, contract in LANE_TABLE.items():
112
            with self.subTest(lane=lane):
113
                rows = sample_rows(lane)
114
                if contract.validate_lane is not None:
115
                    victim = oakbench_rows.LANE_REQUIRED_KEYS[contract.validate_lane][0]
116
                else:
117
                    victim = contract.universal_keys[0]
118
                del rows[0][victim]
119
                errors = check_row_contract(rows, contract)
120
                self.assertTrue(errors, f"deleting {victim!r} must fail the row contract")
121
                self.assertIn(victim, "\n".join(errors))
122
123
    def test_non_numeric_elapsed_fails(self) -> None:
124
        for lane, contract in LANE_TABLE.items():
125
            with self.subTest(lane=lane):
126
                rows = sample_rows(lane)
127
                rows[0]["elapsed_ms"] = "fast"
128
                self.assertTrue(check_row_contract(rows, contract))
129
130
131
class SkipCoherenceNegativeTest(unittest.TestCase):
132
    def test_77_without_reason_fails(self) -> None:
133
        for lane in LANE_TABLE:
134
            with self.subTest(lane=lane):
135
                rows = sample_rows(lane)
136
                bad = copy.deepcopy(rows[0])
137
                bad["returncode"] = oakbench_rows.SKIP_RETURNCODE
138
                bad.pop("skipped", None)
139
                bad.pop("skip_reason", None)
140
                errors = check_skip_coherence(rows + [bad])
141
                self.assertTrue(any("skipped: true" in e for e in errors))
142
                self.assertTrue(any("skip_reason" in e for e in errors))
143
144
    def test_skipped_without_77_fails(self) -> None:
145
        for lane in LANE_TABLE:
146
            with self.subTest(lane=lane):
147
                rows = sample_rows(lane)
148
                bad = copy.deepcopy(rows[0])
149
                bad["returncode"] = 0
150
                bad["skipped"] = True
151
                bad["skip_reason"] = "half-marked skip"
152
                errors = check_skip_coherence(rows + [bad])
153
                self.assertTrue(any("requires returncode 77" in e for e in errors))
154
155
156
class TransportPurityNegativeTest(unittest.TestCase):
157
    def test_mixed_transports_same_subject_scenario_operation_fail(self) -> None:
158
        for lane in LANE_TABLE:
159
            with self.subTest(lane=lane):
160
                rows = sample_rows(lane)
161
                a = copy.deepcopy(rows[0])
162
                b = copy.deepcopy(rows[0])
163
                a.update(
164
                    remote_repo="x",
165
                    remote_transport=remotes.TRANSPORT_NETWORK,
166
                    remote_server="oak.space",
167
                )
168
                a.pop("netshape_profile", None)
169
                b.update(
170
                    remote_repo="y",
171
                    remote_transport=remotes.TRANSPORT_LOCAL_FILE,
172
                    remote_server="local",
173
                )
174
                b.pop("netshape_profile", None)
175
                errors = check_transport_purity(rows + [a, b])
176
                self.assertTrue(any("transport mixing" in e for e in errors))
177
178
    def test_same_operation_different_subjects_is_legal(self) -> None:
179
        # Side-by-side files may hold one operation at different transports
180
        # for different subjects (publish_gate precedent).
181
        rows = sample_rows("core")
182
        a = {**copy.deepcopy(rows[0]), "subject": "git", "remote_repo": "x",
183
             "remote_transport": remotes.TRANSPORT_LOCAL_FILE, "remote_server": "local"}
184
        b = {**copy.deepcopy(rows[0]), "subject": "oak", "remote_repo": "y",
185
             "remote_transport": remotes.TRANSPORT_NETWORK, "remote_server": "oak.space"}
186
        self.assertEqual(check_transport_purity(rows + [a, b]), [])
187
188
    def test_remote_row_without_transport_fails(self) -> None:
189
        rows = sample_rows("core")
190
        bad = copy.deepcopy(rows[0])
191
        bad["remote_repo"] = "oak/bench-tmp"
192
        bad.pop("remote_transport", None)
193
        errors = check_transport_purity(rows + [bad])
194
        self.assertTrue(any("without remote_transport" in e for e in errors))
195
196
    def test_unknown_transport_fails(self) -> None:
197
        rows = sample_rows("core")
198
        bad = copy.deepcopy(rows[0])
199
        bad.update(remote_repo="x", remote_transport="carrier_pigeon", remote_server="loft")
200
        errors = check_transport_purity(rows + [bad])
201
        self.assertTrue(any("unknown transport" in e for e in errors))
202
203
    def test_shaped_row_without_profile_fails(self) -> None:
204
        rows = sample_rows("netshape")
205
        bad = copy.deepcopy(rows[0])
206
        bad["netshape_profile"] = ""
207
        errors = check_transport_purity([bad])
208
        self.assertTrue(any("netshape_profile" in e for e in errors))
209
210
211
class MeasurementSourceNegativeTest(unittest.TestCase):
212
    def test_off_list_source_fails_or_lane_is_exempt(self) -> None:
213
        for lane, contract in LANE_TABLE.items():
214
            with self.subTest(lane=lane):
215
                rows = sample_rows(lane)
216
                rows[0]["measurement_source"] = "made_up_source"
217
                errors = check_measurement_source(
218
                    rows, contract.allowed_sources, required=contract.source_required
219
                )
220
                if contract.allowed_sources is None:
221
                    # Encoded reality: agent/contention nest their sources in
222
                    # metric blocks; mount/netshape carry none. The top-level
223
                    # check is exempt for them, by the table, not by accident.
224
                    self.assertEqual(errors, [])
225
                else:
226
                    self.assertTrue(any("made_up_source" in e for e in errors))
227
228
    def test_required_source_missing_fails(self) -> None:
229
        contract = LANE_TABLE["external"]
230
        rows = sample_rows("external")
231
        del rows[0]["measurement_source"]
232
        errors = check_measurement_source(rows, contract.allowed_sources, required=True)
233
        self.assertTrue(any("missing required measurement_source" in e for e in errors))
234
235
236
class CatalogMembershipNegativeTest(unittest.TestCase):
237
    def test_off_catalog_operation_fails_or_lane_is_exempt(self) -> None:
238
        for lane, contract in LANE_TABLE.items():
239
            with self.subTest(lane=lane):
240
                rows = sample_rows(lane)
241
                bad = copy.deepcopy(rows[0])
242
                bad["operation"] = "made.up.operation"
243
                errors = catalog_errors(contract, rows + [bad])
244
                if contract.catalog_path is None:
245
                    # Encoded reality: workflow/contention/agent/task-loop/
246
                    # netshape/external declare no operation catalog today.
247
                    self.assertEqual(errors, [])
248
                else:
249
                    self.assertTrue(any("made.up.operation" in e for e in errors))
250
251
    def test_catalogs_parse_and_are_nonempty(self) -> None:
252
        for lane, contract in LANE_TABLE.items():
253
            if contract.catalog_path is None:
254
                continue
255
            with self.subTest(lane=lane):
256
                ops = catalog_operations(contract.catalog_path)
257
                self.assertTrue(ops, f"{contract.catalog_path} parsed to an empty catalog")
258
                self.assertTrue(all("." in op or op.isidentifier() for op in ops), ops)
259
260
    def test_core_catalog_contains_known_operations(self) -> None:
261
        ops = catalog_operations(LANE_TABLE["core"].catalog_path)
262
        for op in ("repo.init", "status.clean", "merge.conflict"):
263
            self.assertIn(op, ops)
264
265
    def test_exemptions_are_documented_reality_not_wildcards(self) -> None:
266
        # The core exemption families mirror bench.py's own diagnostic-row
267
        # exclusions (average_by_subject); keep them in sync deliberately.
268
        core = LANE_TABLE["core"]
269
        self.assertIn("proc.spawn", core.catalog_exempt_ops)
270
        self.assertIn("remote.", core.catalog_exempt_prefixes)
271
        mount = LANE_TABLE["mount"]
272
        self.assertIn("acquire.", mount.catalog_exempt_prefixes)
273
274
275
class SampleRowsMirrorRealEmittersTest(unittest.TestCase):
276
    """Spot-check the synthetic rows against the real emitters' fixed points
277
    so the table cannot drift into testing an imaginary suite."""
278
279
    def test_core_sample_passes_rows_py_directly(self) -> None:
280
        self.assertEqual(oakbench_rows.validate_rows(sample_rows("core"), "core"), [])
281
282
    def test_netshape_sample_carries_shaped_pipe_identity(self) -> None:
283
        for row in sample_rows("netshape"):
284
            self.assertEqual(row["remote_transport"], remotes.TRANSPORT_NETWORK_SHAPED)
285
            self.assertTrue(row["netshape_profile"])
286
            self.assertTrue(row["remote_server"])
287
288
    def test_netshape_shaped_rows_compare_legally_same_pipe_only(self) -> None:
289
        rows = sample_rows("netshape")
290
        self.assertTrue(remotes.shaped_comparison_legal(rows[0], rows[1]))
291
        other_pipe = {**rows[1], "netshape_profile": "wan-80ms"}
292
        self.assertFalse(remotes.shaped_comparison_legal(rows[0], other_pipe))
293
294
    def test_external_sample_matches_tperf_runner_source_label(self) -> None:
295
        import tperf_runner
296
297
        self.assertEqual(
298
            LANE_TABLE["external"].allowed_sources,
299
            frozenset({tperf_runner.MEASUREMENT_SOURCE}),
300
        )
301
        for row in sample_rows("external"):
302
            self.assertEqual(row["measurement_source"], tperf_runner.MEASUREMENT_SOURCE)
303
304
    def test_skip_rows_present_in_lanes_that_emit_them(self) -> None:
305
        for lane in ("core", "workflow", "mount", "platform", "netshape"):
306
            with self.subTest(lane=lane):
307
                skips = [r for r in sample_rows(lane) if r.get("skipped")]
308
                self.assertTrue(skips, f"{lane} sample must include a skip row")
309
                for row in skips:
310
                    self.assertEqual(row["returncode"], oakbench_rows.SKIP_RETURNCODE)
311
                    self.assertTrue(str(row["skip_reason"]).strip())
312
313
314
if __name__ == "__main__":
315
    unittest.main()