Viewing files at commit 369435870869Commit detailsView latest

Browse files
tests/test_robustness_runner.py 152 lines · 8.2 KB · python
1
"""Robustness runner: row contract, classification, readers, and a real Git case."""
2
3
from __future__ import annotations
4
5
import json
6
import shutil
7
import sqlite3
8
import subprocess
9
import sys
10
import tempfile
11
import unittest
12
from pathlib import Path
13
14
ROOT = Path(__file__).resolve().parents[1]
15
sys.path.insert(0, str(ROOT / "suites" / "robustness"))
16
sys.path.insert(0, str(ROOT / "scripts"))
17
18
import run_robustness  # noqa: E402
19
from robustness import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, UNSUPPORTED, VIOLATED, known_bugs  # noqa: E402
20
from robustness.engine import actionable, classify  # noqa: E402
21
from robustness.hosted import configured_url  # noqa: E402
22
from robustness.procs import Result  # noqa: E402
23
from robustness.readers import OakGraph  # noqa: E402
24
25
from oakbench.lane_contract import LANE_TABLE  # noqa: E402
26
from oakbench.rows import validate_rows  # noqa: E402
27
from oakbench.subjects import which_measured_git  # noqa: E402
28
29
GIT = which_measured_git()
30
31
32
def result(rc, stdout="", stderr=""):
33
    return Result(["x"], rc, stdout, stderr, 1.0)
34
35
36
class ClassificationTests(unittest.TestCase):
37
    def test_classify(self):
38
        self.assertEqual(classify({"a": True, "b": None}), HELD)
39
        self.assertEqual(classify({"a": True, "b": False}), VIOLATED)
40
        self.assertEqual(classify({"a": True}, harness_error="boom"), HARNESS_ERROR)
41
42
    def test_actionable_errors(self):
43
        self.assertTrue(actionable(result(4, stderr="error: run `oak pull --abort` to recover")))
44
        self.assertTrue(actionable(result(6, stdout=json.dumps({"error": {"code": "remote_diverged", "message": "x"}}))))
45
        self.assertTrue(actionable(result(5, stderr="error: Conflict detected: remote has diverged. Pull changes first.")))
46
        self.assertFalse(actionable(result(4, stderr="error: Working directory has uncommitted changes")))
47
        self.assertFalse(actionable(result(101, stderr="thread 'main' panicked at src/x.rs:1")))
48
        self.assertFalse(actionable(result(-9)))
49
        self.assertFalse(actionable(result(128, stderr="fatal: Unable to write index.")))
50
51
    def test_hosted_refuses_non_loopback(self):
52
        import os
53
        from unittest import mock
54
55
        with mock.patch.dict(os.environ, {"OAK_BENCH_ROBUSTNESS_OAKSPACE_URL": "https://oak.space"}):
56
            url, reason = configured_url()
57
        self.assertIsNone(url)
58
        self.assertIn("refusing non-loopback", reason)
59
60
61
class OakGraphReaderTests(unittest.TestCase):
62
    def test_reachability_from_refs_on_a_crashed_wal_store(self):
63
        with tempfile.TemporaryDirectory() as temp:
64
            db = Path(temp) / "oak.db"
65
            conn = sqlite3.connect(db)
66
            conn.execute("PRAGMA journal_mode=WAL")
67
            conn.execute("CREATE TABLE metadata(key TEXT PRIMARY KEY, value TEXT)")
68
            conn.execute("CREATE TABLE branch_heads(branch_name TEXT PRIMARY KEY, head_hash TEXT)")
69
            conn.execute("CREATE TABLE commits(hash TEXT PRIMARY KEY, parent_hash TEXT, merge_parent_hash TEXT)")
70
            conn.executemany("INSERT INTO commits VALUES (?,?,?)",
71
                             [("c1", None, None), ("c2", "c1", None), ("orphan", "c1", None)])
72
            conn.execute("INSERT INTO branch_heads VALUES ('b', 'c2')")
73
            conn.execute("INSERT INTO metadata VALUES ('head', 'c2'), ('current_branch', 'b')")
74
            conn.commit()  # left in the WAL (connection still open) like a killed writer
75
            with OakGraph(db) as graph:
76
                self.assertEqual(graph.integrity(), "ok")
77
                self.assertEqual(graph.reachable(), {"c1", "c2"})
78
                self.assertNotIn("orphan", graph.reachable())
79
                self.assertTrue(graph.is_ancestor("c1", "c2"))
80
            conn.close()
81
82
83
class RowContractTests(unittest.TestCase):
84
    def _runner(self, temp: Path, extra: list[str] | None = None):
85
        args = run_robustness.parse_args(["--subjects", "git", "--results", str(temp / "results"),
86
                                          "--workdir", str(temp / "work"), "--bench-id", "20260928T000000Z",
87
                                          "--git-bin", GIT or "git", *(extra or [])])
88
        runner = run_robustness.Runner(args)
89
        runner.versions = {"git": "git version test", "oak": "oak test"}
90
        return runner
91
92
    def test_emitted_rows_satisfy_the_robustness_lane_contract(self):
93
        with tempfile.TemporaryDirectory() as temp:
94
            runner = self._runner(Path(temp))
95
            runner.emit("git", "crash_time_v1", "commit", "crash_time",
96
                        {"classification": HELD, "invariants": {"repo_opens": True}, "fault_point": "t=1ms",
97
                         "elapsed_ms": 3.0, "remote_transport": "local_file", "remote_server": "local",
98
                         "remote_repo": "../srv/remote.git"})
99
            runner.emit("oak", "crash_time_v1", "merge_hosted", "crash_time", {"unsupported": "404 on serve"})
100
            runner.emit("git", "crash_time_v1", "commit", "crash_time",
101
                        {"classification": VIOLATED, "invariants": {"repo_opens": False}, "fault_point": "t=2ms"})
102
            rows = runner.rows
103
            self.assertEqual(validate_rows(rows, "robustness"), [])
104
            self.assertEqual([r["returncode"] for r in rows], [0, 77, 1])
105
            self.assertEqual(rows[1]["classification"], UNSUPPORTED)
106
            self.assertTrue(rows[1]["skipped"])
107
            self.assertEqual(rows[2]["violations"], ["repo_opens"])
108
            self.assertEqual([r["run"] for r in rows], [0, 0, 1])
109
            contract = LANE_TABLE["robustness"]
110
            from oakbench.lane_contract import (catalog_operations, check_catalog_membership,
111
                                                check_measurement_source, check_transport_purity)
112
            self.assertEqual(check_measurement_source(rows, contract.allowed_sources), [])
113
            self.assertEqual(check_transport_purity(rows), [])
114
            self.assertEqual(check_catalog_membership(rows, catalog_operations(contract.catalog_path)), [])
115
            partial = Path(temp) / "results" / "20260928T000000Z.robustness.partial.jsonl"
116
            self.assertEqual(len(partial.read_text().splitlines()), 3)
117
118
    def test_summary_table_counts_classes(self):
119
        rows = [{"subject_kind": "oak", "scenario": "s", "operation": "push", "classification": c,
120
                 "violations": ["x"] if c == VIOLATED else [], "fault_point": "p"}
121
                for c in (HELD, HELD, VIOLATED, UNSUPPORTED)]
122
        text = run_robustness.summarize(rows, "id", 60)
123
        self.assertIn("| oak | s | push | 2 | 1 | 0 | 1 | 0 |", text)
124
        self.assertIn("## Violations", text)
125
126
127
@unittest.skipUnless(GIT and shutil.which(GIT), "real git required")
128
class GitEndToEndTests(unittest.TestCase):
129
    def test_time_crash_on_git_commit_is_judged_and_written(self):
130
        with tempfile.TemporaryDirectory() as temp:
131
            args = run_robustness.parse_args(["--subjects", "git", "--families", "crash_time,hosted", "--ops",
132
                                              "commit,merge_hosted", "--quick", "--results", str(Path(temp) / "r"),
133
                                              "--workdir", str(Path(temp) / "w"), "--bench-id", "20260928T000001Z",
134
                                              "--git-bin", GIT])
135
            code = run_robustness.Runner(args).run()
136
            self.assertEqual(code, 0)
137
            rows = [json.loads(line) for line in
138
                    (Path(temp) / "r" / "20260928T000001Z.robustness.jsonl").read_text().splitlines()]
139
            self.assertEqual(validate_rows(rows, "robustness"), [])
140
            commit_rows = [r for r in rows if r["operation"] == "commit"]
141
            self.assertEqual(len(commit_rows), 3)
142
            for row in commit_rows:
143
                # a commit that beats every re-aimed kill (host load) is fault_not_fired, never held
144
                self.assertIn(row["classification"], (HELD, VIOLATED, FAULT_NOT_FIRED), row.get("harness_error"))
145
                self.assertEqual(row["classification"] == FAULT_NOT_FIRED, row["fault_fired"] is False)
146
                self.assertIn("acked_local_reachable", row["invariants"])
147
                self.assertEqual(row["remote_transport"], "local_file")
148
            self.assertTrue(all(r["classification"] == UNSUPPORTED for r in rows if r["operation"] == "merge_hosted"))
149
150
151
if __name__ == "__main__":
152
    unittest.main()