| 1 | """Robustness runner: row contract, classification, readers, and a real Git case."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import json
|
| 6 | import shutil
|
| 7 | import sqlite3
|
| 8 | import subprocess
|
| 9 | import sys
|
| 10 | import tempfile
|
| 11 | import unittest
|
| 12 | from pathlib import Path
|
| 13 |
|
| 14 | ROOT = Path(__file__).resolve().parents[1]
|
| 15 | sys.path.insert(0, str(ROOT / "suites" / "robustness"))
|
| 16 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 17 |
|
| 18 | import run_robustness # noqa: E402
|
| 19 | from robustness import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, UNSUPPORTED, VIOLATED, known_bugs # noqa: E402
|
| 20 | from robustness.engine import actionable, classify # noqa: E402
|
| 21 | from robustness.hosted import configured_url # noqa: E402
|
| 22 | from robustness.procs import Result # noqa: E402
|
| 23 | from robustness.readers import OakGraph # noqa: E402
|
| 24 |
|
| 25 | from oakbench.lane_contract import LANE_TABLE # noqa: E402
|
| 26 | from oakbench.rows import validate_rows # noqa: E402
|
| 27 | from oakbench.subjects import which_measured_git # noqa: E402
|
| 28 |
|
| 29 | GIT = which_measured_git()
|
| 30 |
|
| 31 |
|
| 32 | def result(rc, stdout="", stderr=""):
|
| 33 | return Result(["x"], rc, stdout, stderr, 1.0)
|
| 34 |
|
| 35 |
|
| 36 | class ClassificationTests(unittest.TestCase):
|
| 37 | def test_classify(self):
|
| 38 | self.assertEqual(classify({"a": True, "b": None}), HELD)
|
| 39 | self.assertEqual(classify({"a": True, "b": False}), VIOLATED)
|
| 40 | self.assertEqual(classify({"a": True}, harness_error="boom"), HARNESS_ERROR)
|
| 41 |
|
| 42 | def test_actionable_errors(self):
|
| 43 | self.assertTrue(actionable(result(4, stderr="error: run `oak pull --abort` to recover")))
|
| 44 | self.assertTrue(actionable(result(6, stdout=json.dumps({"error": {"code": "remote_diverged", "message": "x"}}))))
|
| 45 | self.assertTrue(actionable(result(5, stderr="error: Conflict detected: remote has diverged. Pull changes first.")))
|
| 46 | self.assertFalse(actionable(result(4, stderr="error: Working directory has uncommitted changes")))
|
| 47 | self.assertFalse(actionable(result(101, stderr="thread 'main' panicked at src/x.rs:1")))
|
| 48 | self.assertFalse(actionable(result(-9)))
|
| 49 | self.assertFalse(actionable(result(128, stderr="fatal: Unable to write index.")))
|
| 50 |
|
| 51 | def test_hosted_refuses_non_loopback(self):
|
| 52 | import os
|
| 53 | from unittest import mock
|
| 54 |
|
| 55 | with mock.patch.dict(os.environ, {"OAK_BENCH_ROBUSTNESS_OAKSPACE_URL": "https://oak.space"}):
|
| 56 | url, reason = configured_url()
|
| 57 | self.assertIsNone(url)
|
| 58 | self.assertIn("refusing non-loopback", reason)
|
| 59 |
|
| 60 |
|
| 61 | class OakGraphReaderTests(unittest.TestCase):
|
| 62 | def test_reachability_from_refs_on_a_crashed_wal_store(self):
|
| 63 | with tempfile.TemporaryDirectory() as temp:
|
| 64 | db = Path(temp) / "oak.db"
|
| 65 | conn = sqlite3.connect(db)
|
| 66 | conn.execute("PRAGMA journal_mode=WAL")
|
| 67 | conn.execute("CREATE TABLE metadata(key TEXT PRIMARY KEY, value TEXT)")
|
| 68 | conn.execute("CREATE TABLE branch_heads(branch_name TEXT PRIMARY KEY, head_hash TEXT)")
|
| 69 | conn.execute("CREATE TABLE commits(hash TEXT PRIMARY KEY, parent_hash TEXT, merge_parent_hash TEXT)")
|
| 70 | conn.executemany("INSERT INTO commits VALUES (?,?,?)",
|
| 71 | [("c1", None, None), ("c2", "c1", None), ("orphan", "c1", None)])
|
| 72 | conn.execute("INSERT INTO branch_heads VALUES ('b', 'c2')")
|
| 73 | conn.execute("INSERT INTO metadata VALUES ('head', 'c2'), ('current_branch', 'b')")
|
| 74 | conn.commit() # left in the WAL (connection still open) like a killed writer
|
| 75 | with OakGraph(db) as graph:
|
| 76 | self.assertEqual(graph.integrity(), "ok")
|
| 77 | self.assertEqual(graph.reachable(), {"c1", "c2"})
|
| 78 | self.assertNotIn("orphan", graph.reachable())
|
| 79 | self.assertTrue(graph.is_ancestor("c1", "c2"))
|
| 80 | conn.close()
|
| 81 |
|
| 82 |
|
| 83 | class RowContractTests(unittest.TestCase):
|
| 84 | def _runner(self, temp: Path, extra: list[str] | None = None):
|
| 85 | args = run_robustness.parse_args(["--subjects", "git", "--results", str(temp / "results"),
|
| 86 | "--workdir", str(temp / "work"), "--bench-id", "20260928T000000Z",
|
| 87 | "--git-bin", GIT or "git", *(extra or [])])
|
| 88 | runner = run_robustness.Runner(args)
|
| 89 | runner.versions = {"git": "git version test", "oak": "oak test"}
|
| 90 | return runner
|
| 91 |
|
| 92 | def test_emitted_rows_satisfy_the_robustness_lane_contract(self):
|
| 93 | with tempfile.TemporaryDirectory() as temp:
|
| 94 | runner = self._runner(Path(temp))
|
| 95 | runner.emit("git", "crash_time_v1", "commit", "crash_time",
|
| 96 | {"classification": HELD, "invariants": {"repo_opens": True}, "fault_point": "t=1ms",
|
| 97 | "elapsed_ms": 3.0, "remote_transport": "local_file", "remote_server": "local",
|
| 98 | "remote_repo": "../srv/remote.git"})
|
| 99 | runner.emit("oak", "crash_time_v1", "merge_hosted", "crash_time", {"unsupported": "404 on serve"})
|
| 100 | runner.emit("git", "crash_time_v1", "commit", "crash_time",
|
| 101 | {"classification": VIOLATED, "invariants": {"repo_opens": False}, "fault_point": "t=2ms"})
|
| 102 | rows = runner.rows
|
| 103 | self.assertEqual(validate_rows(rows, "robustness"), [])
|
| 104 | self.assertEqual([r["returncode"] for r in rows], [0, 77, 1])
|
| 105 | self.assertEqual(rows[1]["classification"], UNSUPPORTED)
|
| 106 | self.assertTrue(rows[1]["skipped"])
|
| 107 | self.assertEqual(rows[2]["violations"], ["repo_opens"])
|
| 108 | self.assertEqual([r["run"] for r in rows], [0, 0, 1])
|
| 109 | contract = LANE_TABLE["robustness"]
|
| 110 | from oakbench.lane_contract import (catalog_operations, check_catalog_membership,
|
| 111 | check_measurement_source, check_transport_purity)
|
| 112 | self.assertEqual(check_measurement_source(rows, contract.allowed_sources), [])
|
| 113 | self.assertEqual(check_transport_purity(rows), [])
|
| 114 | self.assertEqual(check_catalog_membership(rows, catalog_operations(contract.catalog_path)), [])
|
| 115 | partial = Path(temp) / "results" / "20260928T000000Z.robustness.partial.jsonl"
|
| 116 | self.assertEqual(len(partial.read_text().splitlines()), 3)
|
| 117 |
|
| 118 | def test_summary_table_counts_classes(self):
|
| 119 | rows = [{"subject_kind": "oak", "scenario": "s", "operation": "push", "classification": c,
|
| 120 | "violations": ["x"] if c == VIOLATED else [], "fault_point": "p"}
|
| 121 | for c in (HELD, HELD, VIOLATED, UNSUPPORTED)]
|
| 122 | text = run_robustness.summarize(rows, "id", 60)
|
| 123 | self.assertIn("| oak | s | push | 2 | 1 | 0 | 1 | 0 |", text)
|
| 124 | self.assertIn("## Violations", text)
|
| 125 |
|
| 126 |
|
| 127 | @unittest.skipUnless(GIT and shutil.which(GIT), "real git required")
|
| 128 | class GitEndToEndTests(unittest.TestCase):
|
| 129 | def test_time_crash_on_git_commit_is_judged_and_written(self):
|
| 130 | with tempfile.TemporaryDirectory() as temp:
|
| 131 | args = run_robustness.parse_args(["--subjects", "git", "--families", "crash_time,hosted", "--ops",
|
| 132 | "commit,merge_hosted", "--quick", "--results", str(Path(temp) / "r"),
|
| 133 | "--workdir", str(Path(temp) / "w"), "--bench-id", "20260928T000001Z",
|
| 134 | "--git-bin", GIT])
|
| 135 | code = run_robustness.Runner(args).run()
|
| 136 | self.assertEqual(code, 0)
|
| 137 | rows = [json.loads(line) for line in
|
| 138 | (Path(temp) / "r" / "20260928T000001Z.robustness.jsonl").read_text().splitlines()]
|
| 139 | self.assertEqual(validate_rows(rows, "robustness"), [])
|
| 140 | commit_rows = [r for r in rows if r["operation"] == "commit"]
|
| 141 | self.assertEqual(len(commit_rows), 3)
|
| 142 | for row in commit_rows:
|
| 143 | self.assertIn(row["classification"], (HELD, VIOLATED), row.get("harness_error"))
|
| 144 | self.assertIn("acked_local_reachable", row["invariants"])
|
| 145 | self.assertEqual(row["remote_transport"], "local_file")
|
| 146 | self.assertTrue(all(r["classification"] == UNSUPPORTED for r in rows if r["operation"] == "merge_hosted"))
|
| 147 |
|
| 148 |
|
| 149 | if __name__ == "__main__":
|
| 150 | unittest.main()
|