Viewing files at commit d234051be762Commit detailsView latest

Browse files
tests/test_robustness_invariants.py 306 lines · 15.5 KB · python
1
"""Robustness oracles: fault-fired accounting, torn worktrees, actionability, disk fill, row validation."""
2
3
from __future__ import annotations
4
5
import json
6
import os
7
import platform
8
import shutil
9
import subprocess
10
import sys
11
import tempfile
12
import unittest
13
from pathlib import Path
14
15
ROOT = Path(__file__).resolve().parents[1]
16
sys.path.insert(0, str(ROOT / "suites" / "robustness"))
17
sys.path.insert(0, str(ROOT / "scripts"))
18
19
import run_robustness  # noqa: E402
20
from robustness import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, VIOLATED, crashpoints, known_bugs  # noqa: E402
21
from robustness.concurrency import reported_pushed_head  # noqa: E402
22
from robustness.crash import run_time_point, run_trial, time_fault  # noqa: E402
23
from robustness.engine import actionable, classify, torn_state, torn_verdict, torn_worktree  # noqa: E402
24
from robustness.fixtures import build_git  # noqa: E402
25
from robustness.procs import Result, run_killed_after  # noqa: E402
26
from robustness.readers import worktree_manifest  # noqa: E402
27
28
from oakbench.rows import SKIP_RETURNCODE, validate_rows  # noqa: E402
29
from oakbench.subjects import which_measured_git  # noqa: E402
30
31
GIT = which_measured_git()
32
33
34
def result(rc, stdout="", stderr=""):
35
    return Result(["x"], rc, stdout, stderr, 1.0)
36
37
38
def f(tag):
39
    return {"kind": "file", "size": len(tag), "sha256": tag, "executable": False}
40
41
42
class FaultFiredTests(unittest.TestCase):
43
    def test_not_fired_is_never_held(self):
44
        self.assertEqual(classify({"a": True}, fault_fired=False), FAULT_NOT_FIRED)
45
        self.assertEqual(classify({"a": False}, fault_fired=False), FAULT_NOT_FIRED)
46
        self.assertEqual(classify({"a": True}, fault_fired=True), HELD)
47
        self.assertEqual(classify({"a": True}, fault_fired=None), HELD)
48
        self.assertEqual(classify({"a": True}, harness_error="x", fault_fired=False), HARNESS_ERROR)
49
50
    def test_reader_exception_is_harness_error_not_violation(self):
51
        self.assertEqual(classify({"a": None}, reader_errors=["reader error: boom"]), HARNESS_ERROR)
52
        # an independent subject failure still wins
53
        self.assertEqual(classify({"repo_opens": False, "a": None}, reader_errors=["x"]), VIOLATED)
54
55
    def test_process_that_exits_before_the_kill_is_not_fired(self):
56
        with tempfile.TemporaryDirectory() as temp:
57
            res = run_killed_after(["/bin/sh", "-c", "exit 0"], Path(temp), dict(os.environ), 2.0)
58
            self.assertEqual(res.returncode, 0)
59
60
            class Stub:
61
                def op_argv(self):
62
                    return ["/bin/sh", "-c", "exit 0"]
63
64
                def op_cwd(self):
65
                    return Path(temp)
66
67
                class adapter:  # noqa: N801
68
                    @staticmethod
69
                    def env():
70
                        return dict(os.environ)
71
72
            _, fired, point = time_fault(1.0)(Stub())
73
            self.assertFalse(fired)
74
            self.assertEqual(point, "t=1000.0ms")
75
76
    def test_process_killed_mid_run_is_fired(self):
77
        with tempfile.TemporaryDirectory() as temp:
78
            res = run_killed_after(["/bin/sleep", "5"], Path(temp), dict(os.environ), 0.05)
79
            self.assertEqual(res.returncode, -9)
80
            self.assertTrue(res.killed_by_harness)
81
82
83
@unittest.skipUnless(GIT and shutil.which(GIT), "real git required")
84
class NotFiredEndToEndTests(unittest.TestCase):
85
    def test_git_commit_that_finishes_before_the_kill_is_fault_not_fired(self):
86
        with tempfile.TemporaryDirectory() as temp:
87
            temp = Path(temp)
88
            fx = build_git(GIT, "commit", temp / "fx")
89
            from robustness.crash import calibrate
90
91
            calib = calibrate(fx, GIT, temp / "trials", None)
92
            self.assertTrue(calib.ok, calib.detail)
93
            row = run_trial(fx, GIT, temp / "trials", "late", calib, time_fault(60.0),
94
                            lambda trial, c: shutil.rmtree(trial.dir, ignore_errors=True))
95
            self.assertIs(row["fault_fired"], False)
96
            self.assertEqual(row["classification"], FAULT_NOT_FIRED)
97
            self.assertIn("did not fire", row["skip_reason"])
98
            args = run_robustness.parse_args(["--subjects", "git", "--results", str(temp / "r"), "--workdir",
99
                                              str(temp / "w"), "--bench-id", "20260929T000000Z", "--git-bin", GIT])
100
            runner = run_robustness.Runner(args)
101
            runner.versions = {"git": "git test"}
102
            emitted = runner.emit("git", "crash_time_v1", "commit", "crash_time", row)
103
            self.assertEqual(emitted["returncode"], SKIP_RETURNCODE)
104
            self.assertTrue(emitted["skipped"])
105
            self.assertEqual(validate_rows([emitted], "robustness"), [])
106
107
    def test_time_point_re_aims_from_observed_duration(self):
108
        with tempfile.TemporaryDirectory() as temp:
109
            temp = Path(temp)
110
            fx = build_git(GIT, "commit", temp / "fx")
111
            from robustness.crash import calibrate
112
113
            calib = calibrate(fx, GIT, temp / "trials", None)
114
            calib.elapsed_ms = 120000.0  # a stale calibration: the first kill lands long after the op
115
            row = run_time_point(fx, GIT, temp / "trials", "reaim", calib, 0.5,
116
                                 lambda trial, c: shutil.rmtree(trial.dir, ignore_errors=True))
117
            attempts = row["observations"]["time_attempts"]
118
            self.assertIs(attempts[0]["fired"], False)
119
            self.assertGreater(len(attempts), 1)
120
            self.assertLess(attempts[1]["delay_ms"], attempts[0]["delay_ms"])
121
            if row["fault_fired"] is False:
122
                self.assertEqual(row["classification"], FAULT_NOT_FIRED)
123
            else:
124
                self.assertIn(row["classification"], (HELD, VIOLATED))
125
126
127
class TornWorktreeTests(unittest.TestCase):
128
    PRE = {"a": f("a0"), "b": f("b0"), "c": f("c0")}
129
    POST = {"a": f("a1"), "b": f("b1"), "c": f("c0"), "n": f("n1")}
130
131
    def test_split(self):
132
        wt = {"a": f("a1"), "b": f("b0"), "c": f("c0"), "a.tmp-1": f("x")}
133
        split = torn_worktree(wt, self.PRE, self.POST)
134
        self.assertEqual(split["op_output"], ["a"])
135
        self.assertEqual(split["pre_side"], ["b", "n"])
136
        self.assertEqual(split["stray"], ["a.tmp-1"])
137
138
    def test_states(self):
139
        pre_split = torn_worktree(dict(self.PRE), self.PRE, self.POST)
140
        post_split = torn_worktree(dict(self.POST), self.PRE, self.POST)
141
        self.assertFalse(torn_state(pre_split, "pre"))
142
        self.assertFalse(torn_state(post_split, "post"))
143
        # every file already on the target side while the ref still names the old side
144
        self.assertTrue(torn_state(post_split, "pre"))
145
        # ref moved but the worktree lags
146
        self.assertTrue(torn_state(pre_split, "post"))
147
        mixed = torn_worktree({"a": f("a1"), "b": f("b0"), "c": f("c0")}, self.PRE, self.POST)
148
        self.assertTrue(torn_state(mixed, "pre"))
149
        self.assertTrue(torn_state(mixed, None))
150
        # user edits to files the op does not touch are not the op's output
151
        user = torn_worktree({**self.PRE, "c": f("mine")}, self.PRE, self.POST)
152
        self.assertFalse(torn_state(user, "pre"))
153
154
    def _verdict(self, changes, in_progress, head="old"):
155
        class Adapter:
156
            def head(self, repo):
157
                return head
158
159
            def status_state(self, repo):
160
                return changes, in_progress, result(0 if changes is not None else 4)
161
162
        class T:
163
            adapter = Adapter()
164
165
        from unittest import mock
166
        with mock.patch("robustness.readers.worktree_manifest",
167
                        return_value={"a": f("a1"), "b": f("b0"), "c": f("c0")}):
168
            return torn_verdict(T(), Path("."), self.PRE, self.POST, "old")
169
170
    def test_partial_output_presented_as_user_changes_fails(self):
171
        ok, detail = self._verdict(True, False)
172
        self.assertIs(ok, False)
173
        self.assertEqual(detail["torn"], "PARTIAL_OP_OUTPUT_AS_USER_CHANGES")
174
        self.assertEqual(detail["ref_side"], "pre")
175
176
    def test_flagged_in_progress_or_refused_passes(self):
177
        self.assertIs(self._verdict(True, True)[0], True)
178
        self.assertIs(self._verdict(None, False)[0], True)
179
180
181
@unittest.skipUnless(GIT and shutil.which(GIT), "real git required")
182
class TornWorktreeRealGitTests(unittest.TestCase):
183
    def test_half_switched_git_worktree_is_torn_but_merge_head_is_not(self):
184
        with tempfile.TemporaryDirectory() as temp:
185
            temp = Path(temp)
186
            fx = build_git(GIT, "switch", temp / "fx")
187
            client = fx.pristine / "client"
188
            pre = worktree_manifest(client)
189
            from robustness.engine import Trial
190
191
            trial = Trial(fx, GIT, temp / "trials", "torn")
192
            trial.adapter.must([GIT, "switch", "-q", fx.meta["target"]], trial.client)
193
            post = worktree_manifest(trial.client)
194
            trial.adapter.must([GIT, "switch", "-q", fx.branch], trial.client)
195
            # simulate a crash mid-checkout: some files already hold the target side, HEAD is old
196
            for name in sorted(p for p in post if pre.get(p) != post.get(p))[:5]:
197
                src = subprocess.run([GIT, "show", f"{fx.meta['target']}:{name}"], cwd=trial.client,
198
                                     capture_output=True, check=True).stdout
199
                (trial.client / name).parent.mkdir(parents=True, exist_ok=True)
200
                (trial.client / name).write_bytes(src)
201
            ok, detail = torn_verdict(trial, trial.client, pre, post, fx.meta["pre_head"])
202
            self.assertIs(ok, False, detail)
203
            (trial.client / ".git" / "MERGE_HEAD").write_text(fx.meta["pre_head"] + "\n")
204
            ok, detail = torn_verdict(trial, trial.client, pre, post, fx.meta["pre_head"])
205
            self.assertIs(ok, True, detail)
206
            trial.cleanup()
207
208
209
class ActionableTests(unittest.TestCase):
210
    def test_corruption_internal_and_panic_are_not_actionable(self):
211
        for text in ("error: failed to open oak database: file is not a database",
212
                     "error: internal error in oak core",
213
                     "error: push rejected: corrupted object",
214
                     "fatal: not a git repository (or any of the parent directories): .git",
215
                     "error: Database error: database disk image is malformed",
216
                     "thread 'main' panicked at src/lib.rs:9:5",
217
                     "error: internal error; please try again"):
218
            self.assertFalse(actionable(result(1, stderr=text)), text)
219
        internal = {"error": {"code": "internal", "message": "store malformed"}}
220
        self.assertFalse(actionable(result(1, stdout=json.dumps(internal))))
221
        generic = {"error": {"code": "error", "message": "something failed", "recommended_next_commands": []}}
222
        self.assertFalse(actionable(result(1, stdout=json.dumps(generic))))
223
224
    def test_recovery_commands_and_typed_codes_are_actionable(self):
225
        for text in ("error: working tree has uncommitted changes — run `oak commit` to keep them",
226
                     "Remove this directory and re-run `oak clone`.",
227
                     "The remote branch has commits this clone doesn't have.\nRun 'oak pull' to bring them in",
228
                     "remove the file manually to continue.",
229
                     "hint: Updates were rejected ... ! [rejected] main -> main (fetch first)",
230
                     "error: repository locked by another process; try again"):
231
            self.assertTrue(actionable(result(4, stderr=text)), text)
232
        rec = {"error": {"code": "error", "message": "x", "recommended_next_commands": ["oak status --json"]}}
233
        self.assertTrue(actionable(result(6, stdout=json.dumps(rec))))
234
        self.assertTrue(actionable(result(5, stdout=json.dumps({"error": {"code": "merge_in_progress"}}))))
235
        self.assertFalse(actionable(result(0, stderr="run `oak pull`")))
236
237
238
@unittest.skipUnless(platform.system() == "Darwin" and shutil.which("hdiutil"), "macOS RAM disk required")
239
class DiskFillTests(unittest.TestCase):
240
    def test_fill_leaves_less_than_one_block_and_records_measured_free(self):
241
        from robustness.envfaults import RamDisk, fill, free_bytes
242
243
        with tempfile.TemporaryDirectory() as temp:
244
            mount = Path(temp) / "mnt"
245
            with RamDisk(mount, mib=8):
246
                _, free = fill(mount, 0)
247
                self.assertLess(free, 4096)
248
                self.assertEqual(free, free_bytes(mount))
249
                with self.assertRaises(OSError):
250
                    with (mount / "more.bin").open("wb") as fh:
251
                        fh.write(b"x" * 8192)
252
                        fh.flush()
253
                        os.fsync(fh.fileno())
254
                (mount / "more.bin").unlink(missing_ok=True)
255
                (mount / "p4-filler.bin").unlink()
256
                _, free = fill(mount, 64)
257
                self.assertGreaterEqual(free, 60 * 1024)
258
                self.assertLess(free, 72 * 1024)
259
260
261
class RowValidationTests(unittest.TestCase):
262
    def _row(self, **kw):
263
        row = {"bench_id": "20260929T000000Z", "scenario": "crash_time_v1", "operation": "commit", "run": 0,
264
               "subject": "git", "subject_kind": "git", "elapsed_ms": 1.0, "returncode": 0, "classification": "held",
265
               "fault_family": "crash_time", "fault_point": "t=1ms", "fault_fired": True, "violations": []}
266
        row.update(kw)
267
        return row
268
269
    def test_rejects_unknown_and_inconsistent_classifications(self):
270
        self.assertEqual(validate_rows([self._row()], "robustness"), [])
271
        self.assertTrue(validate_rows([self._row(classification="bogus")], "robustness"))
272
        self.assertTrue(validate_rows([self._row(returncode=1)], "robustness"))
273
        self.assertTrue(validate_rows([self._row(classification="violated", returncode=0)], "robustness"))
274
        self.assertTrue(validate_rows([self._row(fault_fired=False)], "robustness"))
275
        self.assertTrue(validate_rows([self._row(violations=["x"])], "robustness"))
276
        not_fired = self._row(classification="fault_not_fired", returncode=77, fault_fired=False, skipped=True,
277
                              skip_reason="op exited before the kill")
278
        self.assertEqual(validate_rows([not_fired], "robustness"), [])
279
        self.assertTrue(validate_rows([{**not_fired, "returncode": 0, "skipped": False}], "robustness"))
280
281
282
class MiscTests(unittest.TestCase):
283
    def test_known_bug_tags(self):
284
        self.assertEqual(known_bugs("oak", "crash_time_v1", "pull_ff", ["worktree_not_torn"]), ["fb-535"])
285
        self.assertEqual(known_bugs("oak", "crash_syscall_v1", "switch", ["worktree_not_torn", "x"]), ["fb-535"])
286
        self.assertEqual(known_bugs("oak", "conc_same_branch_v1", "concurrent.push",
287
                                    ["no_lost_acknowledged_write", "linearizable"]), ["fb-534"])
288
        self.assertEqual(known_bugs("git", "crash_time_v1", "switch", ["worktree_not_torn"]), [])
289
        self.assertEqual(known_bugs("oak", "crash_time_v1", "commit", ["repo_opens"]), [])
290
291
    def test_no_boundary_reasons_are_subject_specific(self):
292
        self.assertNotIn("git", crashpoints.NO_BOUNDARY_REASON["oak"])
293
        self.assertIn("git", crashpoints.NO_BOUNDARY_REASON["git"])
294
        inst = crashpoints.CrashInstrument(Path("x"), None, Path("g"), None, {}, {"oak": "codesign failed"})
295
        self.assertEqual(crashpoints.unavailable_reason(inst, "", "oak"), "codesign failed")
296
        self.assertEqual(crashpoints.unavailable_reason(inst, "", "git"), "")
297
        self.assertEqual(crashpoints.unavailable_reason(None, "no clang", "git"), "no clang")
298
299
    def test_reported_pushed_head(self):
300
        self.assertEqual(reported_pushed_head(result(0, stdout=json.dumps({"pushed": True, "pushed_head": "abc"}))),
301
                         "abc")
302
        self.assertIsNone(reported_pushed_head(result(0, stdout="plain text")))
303
304
305
if __name__ == "__main__":
306
    unittest.main()