Viewing files at commit f08b4a249972Commit detailsView latest

Browse files
tests/test_robustness_invariants.py 566 lines · 31.4 KB · python
1
"""Robustness oracles: fault-fired accounting, torn worktrees, actionability, disk fill, row validation."""
2
3
from __future__ import annotations
4
5
import json
6
import os
7
import platform
8
import shutil
9
import subprocess
10
import sys
11
import tempfile
12
import unittest
13
from pathlib import Path
14
15
ROOT = Path(__file__).resolve().parents[1]
16
sys.path.insert(0, str(ROOT / "suites" / "robustness"))
17
sys.path.insert(0, str(ROOT / "scripts"))
18
19
import run_robustness  # noqa: E402
20
from robustness import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, VIOLATED, crashpoints, known_bugs  # noqa: E402
21
from robustness.concurrency import reported_pushed_head  # noqa: E402
22
from robustness.crash import run_time_point, run_trial, time_fault  # noqa: E402
23
from robustness.engine import actionable, classify, torn_state, torn_verdict, torn_worktree  # noqa: E402
24
from robustness.fixtures import build_git  # noqa: E402
25
from robustness.procs import Result, run_killed_after  # noqa: E402
26
from robustness.readers import worktree_manifest  # noqa: E402
27
28
from oakbench.rows import SKIP_RETURNCODE, validate_rows  # noqa: E402
29
from oakbench.subjects import which_measured_git  # noqa: E402
30
31
GIT = which_measured_git()
32
33
34
def result(rc, stdout="", stderr=""):
35
    return Result(["x"], rc, stdout, stderr, 1.0)
36
37
38
def f(tag):
39
    return {"kind": "file", "size": len(tag), "sha256": tag, "executable": False}
40
41
42
class FaultFiredTests(unittest.TestCase):
43
    def test_not_fired_is_never_held(self):
44
        self.assertEqual(classify({"a": True}, fault_fired=False), FAULT_NOT_FIRED)
45
        self.assertEqual(classify({"a": False}, fault_fired=False), FAULT_NOT_FIRED)
46
        self.assertEqual(classify({"a": True}, fault_fired=True), HELD)
47
        self.assertEqual(classify({"a": True}, fault_fired=None), HELD)
48
        self.assertEqual(classify({"a": True}, harness_error="x", fault_fired=False), HARNESS_ERROR)
49
50
    def test_reader_exception_is_harness_error_not_violation(self):
51
        self.assertEqual(classify({"a": None}, reader_errors=["reader error: boom"]), HARNESS_ERROR)
52
        # an independent subject failure still wins
53
        self.assertEqual(classify({"repo_opens": False, "a": None}, reader_errors=["x"]), VIOLATED)
54
55
    def test_process_that_exits_before_the_kill_is_not_fired(self):
56
        with tempfile.TemporaryDirectory() as temp:
57
            res = run_killed_after(["/bin/sh", "-c", "exit 0"], Path(temp), dict(os.environ), 2.0)
58
            self.assertEqual(res.returncode, 0)
59
60
            class Stub:
61
                def op_argv(self):
62
                    return ["/bin/sh", "-c", "exit 0"]
63
64
                def op_cwd(self):
65
                    return Path(temp)
66
67
                class adapter:  # noqa: N801
68
                    @staticmethod
69
                    def env():
70
                        return dict(os.environ)
71
72
            _, fired, point = time_fault(1.0)(Stub())
73
            self.assertFalse(fired)
74
            self.assertEqual(point, "t=1000.0ms")
75
76
    def test_process_killed_mid_run_is_fired(self):
77
        with tempfile.TemporaryDirectory() as temp:
78
            res = run_killed_after(["/bin/sleep", "5"], Path(temp), dict(os.environ), 0.05)
79
            self.assertEqual(res.returncode, -9)
80
            self.assertTrue(res.killed_by_harness)
81
82
83
@unittest.skipUnless(GIT and shutil.which(GIT), "real git required")
84
class NotFiredEndToEndTests(unittest.TestCase):
85
    def test_git_commit_that_finishes_before_the_kill_is_fault_not_fired(self):
86
        with tempfile.TemporaryDirectory() as temp:
87
            temp = Path(temp)
88
            fx = build_git(GIT, "commit", temp / "fx")
89
            from robustness.crash import calibrate
90
91
            calib = calibrate(fx, GIT, temp / "trials", None)
92
            self.assertTrue(calib.ok, calib.detail)
93
            row = run_trial(fx, GIT, temp / "trials", "late", calib, time_fault(60.0),
94
                            lambda trial, c: shutil.rmtree(trial.dir, ignore_errors=True))
95
            self.assertIs(row["fault_fired"], False)
96
            self.assertEqual(row["classification"], FAULT_NOT_FIRED)
97
            self.assertIn("did not fire", row["skip_reason"])
98
            args = run_robustness.parse_args(["--subjects", "git", "--results", str(temp / "r"), "--workdir",
99
                                              str(temp / "w"), "--bench-id", "20260929T000000Z", "--git-bin", GIT])
100
            runner = run_robustness.Runner(args)
101
            runner.versions = {"git": "git test"}
102
            emitted = runner.emit("git", "crash_time_v1", "commit", "crash_time", row)
103
            self.assertEqual(emitted["returncode"], SKIP_RETURNCODE)
104
            self.assertTrue(emitted["skipped"])
105
            self.assertEqual(validate_rows([emitted], "robustness"), [])
106
107
    def test_time_point_re_aims_from_observed_duration(self):
108
        with tempfile.TemporaryDirectory() as temp:
109
            temp = Path(temp)
110
            fx = build_git(GIT, "commit", temp / "fx")
111
            from robustness.crash import calibrate
112
113
            calib = calibrate(fx, GIT, temp / "trials", None)
114
            calib.elapsed_ms = 120000.0  # a stale calibration: the first kill lands long after the op
115
            row = run_time_point(fx, GIT, temp / "trials", "reaim", calib, 0.5,
116
                                 lambda trial, c: shutil.rmtree(trial.dir, ignore_errors=True))
117
            attempts = row["observations"]["time_attempts"]
118
            self.assertIs(attempts[0]["fired"], False)
119
            self.assertGreater(len(attempts), 1)
120
            self.assertLess(attempts[1]["delay_ms"], attempts[0]["delay_ms"])
121
            if row["fault_fired"] is False:
122
                self.assertEqual(row["classification"], FAULT_NOT_FIRED)
123
            else:
124
                self.assertIn(row["classification"], (HELD, VIOLATED))
125
126
127
class TornWorktreeTests(unittest.TestCase):
128
    PRE = {"a": f("a0"), "b": f("b0"), "c": f("c0")}
129
    POST = {"a": f("a1"), "b": f("b1"), "c": f("c0"), "n": f("n1")}
130
131
    def test_split(self):
132
        wt = {"a": f("a1"), "b": f("b0"), "c": f("c0"), "a.tmp-1": f("x")}
133
        split = torn_worktree(wt, self.PRE, self.POST)
134
        self.assertEqual(split["op_output"], ["a"])
135
        self.assertEqual(split["pre_side"], ["b", "n"])
136
        self.assertEqual(split["stray"], ["a.tmp-1"])
137
138
    def test_states(self):
139
        pre_split = torn_worktree(dict(self.PRE), self.PRE, self.POST)
140
        post_split = torn_worktree(dict(self.POST), self.PRE, self.POST)
141
        self.assertFalse(torn_state(pre_split, "pre"))
142
        self.assertFalse(torn_state(post_split, "post"))
143
        # every file already on the target side while the ref still names the old side
144
        self.assertTrue(torn_state(post_split, "pre"))
145
        # ref moved but the worktree lags
146
        self.assertTrue(torn_state(pre_split, "post"))
147
        mixed = torn_worktree({"a": f("a1"), "b": f("b0"), "c": f("c0")}, self.PRE, self.POST)
148
        self.assertTrue(torn_state(mixed, "pre"))
149
        self.assertTrue(torn_state(mixed, None))
150
        # user edits to files the op does not touch are not the op's output
151
        user = torn_worktree({**self.PRE, "c": f("mine")}, self.PRE, self.POST)
152
        self.assertFalse(torn_state(user, "pre"))
153
154
    def _verdict(self, changes, in_progress, head="old"):
155
        class Adapter:
156
            def head(self, repo):
157
                return head
158
159
            def status_state(self, repo):
160
                return changes, in_progress, result(0 if changes is not None else 4)
161
162
        class T:
163
            adapter = Adapter()
164
165
        from unittest import mock
166
        with mock.patch("robustness.readers.worktree_manifest",
167
                        return_value={"a": f("a1"), "b": f("b0"), "c": f("c0")}):
168
            return torn_verdict(T(), Path("."), self.PRE, self.POST, "old")
169
170
    def test_partial_output_presented_as_user_changes_fails(self):
171
        ok, detail = self._verdict(True, False)
172
        self.assertIs(ok, False)
173
        self.assertEqual(detail["torn"], "PARTIAL_OP_OUTPUT_AS_USER_CHANGES")
174
        self.assertEqual(detail["ref_side"], "pre")
175
176
    def test_flagged_in_progress_or_refused_passes(self):
177
        self.assertIs(self._verdict(True, True)[0], True)
178
        self.assertIs(self._verdict(None, False)[0], True)
179
180
181
@unittest.skipUnless(GIT and shutil.which(GIT), "real git required")
182
class TornWorktreeRealGitTests(unittest.TestCase):
183
    def test_half_switched_git_worktree_is_torn_but_merge_head_is_not(self):
184
        with tempfile.TemporaryDirectory() as temp:
185
            temp = Path(temp)
186
            fx = build_git(GIT, "switch", temp / "fx")
187
            client = fx.pristine / "client"
188
            pre = worktree_manifest(client)
189
            from robustness.engine import Trial
190
191
            trial = Trial(fx, GIT, temp / "trials", "torn")
192
            trial.adapter.must([GIT, "switch", "-q", fx.meta["target"]], trial.client)
193
            post = worktree_manifest(trial.client)
194
            trial.adapter.must([GIT, "switch", "-q", fx.branch], trial.client)
195
            # simulate a crash mid-checkout: some files already hold the target side, HEAD is old
196
            for name in sorted(p for p in post if pre.get(p) != post.get(p))[:5]:
197
                src = subprocess.run([GIT, "show", f"{fx.meta['target']}:{name}"], cwd=trial.client,
198
                                     capture_output=True, check=True).stdout
199
                (trial.client / name).parent.mkdir(parents=True, exist_ok=True)
200
                (trial.client / name).write_bytes(src)
201
            ok, detail = torn_verdict(trial, trial.client, pre, post, fx.meta["pre_head"])
202
            self.assertIs(ok, False, detail)
203
            (trial.client / ".git" / "MERGE_HEAD").write_text(fx.meta["pre_head"] + "\n")
204
            ok, detail = torn_verdict(trial, trial.client, pre, post, fx.meta["pre_head"])
205
            self.assertIs(ok, True, detail)
206
            trial.cleanup()
207
208
209
@unittest.skipUnless(GIT, "git required")
210
class GitStaleLockTests(unittest.TestCase):
211
    """M1: git locks that make the next commit refuse count as 'in progress', like index.lock.
212
213
    Synthetic, interposer-free: a real repo with a lock file planted, and real
214
    `git commit` confirming which locks actually block."""
215
216
    def repo(self, temp: Path) -> Path:
217
        repo = temp / "r"
218
        env = {**os.environ, "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1"}
219
        for argv in (["init", "-q", "-b", "main", str(repo)],):
220
            subprocess.run([GIT, *argv], check=True, env=env)
221
        for argv in (["config", "user.email", "a@b"], ["config", "user.name", "a"]):
222
            subprocess.run([GIT, *argv], cwd=repo, check=True, env=env)
223
        (repo / "f").write_text("a\n")
224
        subprocess.run([GIT, "add", "f"], cwd=repo, check=True, env=env)
225
        subprocess.run([GIT, "commit", "-qm", "a"], cwd=repo, check=True, env=env)
226
        subprocess.run([GIT, "branch", "other"], cwd=repo, check=True, env=env)
227
        subprocess.run([GIT, "tag", "t1"], cwd=repo, check=True, env=env)
228
        self.env = env
229
        return repo
230
231
    def test_blocking_locks_match_what_git_commit_refuses(self):
232
        from robustness.adapters import GitAdapter
233
234
        blocking = ("index.lock", "HEAD.lock", "refs/heads/main.lock")
235
        harmless = ("config.lock", "refs/heads/other.lock", "refs/tags/t1.lock", "ORIG_HEAD.lock")
236
        with tempfile.TemporaryDirectory() as temp:
237
            repo = self.repo(Path(temp))
238
            adapter = GitAdapter(GIT, Path(temp) / "home")
239
            for n, lock in enumerate(blocking + harmless):
240
                path = repo / ".git" / lock
241
                path.parent.mkdir(parents=True, exist_ok=True)
242
                path.write_text("")
243
                (repo / "f").write_text(f"change {n}\n")  # a pending change status lists
244
                changes, in_progress, _ = adapter.status_state(repo)
245
                commit = subprocess.run([GIT, "commit", "-qam", f"c{n}"], cwd=repo, capture_output=True,
246
                                        text=True, env=self.env)
247
                path.unlink()
248
                if lock in blocking:
249
                    self.assertTrue(in_progress, lock)
250
                    self.assertEqual(adapter.blocking_locks(repo), [], lock)
251
                    self.assertNotEqual(commit.returncode, 0, lock)
252
                    self.assertIn("remove the file manually", commit.stderr, lock)
253
                    self.assertTrue(actionable(result(commit.returncode, stderr=commit.stderr)), lock)
254
                else:
255
                    self.assertTrue(changes, lock)
256
                    self.assertFalse(in_progress, lock)
257
                    self.assertEqual(commit.returncode, 0, (lock, commit.stderr))
258
259
    def test_half_switched_worktree_with_head_lock_is_flagged_not_torn(self):
260
        # `git crash_syscall switch before rename#2`: worktree/index on the target, HEAD
261
        # still names the old branch, HEAD.lock left behind.
262
        with tempfile.TemporaryDirectory() as temp:
263
            temp = Path(temp)
264
            fx = build_git(GIT, "switch", temp / "fx")
265
            client = fx.pristine / "client"
266
            pre = worktree_manifest(client)
267
            from robustness.engine import Trial
268
269
            trial = Trial(fx, GIT, temp / "trials", "torn")
270
            trial.adapter.must([GIT, "switch", "-q", fx.meta["target"]], trial.client)
271
            post = worktree_manifest(trial.client)
272
            (trial.client / ".git" / "HEAD").write_text(f"ref: refs/heads/{fx.branch}\n")
273
            ok, detail = torn_verdict(trial, trial.client, pre, post, fx.meta["pre_head"])
274
            self.assertIs(ok, False, detail)  # without the lock: presented as user changes
275
            (trial.client / ".git" / "HEAD.lock").write_text(f"ref: refs/heads/{fx.meta['target']}\n")
276
            ok, detail = torn_verdict(trial, trial.client, pre, post, fx.meta["pre_head"])
277
            self.assertIs(ok, True, detail)
278
            self.assertEqual(detail["torn"], "flagged in progress")
279
            (trial.client / ".git" / "HEAD.lock").unlink()
280
            (trial.client / ".git" / "config.lock").write_text("")  # does not block a commit
281
            ok, detail = torn_verdict(trial, trial.client, pre, post, fx.meta["pre_head"])
282
            self.assertIs(ok, False, detail)
283
            trial.cleanup()
284
285
286
class ActionableTests(unittest.TestCase):
287
    def test_corruption_internal_and_panic_are_not_actionable(self):
288
        for text in ("error: failed to open oak database: file is not a database",
289
                     "error: internal error in oak core",
290
                     "error: push rejected: corrupted object",
291
                     "fatal: not a git repository (or any of the parent directories): .git",
292
                     "error: Database error: database disk image is malformed",
293
                     "thread 'main' panicked at src/lib.rs:9:5",
294
                     "error: internal error; please try again"):
295
            self.assertFalse(actionable(result(1, stderr=text)), text)
296
        internal = {"error": {"code": "internal", "message": "store malformed"}}
297
        self.assertFalse(actionable(result(1, stdout=json.dumps(internal))))
298
        generic = {"error": {"code": "error", "message": "something failed", "recommended_next_commands": []}}
299
        self.assertFalse(actionable(result(1, stdout=json.dumps(generic))))
300
301
    def test_recovery_commands_and_typed_codes_are_actionable(self):
302
        for text in ("error: working tree has uncommitted changes — run `oak commit` to keep them",
303
                     "Remove this directory and re-run `oak clone`.",
304
                     "The remote branch has commits this clone doesn't have.\nRun 'oak pull' to bring them in",
305
                     "remove the file manually to continue.",
306
                     "hint: Updates were rejected ... ! [rejected] main -> main (fetch first)",
307
                     "error: repository locked by another process; try again"):
308
            self.assertTrue(actionable(result(4, stderr=text)), text)
309
        rec = {"error": {"code": "error", "message": "x", "recommended_next_commands": ["oak status --json"]}}
310
        self.assertTrue(actionable(result(6, stdout=json.dumps(rec))))
311
        self.assertTrue(actionable(result(5, stdout=json.dumps({"error": {"code": "merge_in_progress"}}))))
312
        self.assertFalse(actionable(result(0, stderr="run `oak pull`")))
313
314
    CONFLICT = ("Auto-merging f.txt\nCONFLICT (content): Merge conflict in f.txt\n"
315
                "Automatic merge failed; fix conflicts and then commit the result.\n")
316
    FF = "fatal: Not possible to fast-forward, aborting.\n"
317
    PULL_FF = "From /tmp/srv/remote.git\n * branch            main       -> FETCH_HEAD\n" + FF
318
319
    def test_corruption_beats_any_recovery_mention(self):
320
        from robustness.envfaults import env_actionable
321
322
        cases = [
323
            (["git", "pull"], "fatal: loose object 0123abcd (stored in .git/objects/01/23abcd) is corrupt\n"
324
                              "hint: run git merge --abort"),
325
            (["git", "merge"], "fatal: index file corrupt\nhint: git merge --abort"),
326
            (["git", "merge"], "Auto-merging f.txt\nCONFLICT (content): Merge conflict in f.txt\n"
327
                               "Automatic merge failed; fix conflicts and then commit the result.\n"
328
                               "fatal: bad object HEAD\nrun git commit"),
329
            (["git", "pull"], "fatal: unable to read tree 0123abcd\nretry with git pull"),
330
            (["git", "pull"], "error: object file .git/objects/12/34 is empty\nrun git fsck"),
331
            (["git", "pull"], "fatal: early EOF\nfatal: fetch-pack: invalid index-pack output\ntry again"),
332
            (["oak", "pull"], "error: Database error: database disk image is malformed (oak pull)"),
333
            (["oak", "push"], "error: Server error: HTTP 500 Internal Server Error; run oak push again"),
334
            (["oak", "merge"], "error: internal error in merge engine; use --abort"),
335
            (["oak", "pull"], "error: Database error: unable to open database file\nrun `oak pull` again"),
336
            (["oak", "pull"], "error: Database error: unable to open database file"),
337
        ]
338
        for argv, text in cases:
339
            for rc in (1, 2, 128):
340
                res = Result(argv, rc, "", text, 1.0)
341
                self.assertFalse(actionable(res), text)
342
                self.assertFalse(env_actionable(res), text)
343
            self.assertFalse(actionable(Result(argv, 1, text, "", 1.0)), text)
344
        # the recovery rules still work on healthy output
345
        self.assertTrue(actionable(Result(["git", "merge"], 1, "", "error: Merging is not possible because you "
346
                                          "have unmerged files.\nhint: Fix them up and use git commit", 1.0)))
347
        self.assertTrue(actionable(Result(["oak", "pull"], 1, "", "error: working tree has uncommitted changes "
348
                                          "— run `oak commit` to keep them", 1.0)))
349
350
    def test_git_conflict_and_fast_forward_outcomes_are_actionable(self):
351
        def git(rc, **kw):
352
            return Result(["/usr/bin/git", "merge"], rc, kw.get("stdout", ""), kw.get("stderr", ""), 1.0)
353
        ff_hints = ("hint: Diverging branches can't be fast-forwarded, you need to either:\nhint:\n"
354
                    "hint: \tgit merge --no-ff\nhint:\nhint: or:\nhint:\nhint: \tgit rebase\nhint:\n"
355
                    "hint: Disable this message with \"git config set advice.diverging false\"\n" + self.FF)
356
        self.assertTrue(actionable(git(1, stdout=self.CONFLICT)))
357
        for text in (self.FF, self.PULL_FF, ff_hints):
358
            self.assertTrue(actionable(git(128, stderr=text)), text)
359
360
    def test_git_outcome_hints_are_git_only_and_allowlisted(self):
361
        def git(text, rc=1):
362
            return Result(["git", "pull"], rc, "", text, 1.0)
363
        # the same sentences from another subject are not git's hints
364
        for text in (self.CONFLICT, self.FF):
365
            self.assertFalse(actionable(Result(["oak", "pull"], 1, "", text, 1.0)), text)
366
            self.assertFalse(actionable(result(1, stderr=text)), text)
367
        broken_lines = ("fatal: not a git repository (or any of the parent directories): .git",
368
                        "error: object file .git/objects/ab/cd is empty",
369
                        "fatal: loose object abcd is corrupt",
370
                        "error: RPC failed; HTTP 500 curl 22 The requested URL returned error: 500",
371
                        "fatal: internal error in merge-ort",
372
                        "thread 'main' panicked at src/merge.rs:1:1",
373
                        "error: Database error: database disk image is malformed",
374
                        "fatal: bad object HEAD",
375
                        "error: unable to read sha1 file of f.txt (abcdef)",
376
                        "fatal: invalid object 1234567 for 'f.txt'",
377
                        "fatal: early EOF",
378
                        "Segmentation fault: 11",
379
                        "fatal: unable to read tree 0123456789abcdef",
380
                        "error: object file .git/objects/12/34 is empty",
381
                        "fatal: unable to write new index file",
382
                        "error: Database error: unable to open database file")
383
        for broken in broken_lines:
384
            for hint in (self.CONFLICT, self.FF, self.PULL_FF):
385
                for text in (broken + "\n" + hint, hint + broken + "\n"):
386
                    self.assertFalse(actionable(git(text)), text)
387
        # the phrase must be git's own sentence, not any mention of a failed merge
388
        self.assertFalse(actionable(git("error: automatic merge failed")))
389
        self.assertFalse(actionable(git("error: fast-forward not possible")))
390
391
392
@unittest.skipUnless(platform.system() == "Darwin" and shutil.which("hdiutil"), "macOS RAM disk required")
393
class DiskFillTests(unittest.TestCase):
394
    def test_fill_leaves_less_than_one_block_and_records_measured_free(self):
395
        from robustness.envfaults import RamDisk, fill, free_bytes
396
397
        with tempfile.TemporaryDirectory() as temp:
398
            mount = Path(temp) / "mnt"
399
            with RamDisk(mount, mib=8):
400
                _, free = fill(mount, 0)
401
                self.assertLess(free, 4096)
402
                self.assertEqual(free, free_bytes(mount))
403
                with self.assertRaises(OSError):
404
                    with (mount / "more.bin").open("wb") as fh:
405
                        fh.write(b"x" * 8192)
406
                        fh.flush()
407
                        os.fsync(fh.fileno())
408
                (mount / "more.bin").unlink(missing_ok=True)
409
                (mount / "p4-filler.bin").unlink()
410
                _, free = fill(mount, 64)
411
                self.assertGreaterEqual(free, 60 * 1024)
412
                self.assertLess(free, 72 * 1024)
413
414
415
class EnvFaultFiredTests(unittest.TestCase):
416
    def test_disk_full_judges_every_failure_and_skips_only_a_roomy_success(self):
417
        from robustness.envfaults import disk_full_fired
418
419
        # only a successful op that had room is not fired
420
        fired, evidence = disk_full_fired(1 << 20, 4096, result(0))
421
        self.assertIs(fired, False)
422
        self.assertIn("succeeded", evidence)
423
        for text in ("error: IO error: No space left on device (os error 28)",
424
                     "fatal: unable to write new index file: ENOSPC",
425
                     "Error: database or disk is full", "sqlite: SQLITE_FULL"):
426
            fired, evidence = disk_full_fired(1 << 20, 4096, result(1, stderr=text))
427
            self.assertTrue(fired, text)
428
            self.assertIn("ENOSPC", evidence)
429
        self.assertTrue(disk_full_fired(0, 4096, result(0))[0])  # measured short: fired even if it succeeded
430
        # a failure with room and no ENOSPC text is judged, never skipped
431
        fb548 = "error: Database error: unable to open database file"
432
        panic = "thread 'main' panicked at src/store.rs:9:5:\nnote: run with `RUST_BACKTRACE=1`"
433
        hang = Result(["oak", "pull"], None, "", "", 90000.0, killed_by_harness=True, timed_out=True)
434
        for res in (result(1, stderr=fb548), result(101, stderr=panic), result(1, stderr="error: something else"),
435
                    hang):
436
            for free in (0, 1 << 20):
437
                fired, evidence = disk_full_fired(free, 4096, res)
438
                self.assertIs(fired, True, (res, free))
439
440
    def test_opaque_failures_on_a_full_disk_are_violations(self):
441
        from robustness.envfaults import disk_full_fired, env_actionable
442
443
        fb548 = "error: Database error: unable to open database file"
444
        panic = "thread 'main' panicked at src/store.rs:9:5:\nnote: run with `RUST_BACKTRACE=1`"
445
        for res in (result(1, stderr=fb548), result(101, stderr=panic)):
446
            fired, _ = disk_full_fired(1 << 20, 4096, res)
447
            inv = {"op_no_hang": True, "op_failure_actionable": env_actionable(res)}
448
            self.assertEqual(classify(inv, fault_fired=fired), VIOLATED, res.stderr)
449
        hang = Result(["oak", "pull"], None, "", "", 90000.0, killed_by_harness=True, timed_out=True)
450
        fired, _ = disk_full_fired(1 << 20, 4096, hang)
451
        self.assertEqual(classify({"op_no_hang": False}, fault_fired=fired), VIOLATED)
452
        # the cause named plainly stays actionable
453
        named = result(1, stderr="error: IO error: No space left on device (os error 28)")
454
        self.assertTrue(env_actionable(named))
455
456
    def test_failed_calibration_is_a_harness_error_not_zero_need(self):
457
        from robustness.envfaults import disk_full_trial
458
459
        row = disk_full_trial(None, "oak", Path("/nonexistent"), "x", None, 0, None)
460
        self.assertEqual(row["classification"], HARNESS_ERROR)
461
        self.assertIn("unmeasured", row["harness_error"])
462
463
    def test_readonly_judges_failures_and_skips_only_unexercised_success(self):
464
        from robustness.envfaults import readonly_fired
465
466
        hang = Result(["oak", "pull"], None, "", "", 90000.0, killed_by_harness=True, timed_out=True)
467
        for res in (result(1, stderr="Permission denied"), result(1, stderr="error: unable to open database file"),
468
                    result(101, stderr="thread 'main' panicked at x"), hang):
469
            for writable, exercised in ((True, False), (False, False), (False, None), (False, True)):
470
                self.assertIs(readonly_fired(writable, exercised, res)[0], True, (res, writable, exercised))
471
        self.assertTrue(readonly_fired(False, False, result(0, stderr="warning: permission denied on d3"))[0])
472
        self.assertTrue(readonly_fired(False, True, result(0))[0])
473
        fired, evidence = readonly_fired(False, False, result(0))
474
        self.assertIs(fired, False)
475
        self.assertTrue(evidence.startswith("not exercised"))
476
        self.assertIs(readonly_fired(True, True, result(0))[0], False)  # root: fault never applied
477
        self.assertIsNone(readonly_fired(False, None, result(0))[0])
478
479
    def test_write_snapshot_sees_writes_under_the_locked_path(self):
480
        from robustness.envfaults import write_snapshot
481
482
        with tempfile.TemporaryDirectory() as temp:
483
            d = Path(temp) / "d3"
484
            (d / "sub").mkdir(parents=True)
485
            (d / "a").write_text("a")
486
            before = write_snapshot(d, "subdir")
487
            (d / "sub" / "deep").write_text("x")  # not blocked by a read-only d3 itself
488
            (d / "a").write_text("changed in place")  # in-place write: allowed
489
            self.assertEqual(write_snapshot(d, "subdir"), before)
490
            (d / "b").write_text("new entry")
491
            self.assertNotEqual(write_snapshot(d, "subdir"), before)
492
            store = write_snapshot(d, "store")
493
            os.utime(d / "sub" / "deep", ns=(1, 1))
494
            self.assertNotEqual(write_snapshot(d, "store"), store)
495
            self.assertIsNone(write_snapshot(d / "missing", "store"))
496
497
    @unittest.skipUnless(GIT and os.geteuid() != 0, "git and a non-root user required")
498
    def test_git_readonly_rows_record_fired_or_not_exercised(self):
499
        from robustness.crash import calibrate
500
        from robustness.envfaults import readonly_exercised, readonly_trial
501
502
        with tempfile.TemporaryDirectory() as temp:
503
            temp = Path(temp)
504
            fx = build_git(GIT, "commit", temp / "fx")
505
            calib = calibrate(fx, GIT, temp / "trials", None)
506
            exercised, detail = readonly_exercised(fx, GIT, temp / "trials", "probe", "store")
507
            self.assertTrue(exercised, detail)
508
            row = readonly_trial(fx, GIT, temp / "trials", "ro", calib, "store", exercised, detail)
509
            self.assertIs(row["fault_fired"], True, row)
510
            # a lock-down no write reaches is recorded as not exercised, never held
511
            row = readonly_trial(fx, GIT, temp / "trials", "ro2", calib, "store", False, {})
512
            if row["fault_fired"] is False:
513
                self.assertEqual(row["classification"], FAULT_NOT_FIRED)
514
                self.assertTrue(row["skip_reason"].startswith("not exercised"))
515
            else:  # git failed on the read-only store: judged, never skipped
516
                self.assertNotEqual(row["command"], [])
517
                self.assertIn(row["classification"], (HELD, VIOLATED))
518
                self.assertRegex(row["observations"]["fault_fired_evidence"], "judged|permission")
519
520
521
class RowValidationTests(unittest.TestCase):
522
    def _row(self, **kw):
523
        row = {"bench_id": "20260929T000000Z", "scenario": "crash_time_v1", "operation": "commit", "run": 0,
524
               "subject": "git", "subject_kind": "git", "elapsed_ms": 1.0, "returncode": 0, "classification": "held",
525
               "fault_family": "crash_time", "fault_point": "t=1ms", "fault_fired": True, "violations": []}
526
        row.update(kw)
527
        return row
528
529
    def test_rejects_unknown_and_inconsistent_classifications(self):
530
        self.assertEqual(validate_rows([self._row()], "robustness"), [])
531
        self.assertTrue(validate_rows([self._row(classification="bogus")], "robustness"))
532
        self.assertTrue(validate_rows([self._row(returncode=1)], "robustness"))
533
        self.assertTrue(validate_rows([self._row(classification="violated", returncode=0)], "robustness"))
534
        self.assertTrue(validate_rows([self._row(fault_fired=False)], "robustness"))
535
        self.assertTrue(validate_rows([self._row(violations=["x"])], "robustness"))
536
        not_fired = self._row(classification="fault_not_fired", returncode=77, fault_fired=False, skipped=True,
537
                              skip_reason="op exited before the kill")
538
        self.assertEqual(validate_rows([not_fired], "robustness"), [])
539
        self.assertTrue(validate_rows([{**not_fired, "returncode": 0, "skipped": False}], "robustness"))
540
541
542
class MiscTests(unittest.TestCase):
543
    def test_known_bug_tags(self):
544
        self.assertEqual(known_bugs("oak", "crash_time_v1", "pull_ff", ["worktree_not_torn"]), ["fb-535"])
545
        self.assertEqual(known_bugs("oak", "crash_syscall_v1", "switch", ["worktree_not_torn", "x"]), ["fb-535"])
546
        self.assertEqual(known_bugs("oak", "conc_same_branch_v1", "concurrent.push",
547
                                    ["no_lost_acknowledged_write", "linearizable"]), ["fb-534"])
548
        self.assertEqual(known_bugs("git", "crash_time_v1", "switch", ["worktree_not_torn"]), [])
549
        self.assertEqual(known_bugs("oak", "crash_time_v1", "commit", ["repo_opens"]), [])
550
551
    def test_no_boundary_reasons_are_subject_specific(self):
552
        self.assertNotIn("git", crashpoints.NO_BOUNDARY_REASON["oak"])
553
        self.assertIn("git", crashpoints.NO_BOUNDARY_REASON["git"])
554
        inst = crashpoints.CrashInstrument(Path("x"), None, Path("g"), None, {}, {"oak": "codesign failed"})
555
        self.assertEqual(crashpoints.unavailable_reason(inst, "", "oak"), "codesign failed")
556
        self.assertEqual(crashpoints.unavailable_reason(inst, "", "git"), "")
557
        self.assertEqual(crashpoints.unavailable_reason(None, "no clang", "git"), "no clang")
558
559
    def test_reported_pushed_head(self):
560
        self.assertEqual(reported_pushed_head(result(0, stdout=json.dumps({"pushed": True, "pushed_head": "abc"}))),
561
                         "abc")
562
        self.assertIsNone(reported_pushed_head(result(0, stdout="plain text")))
563
564
565
if __name__ == "__main__":
566
    unittest.main()