Viewing files at commit ef78687eef1aCommit detailsView latest

suites/robustness/robustness/__init__.py 53 lines · 2.5 KB · python
1
"""P4 robustness suite: crash points, fault injection, concurrency invariants.
2
3
VCS-agnostic black-box harness. Subjects are driven only through their CLIs;
4
every invariant is judged by an independent reader (stdlib SQLite for Oak's
5
committed graph, ``oakbench.oak_sqlite_reader`` for Oak committed trees, the
6
real Git plumbing as the Git oracle), never by the binary under test alone.
7
See suites/robustness/README.md for the case catalog and classification.
8
"""
9
10
from __future__ import annotations
11
12
import sys
13
from pathlib import Path
14
15
SUITE_ROOT = Path(__file__).resolve().parents[1]
16
REPO_ROOT = SUITE_ROOT.parents[1]
17
_SCRIPTS = REPO_ROOT / "scripts"
18
if str(_SCRIPTS) not in sys.path:
19
    sys.path.insert(0, str(_SCRIPTS))
20
21
SUITE_VERSION = "p4-robustness-2026-09-29.2"  # .2: evidence-based env fault_fired, git lock rule
22
23
# Case classification vocabulary (row field ``classification``).
24
HELD = "held"
25
VIOLATED = "violated"
26
UNSUPPORTED = "unsupported"
27
HARNESS_ERROR = "harness_error"
28
# The injected fault never took effect (the kill landed after the operation had
29
# already exited, the disk was not actually short of the op's need, ...). The
30
# row records a clean run, so it is never evidence that an invariant held.
31
FAULT_NOT_FIRED = "fault_not_fired"
32
CLASSIFICATIONS = (HELD, VIOLATED, UNSUPPORTED, HARNESS_ERROR, FAULT_NOT_FIRED)
33
34
# Known Oak bugs a violation maps to (subject_kind, family prefix, operations, invariant) -> feedback id.
35
# Rows whose violations match carry ``known_bugs``; unmatched violations are new findings.
36
KNOWN_BUGS = (
37
    ("oak", "crash_", ("switch", "pull_ff", "pull_diverged"), "worktree_not_torn", "fb-535"),
38
    # same defect reached through an I/O error instead of a kill: a pull that fails midway
39
    # leaves its partial output presented as user changes
40
    ("oak", "env_", ("pull_ff",), "worktree_not_torn", "fb-535"),
41
    ("oak", "fault_proxy", ("pull_ff", "pull_diverged", "clone"), "worktree_not_torn", "fb-535"),
42
    ("oak", "conc_same_branch", ("concurrent.push",), "no_lost_acknowledged_write", "fb-534"),
43
    ("oak", "env_fsync_load", ("concurrent.push",), "no_lost_acknowledged_write", "fb-534"),
44
)
45
46
47
def known_bugs(subject_kind: str, scenario: str, operation: str, violations: list) -> list:
48
    """Feedback ids of known bugs matched by a row's violations (sorted, unique)."""
49
    found = set()
50
    for kind, prefix, ops, invariant, bug in KNOWN_BUGS:
51
        if kind == subject_kind and scenario.startswith(prefix) and operation in ops and invariant in violations:
52
            found.add(bug)
53
    return sorted(found)