| 1 | """P4 robustness suite: crash points, fault injection, concurrency invariants.
|
| 2 |
|
| 3 | VCS-agnostic black-box harness. Subjects are driven only through their CLIs;
|
| 4 | every invariant is judged by an independent reader (stdlib SQLite for Oak's
|
| 5 | committed graph, ``oakbench.oak_sqlite_reader`` for Oak committed trees, the
|
| 6 | real Git plumbing as the Git oracle), never by the binary under test alone.
|
| 7 | See suites/robustness/README.md for the case catalog and classification.
|
| 8 | """
|
| 9 |
|
| 10 | from __future__ import annotations
|
| 11 |
|
| 12 | import sys
|
| 13 | from pathlib import Path
|
| 14 |
|
| 15 | SUITE_ROOT = Path(__file__).resolve().parents[1]
|
| 16 | REPO_ROOT = SUITE_ROOT.parents[1]
|
| 17 | _SCRIPTS = REPO_ROOT / "scripts"
|
| 18 | if str(_SCRIPTS) not in sys.path:
|
| 19 | sys.path.insert(0, str(_SCRIPTS))
|
| 20 |
|
| 21 | SUITE_VERSION = "p4-robustness-2026-09-29.2" # .2: evidence-based env fault_fired, git lock rule
|
| 22 |
|
| 23 | # Case classification vocabulary (row field ``classification``).
|
| 24 | HELD = "held"
|
| 25 | VIOLATED = "violated"
|
| 26 | UNSUPPORTED = "unsupported"
|
| 27 | HARNESS_ERROR = "harness_error"
|
| 28 | # The injected fault never took effect (the kill landed after the operation had
|
| 29 | # already exited, the disk was not actually short of the op's need, ...). The
|
| 30 | # row records a clean run, so it is never evidence that an invariant held.
|
| 31 | FAULT_NOT_FIRED = "fault_not_fired"
|
| 32 | CLASSIFICATIONS = (HELD, VIOLATED, UNSUPPORTED, HARNESS_ERROR, FAULT_NOT_FIRED)
|
| 33 |
|
| 34 | # Known Oak bugs a violation maps to (subject_kind, family prefix, operations, invariant) -> feedback id.
|
| 35 | # Rows whose violations match carry ``known_bugs``; unmatched violations are new findings.
|
| 36 | KNOWN_BUGS = (
|
| 37 | ("oak", "crash_", ("switch", "pull_ff", "pull_diverged"), "worktree_not_torn", "fb-535"),
|
| 38 | # same defect reached through an I/O error instead of a kill: a pull that fails midway
|
| 39 | # leaves its partial output presented as user changes
|
| 40 | ("oak", "env_", ("pull_ff",), "worktree_not_torn", "fb-535"),
|
| 41 | ("oak", "fault_proxy", ("pull_ff", "pull_diverged", "clone"), "worktree_not_torn", "fb-535"),
|
| 42 | ("oak", "conc_same_branch", ("concurrent.push",), "no_lost_acknowledged_write", "fb-534"),
|
| 43 | ("oak", "env_fsync_load", ("concurrent.push",), "no_lost_acknowledged_write", "fb-534"),
|
| 44 | )
|
| 45 |
|
| 46 |
|
| 47 | def known_bugs(subject_kind: str, scenario: str, operation: str, violations: list) -> list:
|
| 48 | """Feedback ids of known bugs matched by a row's violations (sorted, unique)."""
|
| 49 | found = set()
|
| 50 | for kind, prefix, ops, invariant, bug in KNOWN_BUGS:
|
| 51 | if kind == subject_kind and scenario.startswith(prefix) and operation in ops and invariant in violations:
|
| 52 | found.add(bug)
|
| 53 | return sorted(found)
|