Log in
benchmarksPublic

Commit 1a441eda

· Browse files at this commit

1a441edafc71

Merged on mrmrs-95dda3
mrmrs · 1 month ago · 15 files +2 added ~13 modified
M scripts/oakbench/rows.py
@@ -123,7 +123,8 @@
123123 ),
124124 # Robustness lane (suites/robustness): one row per fault case, judged by
125125 # independent readers. classification is held | violated | unsupported |
126- # harness_error; unsupported cases are skip rows (returncode 77).
126+ # harness_error | fault_not_fired; unsupported and fault_not_fired cases are
127+ # skip rows (returncode 77). See ROBUSTNESS_CLASS_RETURNCODE.
127128 "robustness": (
128129 "bench_id",
129130 "scenario",
@@ -197,6 +198,38 @@
197198 """Raised after rows are persisted; fails the producing run loudly."""
198199
199200
201+# Robustness lane: every classification and the returncode it must carry.
202+ROBUSTNESS_CLASS_RETURNCODE = {
203+ "held": 0,
204+ "violated": 1,
205+ "harness_error": 2,
206+ "unsupported": SKIP_RETURNCODE,
207+ "fault_not_fired": SKIP_RETURNCODE,
208+}
209+
210+
211+def _robustness_row_errors(index: int, row: dict[str, Any]) -> list[str]:
212+ op = row.get("operation", "?")
213+ classification = row.get("classification")
214+ if classification not in ROBUSTNESS_CLASS_RETURNCODE:
215+ return [f"row {index} ({op}): unknown robustness classification {classification!r}; "
216+ f"known: {', '.join(ROBUSTNESS_CLASS_RETURNCODE)}"]
217+ errors: list[str] = []
218+ expected = ROBUSTNESS_CLASS_RETURNCODE[classification]
219+ if row.get("returncode") != expected:
220+ errors.append(f"row {index} ({op}): classification {classification!r} requires returncode {expected}, "
221+ f"got {row.get('returncode')!r}")
222+ if classification in ("held", "violated") and row.get("fault_fired") is False:
223+ errors.append(f"row {index} ({op}): a case whose fault did not fire cannot be {classification!r} "
224+ "(use fault_not_fired)")
225+ violations = row.get("violations")
226+ if classification == "held" and violations:
227+ errors.append(f"row {index} ({op}): held row lists violations {violations!r}")
228+ if classification == "violated" and "violations" in row and not violations:
229+ errors.append(f"row {index} ({op}): violated row lists no violations")
230+ return errors
231+
232+
200233 def validate_rows(rows: list[dict[str, Any]], lane: str) -> list[str]:
201234 if lane not in LANE_REQUIRED_KEYS:
202235 return [f"unknown lane {lane!r}; known: {', '.join(LANES)}"]
@@ -265,6 +298,8 @@
265298 errors.append(
266299 f"row {index} ({row.get('operation', '?')}): skipped: true requires returncode {SKIP_RETURNCODE}"
267300 )
301+ if lane == "robustness":
302+ errors.extend(_robustness_row_errors(index, row))
268303 return errors
269304
270305