Log in
benchmarksPublic

Commit 5717cc9f

· Browse files at this commit

5717cc9fc7ed

Merged on mrmrs-0aad77
mrmrs · 1 month ago · 15 files ~15 modified
M scripts/oakbench/conformance_gate.py
@@ -26,7 +26,14 @@
2626 measured behaviours are UNMEASURED. publish_gate additionally requires a
2727 recorded profile;
2828 - harness errors, reader-unmeasured legs and required behaviours with no row
29- make the result UNMEASURED.
29+ make the result UNMEASURED;
30+- error patterns must be specific: the loader rejects any pattern that matches
31+ the empty string, matches a nonsense/generic sentinel string (``panicked``,
32+ ``corrupt``, random text ...) or contains no literal character at all
33+ (only wildcards/classes), because such a pattern would hide any error
34+ under the expected failure;
35+- ``pending:`` tickets are listed in every summary; publish_gate refuses any
36+ entry that still carries one (devloop only notes them).
3037
3138 Only the gated subject (Oak) is judged; Git rows are the reference arm.
3239 """
@@ -34,7 +41,14 @@
3441
3542 import json
3643 from pathlib import Path
44+import random
3745 import re
46+import string
47+
48+try: # Python >= 3.11
49+ import re._parser as _sre_parse # type: ignore[import-not-found]
50+except ImportError: # pragma: no cover - Python < 3.11
51+ import sre_parse as _sre_parse # type: ignore[no-redef]
3852
3953 ROOT = Path(__file__).resolve().parents[2]
4054 EXPECTED_FAILURES = ROOT / "config" / "conformance_expected_failures.json"
@@ -52,6 +66,56 @@
5266 "non_utf8_names"})
5367
5468
69+# A legitimate expected-failure pattern pins one concrete error; it must match
70+# none of these. They cover generic failure words (so "panicked"/"corrupt"
71+# cannot hide under an xfail), the kind of text a catch-all pattern swallows,
72+# and deterministic random noise.
73+_rng = random.Random(0x0A4B)
74+SENTINELS = ("", " ", "\n", "x", "panicked", "corrupt", "error", "failed", "zzz-sentinel",
75+ "thread 'main' panicked at src/main.rs:1:1", "database disk image is malformed",
76+ "internal error", "HTTP 500 Internal Server Error", "Segmentation fault: 11",
77+ "zzz-sentinel: no such behaviour exited 99",
78+ *("".join(_rng.choice(string.ascii_letters + string.digits + string.punctuation + " ")
79+ for _ in range(_rng.randint(1, 60))) for _ in range(24)))
80+del _rng
81+
82+
83+def _has_literal(parsed) -> bool:
84+ """True if the parsed regex contains a literal character outside a character class."""
85+ for op, arg in parsed:
86+ name = str(op)
87+ if name == "LITERAL":
88+ return True
89+ if name in ("NOT_LITERAL", "IN", "ANY", "AT", "CATEGORY", "GROUPREF", "GROUPREF_EXISTS"):
90+ continue
91+ if name == "SUBPATTERN":
92+ if _has_literal(arg[-1]):
93+ return True
94+ elif name in ("MAX_REPEAT", "MIN_REPEAT", "POSSESSIVE_REPEAT"):
95+ low, _high, sub = arg
96+ if low > 0 and _has_literal(sub):
97+ return True
98+ elif name == "BRANCH":
99+ # every alternative must carry a literal for the branch to count
100+ if all(_has_literal(alt) for alt in arg[1]):
101+ return True
102+ elif name == "ATOMIC_GROUP":
103+ if _has_literal(arg):
104+ return True
105+ return False
106+
107+
108+def pattern_too_broad(pattern: str) -> str | None:
109+ """Why an expected-failure error pattern is a catch-all, or None if it is specific."""
110+ compiled = re.compile(pattern, re.S)
111+ for sentinel in SENTINELS:
112+ if compiled.fullmatch(sentinel):
113+ return f"matches the sentinel {sentinel!r}"
114+ if not _has_literal(_sre_parse.parse(pattern, re.S)):
115+ return "has no required literal text (wildcards/classes only)"
116+ return None
117+
118+
55119 def fs_dependent_skip_allowed(behaviour: str) -> bool:
56120 from oakbench.conformance_matrix import BY_ID
57121 spec = BY_ID.get(behaviour)
@@ -78,6 +142,13 @@
78142 for leg, pattern in legs.items():
79143 if not isinstance(pattern, str) or not pattern:
80144 raise ValueError(f"conformance expected-failures: {behaviour}/{leg} needs an error pattern")
145+ try:
146+ broad = pattern_too_broad(pattern)
147+ except re.error as exc:
148+ raise ValueError(f"conformance expected-failures: {behaviour}/{leg} pattern invalid: {exc}")
149+ if broad:
150+ raise ValueError(f"conformance expected-failures: {behaviour}/{leg} pattern {pattern!r} is a "
151+ f"catch-all ({broad}); pin the concrete error")
81152 compiled[leg] = re.compile(pattern, re.S)
82153 profiles = entry.get("fs_profiles")
83154 if profiles is not None and (not isinstance(profiles, list) or not profiles):
@@ -89,6 +160,12 @@
89160 return {"subject": data.get("subject", "oak"), "entries": entries, "coverage": coverage}
90161
91162
163+def pending_tickets(expected: dict) -> list:
164+ """[(behaviour, ticket, outcome)] for entries whose ticket is still pending:<slug>."""
165+ return sorted((behaviour, entry["ticket"], entry["outcome"]) for behaviour, entry in expected["entries"].items()
166+ if str(entry["ticket"]).startswith("pending:"))
167+
168+
92169 def row_failed_legs(row: dict) -> dict:
93170 """{failed leg: detail} for a failing row (legacy rows: {'behaviour': failure})."""
94171 legs = row.get("legs")