Log in
benchmarksPublic

Commit 76308956

· Browse files at this commit

7630895676ca

Merged from mrmrs-0aad77
mrmrs · 1 month ago · 15 files ~15 modified
M scripts/oakbench/conformance_gate.py
@@ -26,7 +26,14 @@
2626 measured behaviours are UNMEASURED. publish_gate additionally requires a
2727 recorded profile;
2828 - harness errors, reader-unmeasured legs and required behaviours with no row
29- make the result UNMEASURED.
29+ make the result UNMEASURED;
30+- error patterns must be specific: the loader rejects any pattern that matches
31+ the empty string, matches a nonsense/generic sentinel string (``panicked``,
32+ ``corrupt``, random text ...) or contains no literal character at all
33+ (only wildcards/classes), because such a pattern would hide any error
34+ under the expected failure;
35+- ``pending:`` tickets are listed in every summary; publish_gate refuses any
36+ entry that still carries one (devloop only notes them).
3037
3138 Only the gated subject (Oak) is judged; Git rows are the reference arm.
3239 """
@@ -34,8 +41,15 @@
3441
3542 import json
3643 from pathlib import Path
44+import random
3745 import re
46+import string
3847
48+try: # Python >= 3.11
49+ import re._parser as _sre_parse # type: ignore[import-not-found]
50+except ImportError: # pragma: no cover - Python < 3.11
51+ import sre_parse as _sre_parse # type: ignore[no-redef]
52+
3953 ROOT = Path(__file__).resolve().parents[2]
4054 EXPECTED_FAILURES = ROOT / "config" / "conformance_expected_failures.json"
4155 SCHEMA_VERSION = 2
@@ -52,6 +66,75 @@
5266 "non_utf8_names"})
5367
5468
69+# A legitimate expected-failure pattern pins one concrete error; it must match
70+# none of these. They cover generic failure words (so "panicked"/"corrupt"
71+# cannot hide under an xfail), the kind of text a catch-all pattern swallows,
72+# and deterministic random noise.
73+_rng = random.Random(0x0A4B)
74+SENTINELS = ("", " ", "\n", "x", "panicked", "corrupt", "error", "failed", "zzz-sentinel",
75+ "thread 'main' panicked at src/main.rs:1:1", "database disk image is malformed",
76+ "internal error", "HTTP 500 Internal Server Error", "Segmentation fault: 11",
77+ "zzz-sentinel: no such behaviour exited 99",
78+ *("".join(_rng.choice(string.ascii_letters + string.digits + string.punctuation + " ")
79+ for _ in range(_rng.randint(1, 60))) for _ in range(24)))
80+del _rng
81+# Subject-shaped sentinels: real conformance leg details look like
82+# "oak <cmd> exited N: error: ..." (legacy: "command (...) failed (N): b'...'"),
83+# so a pattern like `oak .* exited \d+: .*` or `.*error: .*` would swallow a panic,
84+# a corrupt store or a server 500 in that exact shape. None may match.
85+_BAD_ERRORS = ("thread 'main' panicked at src/main.rs:1:1", "error: corrupt object 0123abcd",
86+ "error: Database error: database disk image is malformed",
87+ "error: Server error: HTTP 500 Internal Server Error: internal error",
88+ "error: Database error: unable to open database file", "fatal: bad object HEAD")
89+_COMMANDS = ("oak commit --no-verify", "oak push --repo", "oak clone conformance/r1-2", "oak pull", "oak status",
90+ "git commit -q", "git push -q", "git clone -q", "git pull -q")
91+SUBJECT_SENTINELS = tuple(
92+ [f"{cmd} exited {rc}: {err}" for cmd in _COMMANDS for rc in (1, 2, 6, 101, 128) for err in _BAD_ERRORS]
93+ + [f"command ('commit', '--no-verify') failed ({rc}): b\"{err}\\n\"" for rc in (1, 2, 101)
94+ for err in _BAD_ERRORS]
95+ + [f"command ('commit', '--no-verify') failed ({rc}): b'{err}\\n'" for rc in (1, 2, 101)
96+ for err in _BAD_ERRORS if "'" not in err]
97+ + [err for err in _BAD_ERRORS] + [f"oak: {err}" for err in _BAD_ERRORS])
98+SENTINELS = SENTINELS + SUBJECT_SENTINELS
99+del _BAD_ERRORS, _COMMANDS
100+
101+
102+def _has_literal(parsed) -> bool:
103+ """True if the parsed regex contains a literal character outside a character class."""
104+ for op, arg in parsed:
105+ name = str(op)
106+ if name == "LITERAL":
107+ return True
108+ if name in ("NOT_LITERAL", "IN", "ANY", "AT", "CATEGORY", "GROUPREF", "GROUPREF_EXISTS"):
109+ continue
110+ if name == "SUBPATTERN":
111+ if _has_literal(arg[-1]):
112+ return True
113+ elif name in ("MAX_REPEAT", "MIN_REPEAT", "POSSESSIVE_REPEAT"):
114+ low, _high, sub = arg
115+ if low > 0 and _has_literal(sub):
116+ return True
117+ elif name == "BRANCH":
118+ # every alternative must carry a literal for the branch to count
119+ if all(_has_literal(alt) for alt in arg[1]):
120+ return True
121+ elif name == "ATOMIC_GROUP":
122+ if _has_literal(arg):
123+ return True
124+ return False
125+
126+
127+def pattern_too_broad(pattern: str) -> str | None:
128+ """Why an expected-failure error pattern is a catch-all, or None if it is specific."""
129+ compiled = re.compile(pattern, re.S)
130+ for sentinel in SENTINELS:
131+ if compiled.fullmatch(sentinel):
132+ return f"matches the sentinel {sentinel!r}"
133+ if not _has_literal(_sre_parse.parse(pattern, re.S)):
134+ return "has no required literal text (wildcards/classes only)"
135+ return None
136+
137+
55138 def fs_dependent_skip_allowed(behaviour: str) -> bool:
56139 from oakbench.conformance_matrix import BY_ID
57140 spec = BY_ID.get(behaviour)
@@ -78,6 +161,13 @@
78161 for leg, pattern in legs.items():
79162 if not isinstance(pattern, str) or not pattern:
80163 raise ValueError(f"conformance expected-failures: {behaviour}/{leg} needs an error pattern")
164+ try:
165+ broad = pattern_too_broad(pattern)
166+ except re.error as exc:
167+ raise ValueError(f"conformance expected-failures: {behaviour}/{leg} pattern invalid: {exc}")
168+ if broad:
169+ raise ValueError(f"conformance expected-failures: {behaviour}/{leg} pattern {pattern!r} is a "
170+ f"catch-all ({broad}); pin the concrete error")
81171 compiled[leg] = re.compile(pattern, re.S)
82172 profiles = entry.get("fs_profiles")
83173 if profiles is not None and (not isinstance(profiles, list) or not profiles):
@@ -89,6 +179,12 @@
89179 return {"subject": data.get("subject", "oak"), "entries": entries, "coverage": coverage}
90180
91181
182+def pending_tickets(expected: dict) -> list:
183+ """[(behaviour, ticket, outcome)] for entries whose ticket is still pending:<slug>."""
184+ return sorted((behaviour, entry["ticket"], entry["outcome"]) for behaviour, entry in expected["entries"].items()
185+ if str(entry["ticket"]).startswith("pending:"))
186+
187+
92188 def row_failed_legs(row: dict) -> dict:
93189 """{failed leg: detail} for a failing row (legacy rows: {'behaviour': failure})."""
94190 legs = row.get("legs")