| 1 | """Trial engine: copy a pristine fixture, inject one fault, judge invariants.
|
| 2 |
|
| 3 | Invariants (each True / False / None=unmeasured, ADR-0002):
|
| 4 |
|
| 5 | - ``op_no_hang`` the faulted operation ended (killed or exited) within its timeout
|
| 6 | - ``repo_opens`` independent open: SQLite integrity_check (Oak) / git fsck (Git)
|
| 7 | - ``acked_local_reachable`` every commit a local ref named before the op is still reachable
|
| 8 | from a local ref (branch head / HEAD / tag), per the independent reader
|
| 9 | - ``server_intact`` remote store opens, its pre-op heads stay reachable, and the branch
|
| 10 | head is either the pre-op head or the operation's intended head
|
| 11 | - ``worktree_consistent`` the worktree equals the pre- or post-op state, or a committed tree,
|
| 12 | or the subject's status reports it dirty/in-progress (never "clean"
|
| 13 | while differing from HEAD)
|
| 14 | - ``next_op_ok_or_actionable`` rerunning the operation succeeds, or fails with a typed/actionable
|
| 15 | message (not a hang, panic, or signal)
|
| 16 | - ``no_false_success`` a reported success (exit 0) leaves exactly the calibrated outcome
|
| 17 | - ``acked_after_next`` pre-op local commits are still reachable after the next operation
|
| 18 | """
|
| 19 |
|
| 20 | from __future__ import annotations
|
| 21 |
|
| 22 | import json
|
| 23 | import re
|
| 24 | import shutil
|
| 25 | import time
|
| 26 | from dataclasses import dataclass, field
|
| 27 | from pathlib import Path
|
| 28 | from typing import Any, Callable, Optional
|
| 29 |
|
| 30 | from . import FAULT_NOT_FIRED, HARNESS_ERROR, HELD, VIOLATED
|
| 31 | from .adapters import GitAdapter, OakAdapter, Server
|
| 32 | from .fixtures import Fixture
|
| 33 | from .procs import Result, copy_tree
|
| 34 | from .readers import OakGraph
|
| 35 |
|
| 36 | # A failure is actionable only if it tells the user what to do next:
|
| 37 | # a concrete recovery command (a real subcommand of the subject, or a recovery
|
| 38 | # flag), an explicit manual step, a documented retryable condition, or a typed,
|
| 39 | # specific JSON error code / recommended command list. Mentioning "oak" or
|
| 40 | # "git" is not enough ("failed to open oak database", "not a git repository").
|
| 41 | RECOVERY_SUBCOMMANDS = (
|
| 42 | "pull", "push", "commit", "fetch", "clone", "switch", "checkout", "merge", "rebase", "reset", "restore",
|
| 43 | "stash", "add", "rm", "status", "init", "close", "am", "cherry-pick", "revert", "gc", "fsck", "prune",
|
| 44 | "doctor", "sparse", "conflict", "login", "finish", "desc")
|
| 45 | RECOVERY_COMMAND_RE = re.compile(
|
| 46 | r"\b(?:oak|git)\s+(?:" + "|".join(re.escape(c) for c in RECOVERY_SUBCOMMANDS) + r")\b"
|
| 47 | r"|(?:^|\s)--(?:continue|abort|skip|force)\b", re.I)
|
| 48 | RECOVERY_STEP_RE = re.compile(
|
| 49 | r"remove the file manually|remove this directory|move or remove|delete (?:it|this directory|the directory)"
|
| 50 | r"|commit your changes|stash them|\bpull (?:changes )?first\b|\((?:fetch first|non-fast-forward)\)", re.I)
|
| 51 | RETRYABLE_RE = re.compile(r"\btry again\b|\bretry\b|\bre-?run\b|repository (?:is )?locked|lock(?:ed)? by (?:another|pid)",
|
| 52 | re.I)
|
| 53 | # Corruption / internal-failure wording: never actionable on retry wording or a generic code alone.
|
| 54 | NON_ACTIONABLE_RE = re.compile(
|
| 55 | r"panicked at|RUST_BACKTRACE|not a database|malformed|corrupt|internal (?:server )?error|\binternal\b"
|
| 56 | r"|integrity check failed|unexpected (?:eof|end)", re.I)
|
| 57 | GENERIC_ERROR_CODES = {"error", "internal", "internal_error", "unknown", "unknown_error", "io", "io_error", "other",
|
| 58 | "panic", "database", "database_error", "sqlite", "unexpected", "server_error", "http_error"}
|
| 59 |
|
| 60 |
|
| 61 | def _json_error(stdout: str) -> Optional[dict]:
|
| 62 | try:
|
| 63 | data = json.loads(stdout)
|
| 64 | except (ValueError, TypeError):
|
| 65 | return None
|
| 66 | if isinstance(data, dict) and isinstance(data.get("error"), dict):
|
| 67 | return data["error"]
|
| 68 | return None
|
| 69 |
|
| 70 |
|
| 71 | def actionable(res: Result) -> bool:
|
| 72 | """True when a failed command tells the user a concrete next step."""
|
| 73 | text = res.text()
|
| 74 | if res.returncode is None or res.returncode <= 0:
|
| 75 | return False
|
| 76 | if "panicked at" in text or "RUST_BACKTRACE" in text:
|
| 77 | return False
|
| 78 | if RECOVERY_COMMAND_RE.search(text) or RECOVERY_STEP_RE.search(text):
|
| 79 | return True
|
| 80 | broken = bool(NON_ACTIONABLE_RE.search(text))
|
| 81 | error = _json_error(res.stdout)
|
| 82 | if error is not None and not broken:
|
| 83 | code = str(error.get("code") or "").strip().lower()
|
| 84 | if error.get("recommended_next_commands"):
|
| 85 | return True
|
| 86 | if code and code not in GENERIC_ERROR_CODES and not code.startswith(("internal", "unknown", "io_")):
|
| 87 | return True
|
| 88 | return not broken and bool(RETRYABLE_RE.search(text))
|
| 89 |
|
| 90 |
|
| 91 | @dataclass
|
| 92 | class Calibration:
|
| 93 | elapsed_ms: float
|
| 94 | post_tree: Optional[dict]
|
| 95 | post_worktree: Optional[dict]
|
| 96 | post_head: Optional[str]
|
| 97 | post_server_head: Optional[str]
|
| 98 | boundaries: int = 0
|
| 99 | boundary_kinds: list = field(default_factory=list)
|
| 100 | requests: list = field(default_factory=list)
|
| 101 | ok: bool = True
|
| 102 | detail: str = ""
|
| 103 |
|
| 104 |
|
| 105 | class Trial:
|
| 106 | """One copied fixture + (for Oak) its loopback server, ready for one faulted op."""
|
| 107 |
|
| 108 | def __init__(self, fixture: Fixture, adapter_bin: str, work: Path, name: str):
|
| 109 | self.fixture = fixture
|
| 110 | self.dir = work / name
|
| 111 | copy_tree(fixture.pristine, self.dir)
|
| 112 | self.home = self.dir / "home"
|
| 113 | self.kind = fixture.kind
|
| 114 | if self.kind == "oak":
|
| 115 | self.adapter = OakAdapter(adapter_bin, self.home)
|
| 116 | self.server: Optional[Server] = Server(adapter_bin, self.dir / "srv", fixture.port or 0,
|
| 117 | self.dir / "serve.log")
|
| 118 | else:
|
| 119 | self.adapter = GitAdapter(adapter_bin, self.home)
|
| 120 | self.server = None
|
| 121 | self.client = self.dir / "client"
|
| 122 |
|
| 123 | # op plumbing ---------------------------------------------------------------
|
| 124 | def op_cwd(self) -> Path:
|
| 125 | return self.dir if self.fixture.op == "clone" else self.client
|
| 126 |
|
| 127 | def op_argv(self, bin_override: Optional[str] = None, dest: str = "cloned", url: Optional[str] = None) -> list[str]:
|
| 128 | kw: dict[str, Any] = {"bin": bin_override}
|
| 129 | if self.fixture.op == "clone":
|
| 130 | kw.update(dest=dest, branch=self.fixture.branch, url=url or self.fixture.meta["url"])
|
| 131 | if self.fixture.op == "switch":
|
| 132 | kw["target"] = self.fixture.meta["target"]
|
| 133 | return self.adapter.argv(self.fixture.op, **kw)
|
| 134 |
|
| 135 | def pre(self) -> None:
|
| 136 | for argv in self.adapter.pre_argv(self.fixture.op):
|
| 137 | self.adapter.must(argv, self.client)
|
| 138 |
|
| 139 | def start_server(self, port: Optional[int] = None) -> None:
|
| 140 | if self.server is not None:
|
| 141 | self.server.start(port)
|
| 142 |
|
| 143 | def stop_server(self) -> None:
|
| 144 | if self.server is not None:
|
| 145 | self.server.stop()
|
| 146 |
|
| 147 | def settle(self) -> None:
|
| 148 | """Let orphaned server-side children (Git local receive-pack) finish."""
|
| 149 | if self.kind == "git":
|
| 150 | remote = self.dir / "srv" / "remote.git"
|
| 151 | deadline = time.monotonic() + 10
|
| 152 | while time.monotonic() < deadline and any(remote.rglob("*.lock")):
|
| 153 | time.sleep(0.05)
|
| 154 | time.sleep(0.1)
|
| 155 |
|
| 156 | def target_repo(self, dest: str = "cloned") -> Path:
|
| 157 | return self.dir / dest if self.fixture.op == "clone" else self.client
|
| 158 |
|
| 159 | def cleanup(self) -> None:
|
| 160 | self.stop_server()
|
| 161 | shutil.rmtree(self.dir, ignore_errors=True)
|
| 162 |
|
| 163 | # independent observations ---------------------------------------------------
|
| 164 | def server_head(self) -> Optional[str]:
|
| 165 | if self.kind == "oak":
|
| 166 | db = self.server.db() if self.server else None
|
| 167 | if not db or not db.exists():
|
| 168 | return None
|
| 169 | with OakGraph(db) as g:
|
| 170 | return g.branch_heads().get(self.fixture.branch)
|
| 171 | refs = self.adapter.oracle.refs(self.dir / "srv" / "remote.git")
|
| 172 | return refs.get(f"refs/heads/{self.fixture.branch}")
|
| 173 |
|
| 174 | def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
|
| 175 | if self.kind == "oak":
|
| 176 | from .readers import oak_commit_tree
|
| 177 | return oak_commit_tree(self.server.db(), commit)
|
| 178 | return self.adapter.oracle.commit_tree(self.dir / "srv" / "remote.git", commit)
|
| 179 |
|
| 180 | def repo_opens(self, repo: Path) -> Optional[bool]:
|
| 181 | if self.kind == "oak":
|
| 182 | db = repo / ".oak" / "oak.db"
|
| 183 | if not db.exists():
|
| 184 | return False
|
| 185 | try:
|
| 186 | with OakGraph(db) as g:
|
| 187 | return g.integrity() == "ok" and g.metadata("current_branch") is not None
|
| 188 | except Exception: # noqa: BLE001 - any failure to open is the finding
|
| 189 | return False
|
| 190 | if not (repo / ".git").exists():
|
| 191 | return False
|
| 192 | return self.adapter.oracle.fsck(repo) == "ok"
|
| 193 |
|
| 194 | def acked_reachable(self, repo: Path, heads: list[str]) -> tuple[Optional[bool], list[str]]:
|
| 195 | if not heads:
|
| 196 | return None, []
|
| 197 | try:
|
| 198 | if self.kind == "oak":
|
| 199 | with OakGraph(repo / ".oak" / "oak.db") as g:
|
| 200 | reach = g.reachable()
|
| 201 | lost = [h for h in heads if h not in reach]
|
| 202 | else:
|
| 203 | lost = [h for h in heads if not self.adapter.oracle.reachable_from_refs(repo, h)]
|
| 204 | except Exception as exc: # noqa: BLE001 - reader failure is unmeasured, never a subject violation
|
| 205 | return None, [f"reader error: {type(exc).__name__}: {exc}"]
|
| 206 | return not lost, lost
|
| 207 |
|
| 208 | def server_intact(self, intended: Optional[str]) -> tuple[Optional[bool], dict]:
|
| 209 | pre = self.fixture.meta.get("pre_server_heads", {})
|
| 210 | detail: dict = {}
|
| 211 | try:
|
| 212 | if self.kind == "oak":
|
| 213 | with OakGraph(self.server.db()) as g:
|
| 214 | detail["integrity"] = g.integrity()
|
| 215 | heads = g.branch_heads()
|
| 216 | reach = g.ancestors(heads.values())
|
| 217 | missing = g.missing_parents([h for h in heads.values()])
|
| 218 | else:
|
| 219 | remote = self.dir / "srv" / "remote.git"
|
| 220 | detail["integrity"] = self.adapter.oracle.fsck(remote)
|
| 221 | refs = self.adapter.oracle.refs(remote)
|
| 222 | heads = {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
|
| 223 | reach = set()
|
| 224 | for v in pre.values():
|
| 225 | if any(self.adapter.oracle.is_ancestor(remote, v, h) for h in heads.values()):
|
| 226 | reach.add(v)
|
| 227 | missing = []
|
| 228 | except Exception as exc: # noqa: BLE001 - reader failure is unmeasured, never a subject violation
|
| 229 | return None, {"reader_error": f"{type(exc).__name__}: {exc}"}
|
| 230 | head = heads.get(self.fixture.branch)
|
| 231 | detail.update(server_head=head, lost_pre_heads=[v for v in pre.values() if v not in reach],
|
| 232 | torn_graph=missing[:5])
|
| 233 | allowed = {pre.get(self.fixture.branch), intended}
|
| 234 | detail["head_in_allowed"] = head in allowed
|
| 235 | ok = detail["integrity"] == "ok" and not detail["lost_pre_heads"] and not missing and head in allowed
|
| 236 | return ok, detail
|
| 237 |
|
| 238 |
|
| 239 | def worktree_verdict(trial: Trial, repo: Path, allowed: list[Optional[dict]]) -> tuple[Optional[bool], dict]:
|
| 240 | from .readers import worktree_manifest
|
| 241 |
|
| 242 | wt = worktree_manifest(repo)
|
| 243 | if wt is None:
|
| 244 | return None, {"worktree": "unreadable"}
|
| 245 | if any(a is not None and wt == a for a in allowed):
|
| 246 | return True, {"worktree": "pre_or_post"}
|
| 247 | head_tree, status = trial.adapter.tree(repo)
|
| 248 | if head_tree is not None and wt == head_tree:
|
| 249 | return True, {"worktree": "matches_head_tree"}
|
| 250 | dirty, res = trial.adapter.status_dirty(repo)
|
| 251 | if dirty is None:
|
| 252 | # status refused with an error: acceptable only if it is actionable
|
| 253 | ok = actionable(res)
|
| 254 | return ok, {"worktree": "differs; status errored", "status_actionable": ok, "status": res.brief(300),
|
| 255 | "head_tree_reader": status}
|
| 256 | if dirty:
|
| 257 | return True, {"worktree": "differs; reported dirty", "head_tree_reader": status}
|
| 258 | if head_tree is None:
|
| 259 | return None, {"worktree": "differs; status clean; head tree unreadable", "head_tree_reader": status}
|
| 260 | return False, {"worktree": "SILENT_TORN: differs from HEAD tree while status reports clean"}
|
| 261 |
|
| 262 |
|
| 263 | def torn_worktree(wt: Optional[dict], pre: Optional[dict], post: Optional[dict]) -> Optional[dict]:
|
| 264 | """Split a crashed worktree against the interrupted operation's two sides.
|
| 265 |
|
| 266 | Returns None when unmeasurable, else lists of paths the op changes that
|
| 267 | already hold the post-op (target) side (``op_output``) or still hold the
|
| 268 | pre-op side (``pre_side``), plus op artifacts in neither side (``stray``,
|
| 269 | e.g. temp files).
|
| 270 | """
|
| 271 | if wt is None or pre is None or post is None:
|
| 272 | return None
|
| 273 | changed_by_op = {p for p in set(pre) | set(post) if pre.get(p) != post.get(p)}
|
| 274 | op_output, pre_side, stray = [], [], []
|
| 275 | for path in sorted(set(wt) | changed_by_op):
|
| 276 | have = wt.get(path)
|
| 277 | if path in changed_by_op:
|
| 278 | if have == post.get(path):
|
| 279 | op_output.append(path)
|
| 280 | elif have == pre.get(path):
|
| 281 | pre_side.append(path)
|
| 282 | elif path not in pre and path not in post and have is not None:
|
| 283 | stray.append(path)
|
| 284 | return {"op_output": op_output, "pre_side": pre_side, "stray": stray}
|
| 285 |
|
| 286 |
|
| 287 | def torn_state(split: dict, ref_side: Optional[str]) -> bool:
|
| 288 | """Is the worktree out of step with where the ref says the operation is?
|
| 289 |
|
| 290 | ref_side "pre": any of the op's output (or its temp files) is present.
|
| 291 | ref_side "post": any changed path still on the pre side (or temp files).
|
| 292 | unknown ref: the worktree mixes both sides, or has temp files.
|
| 293 | """
|
| 294 | if split["stray"]:
|
| 295 | return True
|
| 296 | if ref_side == "pre":
|
| 297 | return bool(split["op_output"])
|
| 298 | if ref_side == "post":
|
| 299 | return bool(split["pre_side"])
|
| 300 | return bool(split["op_output"]) and bool(split["pre_side"])
|
| 301 |
|
| 302 |
|
| 303 | def torn_verdict(trial: "Trial", repo: Path, pre: Optional[dict], post: Optional[dict],
|
| 304 | pre_head: Optional[str], op_res: Optional[Result] = None) -> tuple[Optional[bool], dict]:
|
| 305 | """Invariant ``worktree_not_torn``.
|
| 306 |
|
| 307 | Fails when, after the crash, the worktree holds the interrupted operation's
|
| 308 | own output out of step with the ref (e.g. files already on the target side
|
| 309 | while HEAD still names the old side, or the op's temp files) and the
|
| 310 | subject presents it as ordinary uncommitted user changes: status lists
|
| 311 | changes, reports no operation in progress, and does not refuse. Committing
|
| 312 | that state would record the op's partial output as the user's own work.
|
| 313 | Passing: worktree in step with the ref, an in-progress flag (MERGE_HEAD /
|
| 314 | progress_state / a stale index.lock that blocks every mutation), a status
|
| 315 | refusal (judged by ``worktree_consistent``), or the failing op itself naming
|
| 316 | a recovery command (e.g. git's "checkout failed ... retry with git restore").
|
| 317 | A killed op prints nothing, so crash rows rely on the persistent flags only.
|
| 318 | """
|
| 319 | from .readers import worktree_manifest
|
| 320 |
|
| 321 | wt = worktree_manifest(repo)
|
| 322 | split = torn_worktree(wt, pre, post)
|
| 323 | if split is None:
|
| 324 | return None, {"torn": "unmeasurable"}
|
| 325 | head = trial.adapter.head(repo)
|
| 326 | # The op either leaves HEAD where it was or moves it to its result (a merge /
|
| 327 | # re-parent commit's hash differs per run, so "moved" is the post side).
|
| 328 | ref_side = None if head is None else ("pre" if head == pre_head else "post")
|
| 329 | detail = {"ref_side": ref_side, "op_output_files": len(split["op_output"]), "stray_files": len(split["stray"]),
|
| 330 | "pre_side_files": len(split["pre_side"]), "sample": (split["stray"] + split["op_output"])[:5]}
|
| 331 | if not torn_state(split, ref_side):
|
| 332 | return True, {**detail, "torn": "no"}
|
| 333 | if op_res is not None and (op_res.returncode or 0) > 0 and RECOVERY_COMMAND_RE.search(op_res.text()):
|
| 334 | return True, {**detail, "torn": "failing op named a recovery command"}
|
| 335 | changes, in_progress, res = trial.adapter.status_state(repo)
|
| 336 | detail["status_rc"] = res.returncode
|
| 337 | if changes is None:
|
| 338 | return True, {**detail, "torn": "status refused (judged by worktree_consistent)"}
|
| 339 | if in_progress:
|
| 340 | return True, {**detail, "torn": "flagged in progress"}
|
| 341 | if changes:
|
| 342 | return False, {**detail, "torn": "PARTIAL_OP_OUTPUT_AS_USER_CHANGES"}
|
| 343 | return True, {**detail, "torn": "status clean (judged by worktree_consistent)"}
|
| 344 |
|
| 345 |
|
| 346 | def classify(invariants: dict[str, Optional[bool]], harness_error: Optional[str] = None,
|
| 347 | fault_fired: Optional[bool] = None, reader_errors: Optional[list] = None) -> str:
|
| 348 | """held / violated / harness_error / fault_not_fired.
|
| 349 |
|
| 350 | ``fault_fired=False`` (the fault demonstrably never took effect) is never
|
| 351 | ``held``: the row is a clean run. ``None`` means the family has no single
|
| 352 | fault to fire (concurrency). Reader exceptions are harness errors unless a
|
| 353 | subject-level invariant already failed independently of the reader.
|
| 354 | """
|
| 355 | if harness_error:
|
| 356 | return HARNESS_ERROR
|
| 357 | if fault_fired is False:
|
| 358 | return FAULT_NOT_FIRED
|
| 359 | if any(v is False for v in invariants.values()):
|
| 360 | return VIOLATED
|
| 361 | if reader_errors:
|
| 362 | return HARNESS_ERROR
|
| 363 | return HELD
|
| 364 |
|
| 365 |
|
| 366 | def converge_check(trial: Trial, calib: Calibration, repo: Path) -> tuple[Optional[bool], dict]:
|
| 367 | """After a successful next op, is the state exactly the calibrated outcome?"""
|
| 368 | from .readers import worktree_manifest
|
| 369 |
|
| 370 | op = trial.fixture.op
|
| 371 | detail: dict = {}
|
| 372 | if op == "push":
|
| 373 | local = trial.adapter.head(repo)
|
| 374 | remote = trial.server_head()
|
| 375 | detail.update(local_head=local, server_head=remote)
|
| 376 | if remote is None or local is None or remote != local:
|
| 377 | return False, detail
|
| 378 | tree, status = trial.server_tree(remote)
|
| 379 | detail["server_tree_reader"] = status
|
| 380 | if tree is None:
|
| 381 | return None, detail
|
| 382 | return tree == calib.post_tree, detail
|
| 383 | if op == "switch":
|
| 384 | branch = trial.adapter.current_branch(repo)
|
| 385 | detail["branch"] = branch
|
| 386 | return branch == trial.fixture.meta["target"] and worktree_manifest(repo) == calib.post_worktree, detail
|
| 387 | tree, status = trial.adapter.tree(repo)
|
| 388 | detail["head_tree_reader"] = status
|
| 389 | if tree is None:
|
| 390 | return (False if status.startswith("corrupt") else None), detail
|
| 391 | same_tree = tree == calib.post_tree
|
| 392 | same_wt = worktree_manifest(repo) == calib.post_worktree
|
| 393 | detail.update(head_tree_matches=same_tree, worktree_matches=same_wt)
|
| 394 | if op == "clone":
|
| 395 | detail["head"] = trial.adapter.head(repo)
|
| 396 | return same_tree and same_wt, detail
|