| 1 | """Trial engine: copy a pristine fixture, inject one fault, judge invariants.
|
| 2 |
|
| 3 | Invariants (each True / False / None=unmeasured, ADR-0002):
|
| 4 |
|
| 5 | - ``op_no_hang`` the faulted operation ended (killed or exited) within its timeout
|
| 6 | - ``repo_opens`` independent open: SQLite integrity_check (Oak) / git fsck (Git)
|
| 7 | - ``acked_local_reachable`` every commit a local ref named before the op is still reachable
|
| 8 | from a local ref (branch head / HEAD / tag), per the independent reader
|
| 9 | - ``server_intact`` remote store opens, its pre-op heads stay reachable, and the branch
|
| 10 | head is either the pre-op head or the operation's intended head
|
| 11 | - ``worktree_consistent`` the worktree equals the pre- or post-op state, or a committed tree,
|
| 12 | or the subject's status reports it dirty/in-progress (never "clean"
|
| 13 | while differing from HEAD)
|
| 14 | - ``next_op_ok_or_actionable`` rerunning the operation succeeds, or fails with a typed/actionable
|
| 15 | message (not a hang, panic, or signal)
|
| 16 | - ``no_false_success`` a reported success (exit 0) leaves exactly the calibrated outcome
|
| 17 | - ``acked_after_next`` pre-op local commits are still reachable after the next operation
|
| 18 | """
|
| 19 |
|
| 20 | from __future__ import annotations
|
| 21 |
|
| 22 | import json
|
| 23 | import re
|
| 24 | import shutil
|
| 25 | import time
|
| 26 | from dataclasses import dataclass, field
|
| 27 | from pathlib import Path
|
| 28 | from typing import Any, Callable, Optional
|
| 29 |
|
| 30 | from . import HARNESS_ERROR, HELD, VIOLATED
|
| 31 | from .adapters import GitAdapter, OakAdapter, Server
|
| 32 | from .fixtures import Fixture
|
| 33 | from .procs import Result, copy_tree
|
| 34 | from .readers import OakGraph
|
| 35 |
|
| 36 | ACTIONABLE_RE = re.compile(
|
| 37 | r"\b(oak|git)\s+[a-z][\w-]*|manually|--continue|--abort|try again|retry|\bpull\b|re-?run|remove this|rejected|move or remove", re.I)
|
| 38 |
|
| 39 |
|
| 40 | def actionable(res: Result) -> bool:
|
| 41 | text = res.text()
|
| 42 | if res.returncode is None or res.returncode < 0:
|
| 43 | return False
|
| 44 | if "panicked at" in text or "RUST_BACKTRACE" in text:
|
| 45 | return False
|
| 46 | try:
|
| 47 | data = json.loads(res.stdout)
|
| 48 | if isinstance(data, dict) and isinstance(data.get("error"), dict) and data["error"].get("code"):
|
| 49 | return True
|
| 50 | except (ValueError, TypeError):
|
| 51 | pass
|
| 52 | return bool(text.strip()) and bool(ACTIONABLE_RE.search(text))
|
| 53 |
|
| 54 |
|
| 55 | @dataclass
|
| 56 | class Calibration:
|
| 57 | elapsed_ms: float
|
| 58 | post_tree: Optional[dict]
|
| 59 | post_worktree: Optional[dict]
|
| 60 | post_head: Optional[str]
|
| 61 | post_server_head: Optional[str]
|
| 62 | boundaries: int = 0
|
| 63 | boundary_kinds: list = field(default_factory=list)
|
| 64 | requests: list = field(default_factory=list)
|
| 65 | ok: bool = True
|
| 66 | detail: str = ""
|
| 67 |
|
| 68 |
|
| 69 | class Trial:
|
| 70 | """One copied fixture + (for Oak) its loopback server, ready for one faulted op."""
|
| 71 |
|
| 72 | def __init__(self, fixture: Fixture, adapter_bin: str, work: Path, name: str):
|
| 73 | self.fixture = fixture
|
| 74 | self.dir = work / name
|
| 75 | copy_tree(fixture.pristine, self.dir)
|
| 76 | self.home = self.dir / "home"
|
| 77 | self.kind = fixture.kind
|
| 78 | if self.kind == "oak":
|
| 79 | self.adapter = OakAdapter(adapter_bin, self.home)
|
| 80 | self.server: Optional[Server] = Server(adapter_bin, self.dir / "srv", fixture.port or 0,
|
| 81 | self.dir / "serve.log")
|
| 82 | else:
|
| 83 | self.adapter = GitAdapter(adapter_bin, self.home)
|
| 84 | self.server = None
|
| 85 | self.client = self.dir / "client"
|
| 86 |
|
| 87 | # op plumbing ---------------------------------------------------------------
|
| 88 | def op_cwd(self) -> Path:
|
| 89 | return self.dir if self.fixture.op == "clone" else self.client
|
| 90 |
|
| 91 | def op_argv(self, bin_override: Optional[str] = None, dest: str = "cloned", url: Optional[str] = None) -> list[str]:
|
| 92 | kw: dict[str, Any] = {"bin": bin_override}
|
| 93 | if self.fixture.op == "clone":
|
| 94 | kw.update(dest=dest, branch=self.fixture.branch, url=url or self.fixture.meta["url"])
|
| 95 | if self.fixture.op == "switch":
|
| 96 | kw["target"] = self.fixture.meta["target"]
|
| 97 | return self.adapter.argv(self.fixture.op, **kw)
|
| 98 |
|
| 99 | def pre(self) -> None:
|
| 100 | for argv in self.adapter.pre_argv(self.fixture.op):
|
| 101 | self.adapter.must(argv, self.client)
|
| 102 |
|
| 103 | def start_server(self, port: Optional[int] = None) -> None:
|
| 104 | if self.server is not None:
|
| 105 | self.server.start(port)
|
| 106 |
|
| 107 | def stop_server(self) -> None:
|
| 108 | if self.server is not None:
|
| 109 | self.server.stop()
|
| 110 |
|
| 111 | def settle(self) -> None:
|
| 112 | """Let orphaned server-side children (Git local receive-pack) finish."""
|
| 113 | if self.kind == "git":
|
| 114 | remote = self.dir / "srv" / "remote.git"
|
| 115 | deadline = time.monotonic() + 10
|
| 116 | while time.monotonic() < deadline and any(remote.rglob("*.lock")):
|
| 117 | time.sleep(0.05)
|
| 118 | time.sleep(0.1)
|
| 119 |
|
| 120 | def target_repo(self, dest: str = "cloned") -> Path:
|
| 121 | return self.dir / dest if self.fixture.op == "clone" else self.client
|
| 122 |
|
| 123 | def cleanup(self) -> None:
|
| 124 | self.stop_server()
|
| 125 | shutil.rmtree(self.dir, ignore_errors=True)
|
| 126 |
|
| 127 | # independent observations ---------------------------------------------------
|
| 128 | def server_head(self) -> Optional[str]:
|
| 129 | if self.kind == "oak":
|
| 130 | db = self.server.db() if self.server else None
|
| 131 | if not db or not db.exists():
|
| 132 | return None
|
| 133 | with OakGraph(db) as g:
|
| 134 | return g.branch_heads().get(self.fixture.branch)
|
| 135 | refs = self.adapter.oracle.refs(self.dir / "srv" / "remote.git")
|
| 136 | return refs.get(f"refs/heads/{self.fixture.branch}")
|
| 137 |
|
| 138 | def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
|
| 139 | if self.kind == "oak":
|
| 140 | from .readers import oak_commit_tree
|
| 141 | return oak_commit_tree(self.server.db(), commit)
|
| 142 | return self.adapter.oracle.commit_tree(self.dir / "srv" / "remote.git", commit)
|
| 143 |
|
| 144 | def repo_opens(self, repo: Path) -> Optional[bool]:
|
| 145 | if self.kind == "oak":
|
| 146 | db = repo / ".oak" / "oak.db"
|
| 147 | if not db.exists():
|
| 148 | return False
|
| 149 | try:
|
| 150 | with OakGraph(db) as g:
|
| 151 | return g.integrity() == "ok" and g.metadata("current_branch") is not None
|
| 152 | except Exception: # noqa: BLE001 - any failure to open is the finding
|
| 153 | return False
|
| 154 | if not (repo / ".git").exists():
|
| 155 | return False
|
| 156 | return self.adapter.oracle.fsck(repo) == "ok"
|
| 157 |
|
| 158 | def acked_reachable(self, repo: Path, heads: list[str]) -> tuple[Optional[bool], list[str]]:
|
| 159 | if not heads:
|
| 160 | return None, []
|
| 161 | try:
|
| 162 | if self.kind == "oak":
|
| 163 | with OakGraph(repo / ".oak" / "oak.db") as g:
|
| 164 | reach = g.reachable()
|
| 165 | lost = [h for h in heads if h not in reach]
|
| 166 | else:
|
| 167 | lost = [h for h in heads if not self.adapter.oracle.reachable_from_refs(repo, h)]
|
| 168 | except Exception as exc: # noqa: BLE001
|
| 169 | return False, [f"reader error: {exc}"]
|
| 170 | return not lost, lost
|
| 171 |
|
| 172 | def server_intact(self, intended: Optional[str]) -> tuple[Optional[bool], dict]:
|
| 173 | pre = self.fixture.meta.get("pre_server_heads", {})
|
| 174 | detail: dict = {}
|
| 175 | try:
|
| 176 | if self.kind == "oak":
|
| 177 | with OakGraph(self.server.db()) as g:
|
| 178 | detail["integrity"] = g.integrity()
|
| 179 | heads = g.branch_heads()
|
| 180 | reach = g.ancestors(heads.values())
|
| 181 | missing = g.missing_parents([h for h in heads.values()])
|
| 182 | else:
|
| 183 | remote = self.dir / "srv" / "remote.git"
|
| 184 | detail["integrity"] = self.adapter.oracle.fsck(remote)
|
| 185 | refs = self.adapter.oracle.refs(remote)
|
| 186 | heads = {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
|
| 187 | reach = set()
|
| 188 | for v in pre.values():
|
| 189 | if any(self.adapter.oracle.is_ancestor(remote, v, h) for h in heads.values()):
|
| 190 | reach.add(v)
|
| 191 | missing = []
|
| 192 | except Exception as exc: # noqa: BLE001
|
| 193 | return False, {"error": str(exc)}
|
| 194 | head = heads.get(self.fixture.branch)
|
| 195 | detail.update(server_head=head, lost_pre_heads=[v for v in pre.values() if v not in reach],
|
| 196 | torn_graph=missing[:5])
|
| 197 | allowed = {pre.get(self.fixture.branch), intended}
|
| 198 | detail["head_in_allowed"] = head in allowed
|
| 199 | ok = detail["integrity"] == "ok" and not detail["lost_pre_heads"] and not missing and head in allowed
|
| 200 | return ok, detail
|
| 201 |
|
| 202 |
|
| 203 | def worktree_verdict(trial: Trial, repo: Path, allowed: list[Optional[dict]]) -> tuple[Optional[bool], dict]:
|
| 204 | from .readers import worktree_manifest
|
| 205 |
|
| 206 | wt = worktree_manifest(repo)
|
| 207 | if wt is None:
|
| 208 | return None, {"worktree": "unreadable"}
|
| 209 | if any(a is not None and wt == a for a in allowed):
|
| 210 | return True, {"worktree": "pre_or_post"}
|
| 211 | head_tree, status = trial.adapter.tree(repo)
|
| 212 | if head_tree is not None and wt == head_tree:
|
| 213 | return True, {"worktree": "matches_head_tree"}
|
| 214 | dirty, res = trial.adapter.status_dirty(repo)
|
| 215 | if dirty is None:
|
| 216 | # status refused with an error: acceptable only if it is actionable
|
| 217 | ok = actionable(res)
|
| 218 | return ok, {"worktree": "differs; status errored", "status_actionable": ok, "status": res.brief(300),
|
| 219 | "head_tree_reader": status}
|
| 220 | if dirty:
|
| 221 | return True, {"worktree": "differs; reported dirty", "head_tree_reader": status}
|
| 222 | if head_tree is None:
|
| 223 | return None, {"worktree": "differs; status clean; head tree unreadable", "head_tree_reader": status}
|
| 224 | return False, {"worktree": "SILENT_TORN: differs from HEAD tree while status reports clean"}
|
| 225 |
|
| 226 |
|
| 227 | def classify(invariants: dict[str, Optional[bool]], harness_error: Optional[str] = None) -> str:
|
| 228 | if harness_error:
|
| 229 | return HARNESS_ERROR
|
| 230 | if any(v is False for v in invariants.values()):
|
| 231 | return VIOLATED
|
| 232 | return HELD
|
| 233 |
|
| 234 |
|
| 235 | def converge_check(trial: Trial, calib: Calibration, repo: Path) -> tuple[Optional[bool], dict]:
|
| 236 | """After a successful next op, is the state exactly the calibrated outcome?"""
|
| 237 | from .readers import worktree_manifest
|
| 238 |
|
| 239 | op = trial.fixture.op
|
| 240 | detail: dict = {}
|
| 241 | if op == "push":
|
| 242 | local = trial.adapter.head(repo)
|
| 243 | remote = trial.server_head()
|
| 244 | detail.update(local_head=local, server_head=remote)
|
| 245 | if remote is None or local is None or remote != local:
|
| 246 | return False, detail
|
| 247 | tree, status = trial.server_tree(remote)
|
| 248 | detail["server_tree_reader"] = status
|
| 249 | if tree is None:
|
| 250 | return None, detail
|
| 251 | return tree == calib.post_tree, detail
|
| 252 | if op == "switch":
|
| 253 | branch = trial.adapter.current_branch(repo)
|
| 254 | detail["branch"] = branch
|
| 255 | return branch == trial.fixture.meta["target"] and worktree_manifest(repo) == calib.post_worktree, detail
|
| 256 | tree, status = trial.adapter.tree(repo)
|
| 257 | detail["head_tree_reader"] = status
|
| 258 | if tree is None:
|
| 259 | return (False if status.startswith("corrupt") else None), detail
|
| 260 | same_tree = tree == calib.post_tree
|
| 261 | same_wt = worktree_manifest(repo) == calib.post_worktree
|
| 262 | detail.update(head_tree_matches=same_tree, worktree_matches=same_wt)
|
| 263 | if op == "clone":
|
| 264 | detail["head"] = trial.adapter.head(repo)
|
| 265 | return same_tree and same_wt, detail
|