| 1 | """Concurrency family: N clients, shared checkouts, stale lock owners.
|
| 2 |
|
| 3 | Scenarios (per subject):
|
| 4 | - same_branch N clones push to ONE branch (commit, push; on rejection pull then
|
| 5 | retry). The push history is checked for linearizability against
|
| 6 | the fast-forward register model and for lost acknowledged writes;
|
| 7 | every acknowledged file must be in the final server tree.
|
| 8 | - distinct N clones each own a branch; every acknowledged head must be the
|
| 9 | server's head for that branch.
|
| 10 | - shared_commit N processes commit concurrently in ONE checkout (workdir lock):
|
| 11 | each exits 0 or with an actionable (lock) error, nothing hangs,
|
| 12 | the store opens, and a final serial commit captures every file.
|
| 13 | - shared_push N processes push concurrently from ONE checkout (publication lock).
|
| 14 | - stale_owner a commit is SIGKILLed at its first durability boundary while
|
| 15 | holding the workdir lock; N contenders must not hang (stale-owner
|
| 16 | reaping) and at least one commit must land.
|
| 17 | """
|
| 18 |
|
| 19 | from __future__ import annotations
|
| 20 |
|
| 21 | import math
|
| 22 | import shutil
|
| 23 | import threading
|
| 24 | import time
|
| 25 | from pathlib import Path
|
| 26 | from typing import Optional
|
| 27 |
|
| 28 | from . import HARNESS_ERROR
|
| 29 | from .adapters import REPO_ID, GitAdapter, OakAdapter, Server, payload
|
| 30 | from .engine import actionable, classify
|
| 31 | from .lincheck import OK, REJECTED, UNKNOWN, Op, ancestry_from_parents, check_partitioned, lost_acknowledged
|
| 32 | from .procs import free_port, run
|
| 33 | from .readers import OakGraph, oak_commit_tree
|
| 34 |
|
| 35 | PUSH_ATTEMPTS = 10
|
| 36 | OP_TIMEOUT = 120.0
|
| 37 |
|
| 38 |
|
| 39 | def _oak_parents(db: Path) -> dict[str, tuple]:
|
| 40 | with OakGraph(db) as g:
|
| 41 | return {h: (p, m) for h, p, m in g.conn.execute("SELECT hash, parent_hash, merge_parent_hash FROM commits")}
|
| 42 |
|
| 43 |
|
| 44 | def _git_parents(adapter: GitAdapter, repo: Path) -> dict[str, tuple]:
|
| 45 | res = run([adapter.bin, "rev-list", "--all", "--parents"], repo, adapter.env(), timeout=60)
|
| 46 | parents = {}
|
| 47 | for line in res.stdout.splitlines():
|
| 48 | parts = line.split()
|
| 49 | if parts:
|
| 50 | parents[parts[0]] = tuple(parts[1:])
|
| 51 | return parents
|
| 52 |
|
| 53 |
|
| 54 | class World:
|
| 55 | """One subject's remote + N worker checkouts."""
|
| 56 |
|
| 57 | def __init__(self, kind: str, binary: str, root: Path, n: int):
|
| 58 | self.kind, self.bin, self.root, self.n = kind, binary, root, n
|
| 59 | if root.exists():
|
| 60 | shutil.rmtree(root)
|
| 61 | root.mkdir(parents=True)
|
| 62 | self.server: Optional[Server] = None
|
| 63 | self.seed = root / "seed"
|
| 64 | self.seed.mkdir()
|
| 65 | if kind == "oak":
|
| 66 | self.adapter = OakAdapter(binary, root / "home")
|
| 67 | self.server = Server(binary, root / "srv", free_port(), root / "serve.log").start()
|
| 68 | self.adapter.must([binary, "init", "."], self.seed)
|
| 69 | payload(self.seed, "base", count=20)
|
| 70 | self.adapter.must([binary, "commit"], self.seed)
|
| 71 | self.adapter.must([binary, "push", "--repo", REPO_ID, "-r", self.server.url], self.seed)
|
| 72 | with OakGraph(self.seed / ".oak" / "oak.db") as g:
|
| 73 | self.branch = g.metadata("current_branch")
|
| 74 | else:
|
| 75 | self.adapter = GitAdapter(binary, root / "home")
|
| 76 | (root / "srv").mkdir()
|
| 77 | g = binary
|
| 78 | self.adapter.must([g, "init", "-q", "--bare", "-b", "main", "remote.git"], root / "srv")
|
| 79 | self.adapter.must([g, "init", "-q", "-b", "main", "."], self.seed)
|
| 80 | payload(self.seed, "base", count=20)
|
| 81 | self.adapter.must([g, "add", "-A"], self.seed)
|
| 82 | self.adapter.must([g, "commit", "-q", "-m", "base"], self.seed)
|
| 83 | self.adapter.must([g, "remote", "add", "origin", "../srv/remote.git"], self.seed)
|
| 84 | self.adapter.must([g, "push", "-q", "-u", "origin", "main"], self.seed)
|
| 85 | self.branch = "main"
|
| 86 | self.workers = []
|
| 87 | for i in range(n):
|
| 88 | dest = root / f"w{i:02d}"
|
| 89 | shutil.copytree(self.seed, dest, symlinks=True) # "second machine" copy (jj bad-locking pattern)
|
| 90 | self.workers.append(dest)
|
| 91 |
|
| 92 | def stop(self) -> None:
|
| 93 | if self.server:
|
| 94 | self.server.stop()
|
| 95 |
|
| 96 | def head(self, repo: Path) -> Optional[str]:
|
| 97 | return self.adapter.head(repo)
|
| 98 |
|
| 99 | def server_heads(self) -> dict[str, str]:
|
| 100 | if self.kind == "oak":
|
| 101 | with OakGraph(self.server.db()) as g:
|
| 102 | return g.branch_heads()
|
| 103 | refs = self.adapter.oracle.refs(self.root / "srv" / "remote.git")
|
| 104 | return {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
|
| 105 |
|
| 106 | def parents(self) -> dict[str, tuple]:
|
| 107 | merged: dict[str, tuple] = {}
|
| 108 | if self.kind == "oak":
|
| 109 | for db in [self.server.db()] + [w / ".oak" / "oak.db" for w in self.workers]:
|
| 110 | if db.exists():
|
| 111 | merged.update(_oak_parents(db))
|
| 112 | else:
|
| 113 | for repo in [self.root / "srv" / "remote.git"] + self.workers:
|
| 114 | merged.update(_git_parents(self.adapter, repo))
|
| 115 | return merged
|
| 116 |
|
| 117 | def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
|
| 118 | if self.kind == "oak":
|
| 119 | return oak_commit_tree(self.server.db(), commit)
|
| 120 | return self.adapter.oracle.commit_tree(self.root / "srv" / "remote.git", commit)
|
| 121 |
|
| 122 | def commit_cmds(self) -> list[list[str]]:
|
| 123 | if self.kind == "oak":
|
| 124 | return [[self.bin, "commit"]]
|
| 125 | return [[self.bin, "add", "-A"], [self.bin, "commit", "-q", "--no-verify", "-m", "w"]]
|
| 126 |
|
| 127 | def push_cmd(self, branch: Optional[str] = None) -> list[str]:
|
| 128 | if self.kind == "oak":
|
| 129 | return [self.bin, "push"]
|
| 130 | return [self.bin, "push", "-q", "origin", f"HEAD:refs/heads/{branch or self.branch}"]
|
| 131 |
|
| 132 | def pull_cmd(self) -> list[str]:
|
| 133 | if self.kind == "oak":
|
| 134 | return [self.bin, "pull"]
|
| 135 | return [self.bin, "pull", "-q", "--no-rebase", "--no-edit", "origin", self.branch]
|
| 136 |
|
| 137 |
|
| 138 | def _outcome(res) -> str:
|
| 139 | if res.returncode == 0:
|
| 140 | return OK
|
| 141 | if res.timed_out or res.returncode is None or res.returncode < 0:
|
| 142 | return UNKNOWN
|
| 143 | text = res.text().lower()
|
| 144 | if "unknown" in text or "not received" in text or "inspect remote state" in text:
|
| 145 | return UNKNOWN
|
| 146 | return REJECTED
|
| 147 |
|
| 148 |
|
| 149 | def same_branch(kind: str, binary: str, root: Path, n: int, rounds: int = 2, distinct: bool = False) -> dict:
|
| 150 | world = World(kind, binary, root, n)
|
| 151 | t0 = time.monotonic()
|
| 152 | ops: list[Op] = []
|
| 153 | acked_files: list[str] = []
|
| 154 | errors: list[str] = []
|
| 155 | unactionable: list[dict] = []
|
| 156 | lock = threading.Lock()
|
| 157 | branches: dict[int, str] = {}
|
| 158 | pulls: list = []
|
| 159 |
|
| 160 | def worker(i: int) -> None:
|
| 161 | repo = world.workers[i]
|
| 162 | env = world.adapter.env()
|
| 163 | branch = world.branch
|
| 164 | try:
|
| 165 | if distinct:
|
| 166 | branch = f"p4w{i:02d}"
|
| 167 | if kind == "oak":
|
| 168 | world.adapter.must([binary, "switch", "-c", branch], repo)
|
| 169 | else:
|
| 170 | world.adapter.must([binary, "switch", "-q", "-c", branch], repo)
|
| 171 | branches[i] = branch
|
| 172 | for r in range(rounds):
|
| 173 | name = f"conc/w{i:02d}-r{r}.txt"
|
| 174 | (repo / "conc").mkdir(exist_ok=True)
|
| 175 | (repo / name).write_text(f"worker {i} round {r}\n")
|
| 176 | for argv in world.commit_cmds():
|
| 177 | res = run(argv, repo, env, timeout=OP_TIMEOUT)
|
| 178 | if res.returncode != 0:
|
| 179 | raise RuntimeError(f"commit failed: {res.stderr[-200:]}")
|
| 180 | for attempt in range(PUSH_ATTEMPTS):
|
| 181 | value = world.head(repo)
|
| 182 | call = time.monotonic() - t0
|
| 183 | res = run(world.push_cmd(branch), repo, env, timeout=OP_TIMEOUT)
|
| 184 | ret = time.monotonic() - t0
|
| 185 | outcome = _outcome(res)
|
| 186 | with lock:
|
| 187 | ops.append(Op(f"w{i}", "push", value, call, ret if outcome != UNKNOWN else math.inf,
|
| 188 | outcome, branch, meta={"rc": res.returncode, "attempt": attempt}))
|
| 189 | if outcome == REJECTED and not actionable(res):
|
| 190 | unactionable.append(res.brief(200))
|
| 191 | if outcome == OK:
|
| 192 | with lock:
|
| 193 | acked_files.append(name)
|
| 194 | break
|
| 195 | pull = run(world.pull_cmd(), repo, env, timeout=OP_TIMEOUT)
|
| 196 | with lock:
|
| 197 | pulls.append(pull.returncode)
|
| 198 | if pull.returncode != 0 and kind == "git":
|
| 199 | # a conflicting merge cannot happen (disjoint files); record and stop
|
| 200 | raise RuntimeError(f"pull failed: {pull.stderr[-200:]}")
|
| 201 | except Exception as exc: # noqa: BLE001
|
| 202 | with lock:
|
| 203 | errors.append(f"w{i}: {exc}")
|
| 204 |
|
| 205 | threads = [threading.Thread(target=worker, args=(i,)) for i in range(n)]
|
| 206 | for thread in threads:
|
| 207 | thread.start()
|
| 208 | for thread in threads:
|
| 209 | thread.join(timeout=OP_TIMEOUT * PUSH_ATTEMPTS * rounds)
|
| 210 | hung = any(thread.is_alive() for thread in threads)
|
| 211 | try:
|
| 212 | final = world.server_heads()
|
| 213 | parents = world.parents()
|
| 214 | is_anc = ancestry_from_parents(parents)
|
| 215 | results = check_partitioned(ops, is_anc, initial={b: None for b in set(o.branch for o in ops)})
|
| 216 | # append the final observation as a read at +end for each branch
|
| 217 | end = time.monotonic() - t0 + 1
|
| 218 | ops_with_reads = ops + [Op("final", "read", final.get(b), end, end, OK, b) for b in set(o.branch for o in ops)]
|
| 219 | results = check_partitioned(ops_with_reads, is_anc)
|
| 220 | lost = lost_acknowledged(ops, final, is_anc)
|
| 221 | missing_files: list[str] = []
|
| 222 | tree_status = "skipped"
|
| 223 | if not distinct and final.get(world.branch):
|
| 224 | tree, tree_status = world.server_tree(final[world.branch])
|
| 225 | if tree is not None:
|
| 226 | missing_files = [f for f in acked_files if f not in tree]
|
| 227 | integrity = "ok"
|
| 228 | if kind == "oak":
|
| 229 | with OakGraph(world.server.db()) as g:
|
| 230 | integrity = g.integrity()
|
| 231 | else:
|
| 232 | integrity = world.adapter.oracle.fsck(world.root / "srv" / "remote.git")
|
| 233 | lin = [r.linearizable for r in results.values()]
|
| 234 | inv = {
|
| 235 | "no_hang": not hung,
|
| 236 | "no_lost_acknowledged_write": not lost,
|
| 237 | "linearizable": (None if any(v is None for v in lin) else all(lin)) if lin else None,
|
| 238 | "acked_content_on_server": (not missing_files) if tree_status == "ok" else None,
|
| 239 | "server_opens": integrity == "ok",
|
| 240 | "rejections_actionable": not unactionable,
|
| 241 | }
|
| 242 | obs = {
|
| 243 | "pushes": len(ops),
|
| 244 | "acknowledged": sum(1 for o in ops if o.outcome == OK),
|
| 245 | "rejected": sum(1 for o in ops if o.outcome == REJECTED),
|
| 246 | "unknown": sum(1 for o in ops if o.outcome == UNKNOWN),
|
| 247 | "pull_returncodes": {str(rc): pulls.count(rc) for rc in set(pulls)},
|
| 248 | "lost": [o.value for o in lost][:5],
|
| 249 | "missing_files": missing_files[:5],
|
| 250 | "lincheck": {b: {"linearizable": r.linearizable, "explored": r.explored, "reason": r.reason}
|
| 251 | for b, r in results.items()},
|
| 252 | "history": [[o.client, o.outcome, (o.value or "")[:12], round(o.call, 3),
|
| 253 | None if o.ret == math.inf else round(o.ret, 3), o.meta.get("rc")]
|
| 254 | for o in sorted(ops, key=lambda o: o.call)][:400],
|
| 255 | "final_heads": {b: (h or "")[:12] for b, h in final.items()},
|
| 256 | "worker_errors": errors[:5],
|
| 257 | "unactionable_rejections": unactionable[:3],
|
| 258 | "tree_reader": tree_status,
|
| 259 | }
|
| 260 | classification = classify(inv, harness_error="; ".join(errors[:3]) if errors else None)
|
| 261 | finally:
|
| 262 | world.stop()
|
| 263 | return {"invariants": inv, "observations": obs, "classification": classification,
|
| 264 | "elapsed_ms": round((time.monotonic() - t0) * 1000.0, 3),
|
| 265 | **({"harness_error": "; ".join(errors[:3])} if errors else {})}
|
| 266 |
|
| 267 |
|
| 268 | def shared_checkout(kind: str, binary: str, root: Path, n: int, what: str) -> dict:
|
| 269 | """what: commit | push | stale_owner"""
|
| 270 | world = World(kind, binary, root, 1)
|
| 271 | repo = world.workers[0]
|
| 272 | env = world.adapter.env()
|
| 273 | t0 = time.monotonic()
|
| 274 | results = []
|
| 275 | lock = threading.Lock()
|
| 276 | obs: dict = {}
|
| 277 | try:
|
| 278 | stale = None
|
| 279 | if what == "stale_owner":
|
| 280 | if kind != "oak":
|
| 281 | raise NotImplementedError("stale_owner is Oak-only (git leaves index.lock by design)")
|
| 282 | from . import crashpoints
|
| 283 |
|
| 284 | inst, why = crashpoints.build(root / "inst", binary, None)
|
| 285 | if inst is None:
|
| 286 | raise NotImplementedError(why)
|
| 287 | (repo / "stale.txt").write_text("stale owner\n")
|
| 288 | stale = run([str(inst.oak_bin), "commit"], repo, world.adapter.env(inst.env(1)), timeout=30)
|
| 289 | obs["stale_owner_rc"] = stale.returncode
|
| 290 | if what == "push":
|
| 291 | (repo / "pub.txt").write_text("publication\n")
|
| 292 | for argv in world.commit_cmds():
|
| 293 | world.adapter.must(argv, repo)
|
| 294 | for i in range(n):
|
| 295 | (repo / f"shared-{i:02d}.txt").write_text(f"shared {i}\n")
|
| 296 |
|
| 297 | def contender(i: int) -> None:
|
| 298 | if what == "push":
|
| 299 | cmds = [world.push_cmd()]
|
| 300 | else:
|
| 301 | cmds = world.commit_cmds()
|
| 302 | for argv in cmds:
|
| 303 | res = run(argv, repo, env, timeout=OP_TIMEOUT)
|
| 304 | with lock:
|
| 305 | results.append(res)
|
| 306 | if res.returncode != 0:
|
| 307 | break
|
| 308 |
|
| 309 | threads = [threading.Thread(target=contender, args=(i,)) for i in range(n)]
|
| 310 | for thread in threads:
|
| 311 | thread.start()
|
| 312 | for thread in threads:
|
| 313 | thread.join(timeout=OP_TIMEOUT * 3)
|
| 314 | hung = any(thread.is_alive() for thread in threads) or any(r.timed_out for r in results)
|
| 315 | failures = [r for r in results if r.returncode != 0]
|
| 316 | final_ok = True
|
| 317 | missing: list[str] = []
|
| 318 | if what in ("commit", "stale_owner"):
|
| 319 | for argv in world.commit_cmds():
|
| 320 | res = run(argv, repo, env, timeout=OP_TIMEOUT)
|
| 321 | tree, status = world.adapter.tree(repo)
|
| 322 | obs["tree_reader"] = status
|
| 323 | if tree is not None:
|
| 324 | missing = [f"shared-{i:02d}.txt" for i in range(n) if f"shared-{i:02d}.txt" not in tree]
|
| 325 | final_ok = None if tree is None else not missing
|
| 326 | else:
|
| 327 | final_ok = world.server_heads().get(world.branch) == world.head(repo)
|
| 328 | if kind == "oak":
|
| 329 | with OakGraph(repo / ".oak" / "oak.db") as g:
|
| 330 | opens = g.integrity() == "ok"
|
| 331 | else:
|
| 332 | opens = world.adapter.oracle.fsck(repo) == "ok"
|
| 333 | successes = sum(1 for r in results if r.returncode == 0)
|
| 334 | inv = {
|
| 335 | "no_hang": not hung,
|
| 336 | "failures_actionable": all(actionable(r) for r in failures),
|
| 337 | "repo_opens": opens,
|
| 338 | "final_state_complete": final_ok,
|
| 339 | }
|
| 340 | if what == "stale_owner":
|
| 341 | inv["stale_owner_reaped"] = successes > 0
|
| 342 | obs.update(contenders=n, successes=successes, failures=len(failures), missing=missing[:5],
|
| 343 | failure_samples=[r.brief(200) for r in failures if not actionable(r)][:2],
|
| 344 | max_contender_ms=round(max((r.elapsed_ms for r in results), default=0.0), 1))
|
| 345 | classification = classify(inv)
|
| 346 | error = None
|
| 347 | except NotImplementedError as exc:
|
| 348 | return {"unsupported": str(exc)}
|
| 349 | except Exception as exc: # noqa: BLE001
|
| 350 | inv, classification, error = {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
|
| 351 | finally:
|
| 352 | world.stop()
|
| 353 | row = {"invariants": inv, "observations": obs, "classification": classification,
|
| 354 | "elapsed_ms": round((time.monotonic() - t0) * 1000.0, 3)}
|
| 355 | if error:
|
| 356 | row["harness_error"] = error
|
| 357 | return row
|