Viewing files at commit f2acb751aa96Commit detailsView latest

suites/robustness/robustness/concurrency.py 357 lines · 15.8 KB · python
1
"""Concurrency family: N clients, shared checkouts, stale lock owners.
2
3
Scenarios (per subject):
4
- same_branch   N clones push to ONE branch (commit, push; on rejection pull then
5
                retry). The push history is checked for linearizability against
6
                the fast-forward register model and for lost acknowledged writes;
7
                every acknowledged file must be in the final server tree.
8
- distinct      N clones each own a branch; every acknowledged head must be the
9
                server's head for that branch.
10
- shared_commit N processes commit concurrently in ONE checkout (workdir lock):
11
                each exits 0 or with an actionable (lock) error, nothing hangs,
12
                the store opens, and a final serial commit captures every file.
13
- shared_push   N processes push concurrently from ONE checkout (publication lock).
14
- stale_owner   a commit is SIGKILLed at its first durability boundary while
15
                holding the workdir lock; N contenders must not hang (stale-owner
16
                reaping) and at least one commit must land.
17
"""
18
19
from __future__ import annotations
20
21
import math
22
import shutil
23
import threading
24
import time
25
from pathlib import Path
26
from typing import Optional
27
28
from . import HARNESS_ERROR
29
from .adapters import REPO_ID, GitAdapter, OakAdapter, Server, payload
30
from .engine import actionable, classify
31
from .lincheck import OK, REJECTED, UNKNOWN, Op, ancestry_from_parents, check_partitioned, lost_acknowledged
32
from .procs import free_port, run
33
from .readers import OakGraph, oak_commit_tree
34
35
PUSH_ATTEMPTS = 10
36
OP_TIMEOUT = 120.0
37
38
39
def _oak_parents(db: Path) -> dict[str, tuple]:
40
    with OakGraph(db) as g:
41
        return {h: (p, m) for h, p, m in g.conn.execute("SELECT hash, parent_hash, merge_parent_hash FROM commits")}
42
43
44
def _git_parents(adapter: GitAdapter, repo: Path) -> dict[str, tuple]:
45
    res = run([adapter.bin, "rev-list", "--all", "--parents"], repo, adapter.env(), timeout=60)
46
    parents = {}
47
    for line in res.stdout.splitlines():
48
        parts = line.split()
49
        if parts:
50
            parents[parts[0]] = tuple(parts[1:])
51
    return parents
52
53
54
class World:
55
    """One subject's remote + N worker checkouts."""
56
57
    def __init__(self, kind: str, binary: str, root: Path, n: int):
58
        self.kind, self.bin, self.root, self.n = kind, binary, root, n
59
        if root.exists():
60
            shutil.rmtree(root)
61
        root.mkdir(parents=True)
62
        self.server: Optional[Server] = None
63
        self.seed = root / "seed"
64
        self.seed.mkdir()
65
        if kind == "oak":
66
            self.adapter = OakAdapter(binary, root / "home")
67
            self.server = Server(binary, root / "srv", free_port(), root / "serve.log").start()
68
            self.adapter.must([binary, "init", "."], self.seed)
69
            payload(self.seed, "base", count=20)
70
            self.adapter.must([binary, "commit"], self.seed)
71
            self.adapter.must([binary, "push", "--repo", REPO_ID, "-r", self.server.url], self.seed)
72
            with OakGraph(self.seed / ".oak" / "oak.db") as g:
73
                self.branch = g.metadata("current_branch")
74
        else:
75
            self.adapter = GitAdapter(binary, root / "home")
76
            (root / "srv").mkdir()
77
            g = binary
78
            self.adapter.must([g, "init", "-q", "--bare", "-b", "main", "remote.git"], root / "srv")
79
            self.adapter.must([g, "init", "-q", "-b", "main", "."], self.seed)
80
            payload(self.seed, "base", count=20)
81
            self.adapter.must([g, "add", "-A"], self.seed)
82
            self.adapter.must([g, "commit", "-q", "-m", "base"], self.seed)
83
            self.adapter.must([g, "remote", "add", "origin", "../srv/remote.git"], self.seed)
84
            self.adapter.must([g, "push", "-q", "-u", "origin", "main"], self.seed)
85
            self.branch = "main"
86
        self.workers = []
87
        for i in range(n):
88
            dest = root / f"w{i:02d}"
89
            shutil.copytree(self.seed, dest, symlinks=True)  # "second machine" copy (jj bad-locking pattern)
90
            self.workers.append(dest)
91
92
    def stop(self) -> None:
93
        if self.server:
94
            self.server.stop()
95
96
    def head(self, repo: Path) -> Optional[str]:
97
        return self.adapter.head(repo)
98
99
    def server_heads(self) -> dict[str, str]:
100
        if self.kind == "oak":
101
            with OakGraph(self.server.db()) as g:
102
                return g.branch_heads()
103
        refs = self.adapter.oracle.refs(self.root / "srv" / "remote.git")
104
        return {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
105
106
    def parents(self) -> dict[str, tuple]:
107
        merged: dict[str, tuple] = {}
108
        if self.kind == "oak":
109
            for db in [self.server.db()] + [w / ".oak" / "oak.db" for w in self.workers]:
110
                if db.exists():
111
                    merged.update(_oak_parents(db))
112
        else:
113
            for repo in [self.root / "srv" / "remote.git"] + self.workers:
114
                merged.update(_git_parents(self.adapter, repo))
115
        return merged
116
117
    def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
118
        if self.kind == "oak":
119
            return oak_commit_tree(self.server.db(), commit)
120
        return self.adapter.oracle.commit_tree(self.root / "srv" / "remote.git", commit)
121
122
    def commit_cmds(self) -> list[list[str]]:
123
        if self.kind == "oak":
124
            return [[self.bin, "commit"]]
125
        return [[self.bin, "add", "-A"], [self.bin, "commit", "-q", "--no-verify", "-m", "w"]]
126
127
    def push_cmd(self, branch: Optional[str] = None) -> list[str]:
128
        if self.kind == "oak":
129
            return [self.bin, "push"]
130
        return [self.bin, "push", "-q", "origin", f"HEAD:refs/heads/{branch or self.branch}"]
131
132
    def pull_cmd(self) -> list[str]:
133
        if self.kind == "oak":
134
            return [self.bin, "pull"]
135
        return [self.bin, "pull", "-q", "--no-rebase", "--no-edit", "origin", self.branch]
136
137
138
def _outcome(res) -> str:
139
    if res.returncode == 0:
140
        return OK
141
    if res.timed_out or res.returncode is None or res.returncode < 0:
142
        return UNKNOWN
143
    text = res.text().lower()
144
    if "unknown" in text or "not received" in text or "inspect remote state" in text:
145
        return UNKNOWN
146
    return REJECTED
147
148
149
def same_branch(kind: str, binary: str, root: Path, n: int, rounds: int = 2, distinct: bool = False) -> dict:
150
    world = World(kind, binary, root, n)
151
    t0 = time.monotonic()
152
    ops: list[Op] = []
153
    acked_files: list[str] = []
154
    errors: list[str] = []
155
    unactionable: list[dict] = []
156
    lock = threading.Lock()
157
    branches: dict[int, str] = {}
158
    pulls: list = []
159
160
    def worker(i: int) -> None:
161
        repo = world.workers[i]
162
        env = world.adapter.env()
163
        branch = world.branch
164
        try:
165
            if distinct:
166
                branch = f"p4w{i:02d}"
167
                if kind == "oak":
168
                    world.adapter.must([binary, "switch", "-c", branch], repo)
169
                else:
170
                    world.adapter.must([binary, "switch", "-q", "-c", branch], repo)
171
            branches[i] = branch
172
            for r in range(rounds):
173
                name = f"conc/w{i:02d}-r{r}.txt"
174
                (repo / "conc").mkdir(exist_ok=True)
175
                (repo / name).write_text(f"worker {i} round {r}\n")
176
                for argv in world.commit_cmds():
177
                    res = run(argv, repo, env, timeout=OP_TIMEOUT)
178
                    if res.returncode != 0:
179
                        raise RuntimeError(f"commit failed: {res.stderr[-200:]}")
180
                for attempt in range(PUSH_ATTEMPTS):
181
                    value = world.head(repo)
182
                    call = time.monotonic() - t0
183
                    res = run(world.push_cmd(branch), repo, env, timeout=OP_TIMEOUT)
184
                    ret = time.monotonic() - t0
185
                    outcome = _outcome(res)
186
                    with lock:
187
                        ops.append(Op(f"w{i}", "push", value, call, ret if outcome != UNKNOWN else math.inf,
188
                                      outcome, branch, meta={"rc": res.returncode, "attempt": attempt}))
189
                        if outcome == REJECTED and not actionable(res):
190
                            unactionable.append(res.brief(200))
191
                    if outcome == OK:
192
                        with lock:
193
                            acked_files.append(name)
194
                        break
195
                    pull = run(world.pull_cmd(), repo, env, timeout=OP_TIMEOUT)
196
                    with lock:
197
                        pulls.append(pull.returncode)
198
                    if pull.returncode != 0 and kind == "git":
199
                        # a conflicting merge cannot happen (disjoint files); record and stop
200
                        raise RuntimeError(f"pull failed: {pull.stderr[-200:]}")
201
        except Exception as exc:  # noqa: BLE001
202
            with lock:
203
                errors.append(f"w{i}: {exc}")
204
205
    threads = [threading.Thread(target=worker, args=(i,)) for i in range(n)]
206
    for thread in threads:
207
        thread.start()
208
    for thread in threads:
209
        thread.join(timeout=OP_TIMEOUT * PUSH_ATTEMPTS * rounds)
210
    hung = any(thread.is_alive() for thread in threads)
211
    try:
212
        final = world.server_heads()
213
        parents = world.parents()
214
        is_anc = ancestry_from_parents(parents)
215
        results = check_partitioned(ops, is_anc, initial={b: None for b in set(o.branch for o in ops)})
216
        # append the final observation as a read at +end for each branch
217
        end = time.monotonic() - t0 + 1
218
        ops_with_reads = ops + [Op("final", "read", final.get(b), end, end, OK, b) for b in set(o.branch for o in ops)]
219
        results = check_partitioned(ops_with_reads, is_anc)
220
        lost = lost_acknowledged(ops, final, is_anc)
221
        missing_files: list[str] = []
222
        tree_status = "skipped"
223
        if not distinct and final.get(world.branch):
224
            tree, tree_status = world.server_tree(final[world.branch])
225
            if tree is not None:
226
                missing_files = [f for f in acked_files if f not in tree]
227
        integrity = "ok"
228
        if kind == "oak":
229
            with OakGraph(world.server.db()) as g:
230
                integrity = g.integrity()
231
        else:
232
            integrity = world.adapter.oracle.fsck(world.root / "srv" / "remote.git")
233
        lin = [r.linearizable for r in results.values()]
234
        inv = {
235
            "no_hang": not hung,
236
            "no_lost_acknowledged_write": not lost,
237
            "linearizable": (None if any(v is None for v in lin) else all(lin)) if lin else None,
238
            "acked_content_on_server": (not missing_files) if tree_status == "ok" else None,
239
            "server_opens": integrity == "ok",
240
            "rejections_actionable": not unactionable,
241
        }
242
        obs = {
243
            "pushes": len(ops),
244
            "acknowledged": sum(1 for o in ops if o.outcome == OK),
245
            "rejected": sum(1 for o in ops if o.outcome == REJECTED),
246
            "unknown": sum(1 for o in ops if o.outcome == UNKNOWN),
247
            "pull_returncodes": {str(rc): pulls.count(rc) for rc in set(pulls)},
248
            "lost": [o.value for o in lost][:5],
249
            "missing_files": missing_files[:5],
250
            "lincheck": {b: {"linearizable": r.linearizable, "explored": r.explored, "reason": r.reason}
251
                         for b, r in results.items()},
252
            "history": [[o.client, o.outcome, (o.value or "")[:12], round(o.call, 3),
253
                         None if o.ret == math.inf else round(o.ret, 3), o.meta.get("rc")]
254
                        for o in sorted(ops, key=lambda o: o.call)][:400],
255
            "final_heads": {b: (h or "")[:12] for b, h in final.items()},
256
            "worker_errors": errors[:5],
257
            "unactionable_rejections": unactionable[:3],
258
            "tree_reader": tree_status,
259
        }
260
        classification = classify(inv, harness_error="; ".join(errors[:3]) if errors else None)
261
    finally:
262
        world.stop()
263
    return {"invariants": inv, "observations": obs, "classification": classification,
264
            "elapsed_ms": round((time.monotonic() - t0) * 1000.0, 3),
265
            **({"harness_error": "; ".join(errors[:3])} if errors else {})}
266
267
268
def shared_checkout(kind: str, binary: str, root: Path, n: int, what: str) -> dict:
269
    """what: commit | push | stale_owner"""
270
    world = World(kind, binary, root, 1)
271
    repo = world.workers[0]
272
    env = world.adapter.env()
273
    t0 = time.monotonic()
274
    results = []
275
    lock = threading.Lock()
276
    obs: dict = {}
277
    try:
278
        stale = None
279
        if what == "stale_owner":
280
            if kind != "oak":
281
                raise NotImplementedError("stale_owner is Oak-only (git leaves index.lock by design)")
282
            from . import crashpoints
283
284
            inst, why = crashpoints.build(root / "inst", binary, None)
285
            if inst is None:
286
                raise NotImplementedError(why)
287
            (repo / "stale.txt").write_text("stale owner\n")
288
            stale = run([str(inst.oak_bin), "commit"], repo, world.adapter.env(inst.env(1)), timeout=30)
289
            obs["stale_owner_rc"] = stale.returncode
290
        if what == "push":
291
            (repo / "pub.txt").write_text("publication\n")
292
            for argv in world.commit_cmds():
293
                world.adapter.must(argv, repo)
294
        for i in range(n):
295
            (repo / f"shared-{i:02d}.txt").write_text(f"shared {i}\n")
296
297
        def contender(i: int) -> None:
298
            if what == "push":
299
                cmds = [world.push_cmd()]
300
            else:
301
                cmds = world.commit_cmds()
302
            for argv in cmds:
303
                res = run(argv, repo, env, timeout=OP_TIMEOUT)
304
                with lock:
305
                    results.append(res)
306
                if res.returncode != 0:
307
                    break
308
309
        threads = [threading.Thread(target=contender, args=(i,)) for i in range(n)]
310
        for thread in threads:
311
            thread.start()
312
        for thread in threads:
313
            thread.join(timeout=OP_TIMEOUT * 3)
314
        hung = any(thread.is_alive() for thread in threads) or any(r.timed_out for r in results)
315
        failures = [r for r in results if r.returncode != 0]
316
        final_ok = True
317
        missing: list[str] = []
318
        if what in ("commit", "stale_owner"):
319
            for argv in world.commit_cmds():
320
                res = run(argv, repo, env, timeout=OP_TIMEOUT)
321
            tree, status = world.adapter.tree(repo)
322
            obs["tree_reader"] = status
323
            if tree is not None:
324
                missing = [f"shared-{i:02d}.txt" for i in range(n) if f"shared-{i:02d}.txt" not in tree]
325
            final_ok = None if tree is None else not missing
326
        else:
327
            final_ok = world.server_heads().get(world.branch) == world.head(repo)
328
        if kind == "oak":
329
            with OakGraph(repo / ".oak" / "oak.db") as g:
330
                opens = g.integrity() == "ok"
331
        else:
332
            opens = world.adapter.oracle.fsck(repo) == "ok"
333
        successes = sum(1 for r in results if r.returncode == 0)
334
        inv = {
335
            "no_hang": not hung,
336
            "failures_actionable": all(actionable(r) for r in failures),
337
            "repo_opens": opens,
338
            "final_state_complete": final_ok,
339
        }
340
        if what == "stale_owner":
341
            inv["stale_owner_reaped"] = successes > 0
342
        obs.update(contenders=n, successes=successes, failures=len(failures), missing=missing[:5],
343
                   failure_samples=[r.brief(200) for r in failures if not actionable(r)][:2],
344
                   max_contender_ms=round(max((r.elapsed_ms for r in results), default=0.0), 1))
345
        classification = classify(inv)
346
        error = None
347
    except NotImplementedError as exc:
348
        return {"unsupported": str(exc)}
349
    except Exception as exc:  # noqa: BLE001
350
        inv, classification, error = {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
351
    finally:
352
        world.stop()
353
    row = {"invariants": inv, "observations": obs, "classification": classification,
354
           "elapsed_ms": round((time.monotonic() - t0) * 1000.0, 3)}
355
    if error:
356
        row["harness_error"] = error
357
    return row