Viewing files at commit 5717cc9fc7edCommit detailsView latest

suites/robustness/robustness/concurrency.py 384 lines · 17.3 KB · python
1
"""Concurrency family: N clients, shared checkouts, stale lock owners.
2
3
Scenarios (per subject):
4
- same_branch   N clones push to ONE branch (commit, push; on rejection pull then
5
                retry). The push history is checked for linearizability against
6
                the fast-forward register model and for lost acknowledged writes;
7
                every acknowledged file must be in the final server tree.
8
- distinct      N clones each own a branch; every acknowledged head must be the
9
                server's head for that branch.
10
- shared_commit N processes commit concurrently in ONE checkout (workdir lock):
11
                each exits 0 or with an actionable (lock) error, nothing hangs,
12
                the store opens, and a final serial commit captures every file.
13
- shared_push   N processes push concurrently from ONE checkout (publication lock).
14
- stale_owner   a commit is SIGKILLed at its first durability boundary while
15
                holding the workdir lock; N contenders must not hang (stale-owner
16
                reaping) and at least one commit must land.
17
"""
18
19
from __future__ import annotations
20
21
import json
22
import math
23
import shutil
24
import threading
25
import time
26
from pathlib import Path
27
from typing import Optional
28
29
from . import FAULT_NOT_FIRED, HARNESS_ERROR
30
from .adapters import REPO_ID, GitAdapter, OakAdapter, Server, payload
31
from .engine import actionable, classify
32
from .lincheck import OK, REJECTED, UNKNOWN, Op, ancestry_from_parents, check_partitioned, lost_acknowledged
33
from .procs import free_port, run
34
from .readers import OakGraph, oak_commit_tree
35
36
PUSH_ATTEMPTS = 10
37
OP_TIMEOUT = 120.0
38
39
40
def _oak_parents(db: Path) -> dict[str, tuple]:
41
    with OakGraph(db) as g:
42
        return {h: (p, m) for h, p, m in g.conn.execute("SELECT hash, parent_hash, merge_parent_hash FROM commits")}
43
44
45
def _git_parents(adapter: GitAdapter, repo: Path) -> dict[str, tuple]:
46
    res = run([adapter.bin, "rev-list", "--all", "--parents"], repo, adapter.env(), timeout=60)
47
    parents = {}
48
    for line in res.stdout.splitlines():
49
        parts = line.split()
50
        if parts:
51
            parents[parts[0]] = tuple(parts[1:])
52
    return parents
53
54
55
class World:
56
    """One subject's remote + N worker checkouts."""
57
58
    def __init__(self, kind: str, binary: str, root: Path, n: int):
59
        self.kind, self.bin, self.root, self.n = kind, binary, root, n
60
        if root.exists():
61
            shutil.rmtree(root)
62
        root.mkdir(parents=True)
63
        self.server: Optional[Server] = None
64
        self.seed = root / "seed"
65
        self.seed.mkdir()
66
        if kind == "oak":
67
            self.adapter = OakAdapter(binary, root / "home")
68
            self.server = Server(binary, root / "srv", free_port(), root / "serve.log").start()
69
            self.adapter.must([binary, "init", "."], self.seed)
70
            payload(self.seed, "base", count=20)
71
            self.adapter.must([binary, "commit"], self.seed)
72
            self.adapter.must([binary, "push", "--repo", REPO_ID, "-r", self.server.url], self.seed)
73
            with OakGraph(self.seed / ".oak" / "oak.db") as g:
74
                self.branch = g.metadata("current_branch")
75
        else:
76
            self.adapter = GitAdapter(binary, root / "home")
77
            (root / "srv").mkdir()
78
            g = binary
79
            self.adapter.must([g, "init", "-q", "--bare", "-b", "main", "remote.git"], root / "srv")
80
            self.adapter.must([g, "init", "-q", "-b", "main", "."], self.seed)
81
            payload(self.seed, "base", count=20)
82
            self.adapter.must([g, "add", "-A"], self.seed)
83
            self.adapter.must([g, "commit", "-q", "-m", "base"], self.seed)
84
            self.adapter.must([g, "remote", "add", "origin", "../srv/remote.git"], self.seed)
85
            self.adapter.must([g, "push", "-q", "-u", "origin", "main"], self.seed)
86
            self.branch = "main"
87
        self.workers = []
88
        for i in range(n):
89
            dest = root / f"w{i:02d}"
90
            shutil.copytree(self.seed, dest, symlinks=True)  # "second machine" copy (jj bad-locking pattern)
91
            self.workers.append(dest)
92
93
    def stop(self) -> None:
94
        if self.server:
95
            self.server.stop()
96
97
    def head(self, repo: Path) -> Optional[str]:
98
        return self.adapter.head(repo)
99
100
    def server_heads(self) -> dict[str, str]:
101
        if self.kind == "oak":
102
            with OakGraph(self.server.db()) as g:
103
                return g.branch_heads()
104
        refs = self.adapter.oracle.refs(self.root / "srv" / "remote.git")
105
        return {k.removeprefix("refs/heads/"): v for k, v in refs.items() if k.startswith("refs/heads/")}
106
107
    def parents(self) -> dict[str, tuple]:
108
        merged: dict[str, tuple] = {}
109
        if self.kind == "oak":
110
            for db in [self.server.db()] + [w / ".oak" / "oak.db" for w in self.workers]:
111
                if db.exists():
112
                    merged.update(_oak_parents(db))
113
        else:
114
            for repo in [self.root / "srv" / "remote.git"] + self.workers:
115
                merged.update(_git_parents(self.adapter, repo))
116
        return merged
117
118
    def server_tree(self, commit: str) -> tuple[Optional[dict], str]:
119
        if self.kind == "oak":
120
            return oak_commit_tree(self.server.db(), commit)
121
        return self.adapter.oracle.commit_tree(self.root / "srv" / "remote.git", commit)
122
123
    def commit_cmds(self) -> list[list[str]]:
124
        if self.kind == "oak":
125
            return [[self.bin, "commit"]]
126
        return [[self.bin, "add", "-A"], [self.bin, "commit", "-q", "--no-verify", "-m", "w"]]
127
128
    def push_cmd(self, branch: Optional[str] = None) -> list[str]:
129
        if self.kind == "oak":
130
            return [self.bin, "push", "--json"]
131
        return [self.bin, "push", "-q", "origin", f"HEAD:refs/heads/{branch or self.branch}"]
132
133
    def pull_cmd(self) -> list[str]:
134
        if self.kind == "oak":
135
            return [self.bin, "pull"]
136
        return [self.bin, "pull", "-q", "--no-rebase", "--no-edit", "origin", self.branch]
137
138
139
def reported_pushed_head(res) -> Optional[str]:
140
    """The head the subject says it published (Oak ``push --json`` pushed_head); None if not reported."""
141
    try:
142
        data = json.loads(res.stdout)
143
    except (ValueError, TypeError):
144
        return None
145
    head = data.get("pushed_head") if isinstance(data, dict) else None
146
    return head if isinstance(head, str) and head else None
147
148
149
def _outcome(res) -> str:
150
    if res.returncode == 0:
151
        return OK
152
    if res.timed_out or res.returncode is None or res.returncode < 0:
153
        return UNKNOWN
154
    text = res.text().lower()
155
    if "unknown" in text or "not received" in text or "inspect remote state" in text:
156
        return UNKNOWN
157
    return REJECTED
158
159
160
def same_branch(kind: str, binary: str, root: Path, n: int, rounds: int = 2, distinct: bool = False) -> dict:
161
    world = World(kind, binary, root, n)
162
    t0 = time.monotonic()
163
    ops: list[Op] = []
164
    acked_files: list[str] = []
165
    errors: list[str] = []
166
    unactionable: list[dict] = []
167
    lock = threading.Lock()
168
    branches: dict[int, str] = {}
169
    pulls: list = []
170
    head_mismatches: list[dict] = []
171
172
    def worker(i: int) -> None:
173
        repo = world.workers[i]
174
        env = world.adapter.env()
175
        branch = world.branch
176
        try:
177
            if distinct:
178
                branch = f"p4w{i:02d}"
179
                if kind == "oak":
180
                    world.adapter.must([binary, "switch", "-c", branch], repo)
181
                else:
182
                    world.adapter.must([binary, "switch", "-q", "-c", branch], repo)
183
            branches[i] = branch
184
            for r in range(rounds):
185
                name = f"conc/w{i:02d}-r{r}.txt"
186
                (repo / "conc").mkdir(exist_ok=True)
187
                (repo / name).write_text(f"worker {i} round {r}\n")
188
                for argv in world.commit_cmds():
189
                    res = run(argv, repo, env, timeout=OP_TIMEOUT)
190
                    if res.returncode != 0:
191
                        raise RuntimeError(f"commit failed: {res.stderr[-200:]}")
192
                for attempt in range(PUSH_ATTEMPTS):
193
                    value = world.head(repo)
194
                    call = time.monotonic() - t0
195
                    res = run(world.push_cmd(branch), repo, env, timeout=OP_TIMEOUT)
196
                    ret = time.monotonic() - t0
197
                    outcome = _outcome(res)
198
                    reported = reported_pushed_head(res) if outcome == OK else None
199
                    with lock:
200
                        if reported is not None and reported != value:
201
                            head_mismatches.append({"worker": i, "local_head": value, "pushed_head": reported})
202
                        ops.append(Op(f"w{i}", "push", value, call, ret if outcome != UNKNOWN else math.inf,
203
                                      outcome, branch, meta={"rc": res.returncode, "attempt": attempt,
204
                                                             "pushed_head_reported": reported is not None}))
205
                        if outcome == REJECTED and not actionable(res):
206
                            unactionable.append(res.brief(200))
207
                    if outcome == OK:
208
                        with lock:
209
                            acked_files.append(name)
210
                        break
211
                    pull = run(world.pull_cmd(), repo, env, timeout=OP_TIMEOUT)
212
                    with lock:
213
                        pulls.append(pull.returncode)
214
                    if pull.returncode != 0 and kind == "git":
215
                        # a conflicting merge cannot happen (disjoint files); record and stop
216
                        raise RuntimeError(f"pull failed: {pull.stderr[-200:]}")
217
        except Exception as exc:  # noqa: BLE001
218
            with lock:
219
                errors.append(f"w{i}: {exc}")
220
221
    threads = [threading.Thread(target=worker, args=(i,)) for i in range(n)]
222
    for thread in threads:
223
        thread.start()
224
    for thread in threads:
225
        thread.join(timeout=OP_TIMEOUT * PUSH_ATTEMPTS * rounds)
226
    hung = any(thread.is_alive() for thread in threads)
227
    try:
228
        final = world.server_heads()
229
        parents = world.parents()
230
        is_anc = ancestry_from_parents(parents)
231
        # the final server observation is a read at +end for each branch
232
        end = time.monotonic() - t0 + 1
233
        ops_with_reads = ops + [Op("final", "read", final.get(b), end, end, OK, b) for b in set(o.branch for o in ops)]
234
        results = check_partitioned(ops_with_reads, is_anc)
235
        lost = lost_acknowledged(ops, final, is_anc)
236
        missing_files: list[str] = []
237
        tree_status = "skipped"
238
        if not distinct and final.get(world.branch):
239
            tree, tree_status = world.server_tree(final[world.branch])
240
            if tree is not None:
241
                missing_files = [f for f in acked_files if f not in tree]
242
        integrity = "ok"
243
        if kind == "oak":
244
            with OakGraph(world.server.db()) as g:
245
                integrity = g.integrity()
246
        else:
247
            integrity = world.adapter.oracle.fsck(world.root / "srv" / "remote.git")
248
        lin = [r.linearizable for r in results.values()]
249
        inv = {
250
            "no_hang": not hung,
251
            "no_lost_acknowledged_write": not lost,
252
            "linearizable": (None if any(v is None for v in lin) else all(lin)) if lin else None,
253
            "acked_content_on_server": (not missing_files) if tree_status == "ok" else None,
254
            "server_opens": integrity == "ok",
255
            "rejections_actionable": not unactionable,
256
            # an acknowledged push must report the commit the client had (Oak push --json pushed_head)
257
            "acked_head_matches_reported": (not head_mismatches) if any(o.meta.get("pushed_head_reported")
258
                                                                        for o in ops) else None,
259
        }
260
        obs = {
261
            "pushes": len(ops),
262
            "acknowledged": sum(1 for o in ops if o.outcome == OK),
263
            "rejected": sum(1 for o in ops if o.outcome == REJECTED),
264
            "unknown": sum(1 for o in ops if o.outcome == UNKNOWN),
265
            "pull_returncodes": {str(rc): pulls.count(rc) for rc in set(pulls)},
266
            "lost": [o.value for o in lost][:5],
267
            "missing_files": missing_files[:5],
268
            "lincheck": {b: {"linearizable": r.linearizable, "explored": r.explored, "reason": r.reason}
269
                         for b, r in results.items()},
270
            "history": [[o.client, o.outcome, (o.value or "")[:12], round(o.call, 3),
271
                         None if o.ret == math.inf else round(o.ret, 3), o.meta.get("rc")]
272
                        for o in sorted(ops, key=lambda o: o.call)][:400],
273
            "final_heads": {b: (h or "")[:12] for b, h in final.items()},
274
            "worker_errors": errors[:5],
275
            "unactionable_rejections": unactionable[:3],
276
            "pushed_head_mismatches": head_mismatches[:5],
277
            "tree_reader": tree_status,
278
        }
279
        classification = classify(inv, harness_error="; ".join(errors[:3]) if errors else None)
280
    finally:
281
        world.stop()
282
    return {"invariants": inv, "observations": obs, "classification": classification,
283
            "elapsed_ms": round((time.monotonic() - t0) * 1000.0, 3),
284
            **({"harness_error": "; ".join(errors[:3])} if errors else {})}
285
286
287
def shared_checkout(kind: str, binary: str, root: Path, n: int, what: str) -> dict:
288
    """what: commit | push | stale_owner"""
289
    world = World(kind, binary, root, 1)
290
    repo = world.workers[0]
291
    env = world.adapter.env()
292
    t0 = time.monotonic()
293
    results = []
294
    lock = threading.Lock()
295
    obs: dict = {}
296
    try:
297
        stale = None
298
        stale_fired: Optional[bool] = None
299
        if what == "stale_owner":
300
            if kind != "oak":
301
                raise NotImplementedError("stale_owner is Oak-only (git leaves index.lock by design)")
302
            from . import crashpoints
303
304
            inst, why = crashpoints.build(root / "inst", binary, None)
305
            why = crashpoints.unavailable_reason(inst, why, "oak")
306
            if why:
307
                raise NotImplementedError(why)
308
            (repo / "stale.txt").write_text("stale owner\n")
309
            stale = run([str(inst.oak_bin), "commit"], repo, world.adapter.env(inst.env(1)), timeout=30)
310
            obs["stale_owner_rc"] = stale.returncode
311
            stale_fired = stale.returncode == -9
312
        if what == "push":
313
            (repo / "pub.txt").write_text("publication\n")
314
            for argv in world.commit_cmds():
315
                world.adapter.must(argv, repo)
316
        for i in range(n):
317
            (repo / f"shared-{i:02d}.txt").write_text(f"shared {i}\n")
318
319
        def contender(i: int) -> None:
320
            if what == "push":
321
                cmds = [world.push_cmd()]
322
            else:
323
                cmds = world.commit_cmds()
324
            for argv in cmds:
325
                res = run(argv, repo, env, timeout=OP_TIMEOUT)
326
                with lock:
327
                    results.append(res)
328
                if res.returncode != 0:
329
                    break
330
331
        threads = [threading.Thread(target=contender, args=(i,)) for i in range(n)]
332
        for thread in threads:
333
            thread.start()
334
        for thread in threads:
335
            thread.join(timeout=OP_TIMEOUT * 3)
336
        hung = any(thread.is_alive() for thread in threads) or any(r.timed_out for r in results)
337
        failures = [r for r in results if r.returncode != 0]
338
        final_ok = True
339
        missing: list[str] = []
340
        if what in ("commit", "stale_owner"):
341
            for argv in world.commit_cmds():
342
                res = run(argv, repo, env, timeout=OP_TIMEOUT)
343
            tree, status = world.adapter.tree(repo)
344
            obs["tree_reader"] = status
345
            if tree is not None:
346
                missing = [f"shared-{i:02d}.txt" for i in range(n) if f"shared-{i:02d}.txt" not in tree]
347
            final_ok = None if tree is None else not missing
348
        else:
349
            final_ok = world.server_heads().get(world.branch) == world.head(repo)
350
        if kind == "oak":
351
            with OakGraph(repo / ".oak" / "oak.db") as g:
352
                opens = g.integrity() == "ok"
353
        else:
354
            opens = world.adapter.oracle.fsck(repo) == "ok"
355
        successes = sum(1 for r in results if r.returncode == 0)
356
        inv = {
357
            "no_hang": not hung,
358
            "failures_actionable": all(actionable(r) for r in failures),
359
            "repo_opens": opens,
360
            "final_state_complete": final_ok,
361
        }
362
        if what == "stale_owner":
363
            inv["stale_owner_reaped"] = successes > 0
364
        obs.update(contenders=n, successes=successes, failures=len(failures), missing=missing[:5],
365
                   failure_samples=[r.brief(200) for r in failures if not actionable(r)][:2],
366
                   max_contender_ms=round(max((r.elapsed_ms for r in results), default=0.0), 1))
367
        classification = classify(inv, fault_fired=stale_fired)
368
        error = None
369
    except NotImplementedError as exc:
370
        return {"unsupported": str(exc)}
371
    except Exception as exc:  # noqa: BLE001
372
        inv, classification, error = {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
373
    finally:
374
        world.stop()
375
    row = {"invariants": inv, "observations": obs, "classification": classification,
376
           "elapsed_ms": round((time.monotonic() - t0) * 1000.0, 3)}
377
    if what == "stale_owner":
378
        row["fault_fired"] = stale_fired
379
        if classification == FAULT_NOT_FIRED:
380
            row["skip_reason"] = (f"lock owner was not killed at its first durability boundary "
381
                                  f"(rc={obs.get('stale_owner_rc')})")
382
    if error:
383
        row["harness_error"] = error
384
    return row