Viewing files at commit c44c91118a6cCommit detailsView latest

suites/robustness/robustness/proxycases.py 220 lines · 9.7 KB · python
1
"""Network-fault family: the fault proxy sits between the client and a loopback server.
2
3
Oak: client -> proxy (fixture port) -> ``oak serve`` (ephemeral port).
4
Git: client -> proxy (fixture port) -> stdlib smart-HTTP ``git http-backend``.
5
6
A clean record run enumerates the operation's requests ("METHOD /path" +
7
occurrence). Each trial then injects exactly one per-request fault (or one
8
global slow/jitter mode) and is judged by the crash-family invariants plus:
9
10
- ``no_false_success``            exit 0 => exactly the calibrated outcome
11
- ``unknown_not_reported_success`` when the reply to a mutating request was
12
  lost (drop/truncate/kill-after-upstream) the client must not exit 0 without
13
  having talked to the server again (reconciled) afterwards
14
"""
15
16
from __future__ import annotations
17
18
import os
19
import signal
20
import subprocess
21
import time
22
from pathlib import Path
23
from typing import Callable, Optional
24
25
from . import FAULT_NOT_FIRED, HARNESS_ERROR
26
from .crash import evaluate
27
from .engine import Calibration, Trial, classify, converge_check
28
from .faultproxy import GLOBAL_KINDS, MUTATING_METHODS, Fault, FaultProxy
29
from .fixtures import Fixture
30
from .procs import Result, free_port
31
from .readers import worktree_manifest
32
33
PROXY_OPS = ("push", "pull_ff", "pull_diverged", "clone")
34
PER_REQUEST = ("fail_status", "drop_after_send", "truncate_reply", "kill_before_forward", "kill_after_upstream")
35
MAX_POINTS = 8
36
DETERMINISM_ENV = {"OAK_UPLOAD_CONCURRENCY": "1", "OAK_DOWNLOAD_CONCURRENCY": "1"}
37
38
39
class ProxiedTrial:
40
    """A Trial whose server listens behind a FaultProxy on the fixture port."""
41
42
    def __init__(self, fixture: Fixture, subject_bin: str, work: Path, name: str, faults: list[Fault]):
43
        self.trial = Trial(fixture, subject_bin, work, name)
44
        self.faults = faults
45
        self.pid: Optional[int] = None
46
        self.proxy: Optional[FaultProxy] = None
47
        self.gitserver = None
48
49
    def kill_client(self) -> None:
50
        if self.pid:
51
            try:
52
                os.kill(self.pid, signal.SIGKILL)
53
            except ProcessLookupError:
54
                pass
55
56
    def start(self) -> None:
57
        t = self.trial
58
        upstream = free_port()
59
        if t.kind == "oak":
60
            t.server.start(upstream)
61
        else:
62
            from oakbench.gitserver import GitHTTPServer
63
64
            self.gitserver = GitHTTPServer(t.dir / "srv", port=upstream, git_bin=t.adapter.bin).start()
65
            t.adapter.must([t.adapter.bin, "remote", "set-url", "origin", self.url + "/remote.git"], t.client)
66
        self.proxy = FaultProxy(upstream, self.faults, kill_hook=self.kill_client,
67
                                listen_port=t.fixture.port).start()
68
69
    @property
70
    def url(self) -> str:
71
        return f"http://127.0.0.1:{self.trial.fixture.port}"
72
73
    def clone_url(self) -> str:
74
        return self.url if self.trial.kind == "oak" else self.url + "/remote.git"
75
76
    def stop(self) -> None:
77
        if self.proxy:
78
            self.proxy.stop()
79
        if self.gitserver:
80
            self.gitserver.stop()
81
        self.trial.stop_server()
82
83
    def run_op(self, timeout: float = 120) -> Result:
84
        t = self.trial
85
        argv = t.op_argv(url=self.clone_url())
86
        # Per-request faults need a deterministic request sequence; global jitter/slow
87
        # modes keep the subject's default concurrency so jitter actually reorders replies.
88
        deterministic = not any(f.kind in GLOBAL_KINDS for f in self.faults)
89
        env = t.adapter.env(DETERMINISM_ENV if deterministic else None)
90
        start = time.perf_counter()
91
        proc = subprocess.Popen(argv, cwd=str(t.op_cwd()), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
92
                                stdin=subprocess.DEVNULL, start_new_session=True)
93
        self.pid = proc.pid
94
        try:
95
            out, err = proc.communicate(timeout=timeout)
96
            timed_out = False
97
        except subprocess.TimeoutExpired:
98
            os.killpg(proc.pid, signal.SIGKILL)
99
            out, err = proc.communicate()
100
            timed_out = True
101
        self.pid = None
102
        return Result(argv, None if timed_out else proc.returncode, out.decode("utf-8", "replace"),
103
                      err.decode("utf-8", "replace"), (time.perf_counter() - start) * 1000.0,
104
                      killed_by_harness=timed_out, timed_out=timed_out)
105
106
107
def record(fixture: Fixture, subject_bin: str, work: Path) -> tuple[Calibration, list[tuple[str, int]], list]:
108
    pt = ProxiedTrial(fixture, subject_bin, work, "proxy-record", [])
109
    try:
110
        pt.start()
111
        pt.trial.pre()
112
        res = pt.run_op()
113
        pt.trial.settle()
114
        if res.returncode != 0:
115
            return Calibration(res.elapsed_ms, None, None, None, None, ok=False,
116
                               detail=f"unfaulted proxied op failed rc={res.returncode}: {res.stderr[-300:]}"), [], []
117
        repo = pt.trial.target_repo()
118
        tree, status = pt.trial.adapter.tree(repo)
119
        calib = Calibration(res.elapsed_ms, tree, worktree_manifest(repo), pt.trial.adapter.head(repo),
120
                            pt.trial.server_head(), detail=f"tree_reader={status}")
121
        keys = pt.proxy.keys_in_order()
122
        methods = [(r.key, r.occurrence, r.method) for r in sorted(pt.proxy.records, key=lambda r: r.index)]
123
        calib.requests = [f"{k}#{o}" for k, o in keys]
124
        return calib, keys, methods
125
    finally:
126
        pt.stop()
127
        pt.trial.cleanup()
128
129
130
def fault_points(keys: list[tuple[str, int]]) -> list[tuple[str, int]]:
131
    """Every distinct request key's first and last occurrence, capped at MAX_POINTS."""
132
    last: dict[str, int] = {}
133
    order: list[str] = []
134
    for key, occ in keys:
135
        if key not in last:
136
            order.append(key)
137
        last[key] = max(last.get(key, 0), occ)
138
    points: list[tuple[str, int]] = []
139
    for key in order:
140
        points.append((key, 1))
141
        if last[key] > 1:
142
            points.append((key, last[key]))
143
    if len(points) > MAX_POINTS:
144
        # Mutating requests (unknown-outcome risk) always stay; reads fill the rest evenly.
145
        mutating = [p for p in points if p[0].split(" ", 1)[0] in MUTATING_METHODS]
146
        reads = [p for p in points if p not in mutating]
147
        room = max(0, MAX_POINTS - len(mutating))
148
        step = len(reads) / room if room else 0
149
        chosen = set(mutating) | {reads[int(i * step)] for i in range(room)} if reads else set(mutating)
150
        points = [p for p in points if p in chosen]
151
    return points
152
153
154
def run_proxy_trial(fixture: Fixture, subject_bin: str, work: Path, name: str, calib: Calibration,
155
                    fault: Fault, keep: Callable[[Trial, str], None]) -> dict:
156
    pt = ProxiedTrial(fixture, subject_bin, work, name, [fault])
157
    started = time.perf_counter()
158
    op_res: Optional[Result] = None
159
    try:
160
        pt.start()
161
        pt.trial.pre()
162
        op_res = pt.run_op()
163
        fired = [f for f in pt.proxy.fired()]
164
        # judge with the crash-family evaluator (it also runs the next op through the proxy,
165
        # whose one-shot fault is already consumed)
166
        inv, obs = evaluate(pt.trial, calib, op_res, bool(fired))
167
        obs["requests"] = len(pt.proxy.records)
168
        faulted = [r for r in pt.proxy.records if r.tag and not r.tag.startswith("SLOW") and "CLIENT_GONE" not in r.tag]
169
        obs["faulted_requests"] = [f"{r.key}#{r.occurrence}:{r.tag}" for r in faulted][:3]
170
        if op_res.returncode == 0:
171
            ok, detail = converge_check(pt.trial, calib, pt.trial.target_repo())
172
            inv["op_success_is_true"] = ok
173
            obs["op_converge"] = detail
174
            unknown = [r for r in faulted if r.method in MUTATING_METHODS
175
                       and r.tag in ("REPLY_DROPPED", "REPLY_TRUNCATED")]
176
            if unknown and fault.kind not in GLOBAL_KINDS:
177
                first = min(r.index for r in unknown)
178
                later = [r for r in pt.proxy.records if r.index > first]
179
                inv["unknown_not_reported_success"] = bool(later)
180
                obs["reconciled_after_unknown"] = len(later)
181
        if op_res.returncode not in (0, None) and fixture.op == "push":
182
            # Known-failure report for a mutation that was in fact applied: not an
183
            # invariant (failure is the safe direction) but worth surfacing.
184
            obs["applied_but_reported_failure"] = pt.trial.server_head() == calib.post_server_head
185
        classification = classify(inv, fault_fired=bool(fired), reader_errors=obs.get("reader_errors"))
186
        error = None
187
    except Exception as exc:  # noqa: BLE001
188
        inv, obs, classification, error = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
189
        fired = []
190
    finally:
191
        pt.stop()
192
    row = {
193
        "fault_point": fault.describe(),
194
        "fault_fired": bool(fired),
195
        "invariants": inv,
196
        "violations": sorted(k for k, v in inv.items() if v is False),
197
        "observations": obs,
198
        "classification": classification,
199
        "elapsed_ms": round(op_res.elapsed_ms if op_res else (time.perf_counter() - started) * 1000.0, 3),
200
        "command": op_res.argv if op_res else [],
201
    }
202
    if error:
203
        row["harness_error"] = error
204
    elif classification == HARNESS_ERROR and obs.get("reader_errors"):
205
        row["harness_error"] = "independent reader failed: " + "; ".join(obs["reader_errors"][:2])
206
    if classification == FAULT_NOT_FIRED:
207
        row["skip_reason"] = f"proxy fault {fault.describe()} never matched a request"
208
    keep(pt.trial, classification)
209
    return row
210
211
212
def plan(keys: list[tuple[str, int]], methods: list, seeds: tuple[int, ...] = (1, 2)) -> list[Fault]:
213
    faults: list[Fault] = []
214
    for key, occ in fault_points(keys):
215
        for kind in PER_REQUEST:
216
            faults.append(Fault(kind, key=key, occurrence=occ, status=503))
217
    faults.append(Fault("slow_reply", bytes_per_s=256 * 1024))
218
    for seed in seeds:
219
        faults.append(Fault("jitter", max_ms=120, seed=seed))
220
    return faults