| 1 | """Network-fault family: the fault proxy sits between the client and a loopback server.
|
| 2 |
|
| 3 | Oak: client -> proxy (fixture port) -> ``oak serve`` (ephemeral port).
|
| 4 | Git: client -> proxy (fixture port) -> stdlib smart-HTTP ``git http-backend``.
|
| 5 |
|
| 6 | A clean record run enumerates the operation's requests ("METHOD /path" +
|
| 7 | occurrence). Each trial then injects exactly one per-request fault (or one
|
| 8 | global slow/jitter mode) and is judged by the crash-family invariants plus:
|
| 9 |
|
| 10 | - ``no_false_success`` exit 0 => exactly the calibrated outcome
|
| 11 | - ``unknown_not_reported_success`` when the reply to a mutating request was
|
| 12 | lost (drop/truncate/kill-after-upstream) the client must not exit 0 without
|
| 13 | having talked to the server again (reconciled) afterwards
|
| 14 | """
|
| 15 |
|
| 16 | from __future__ import annotations
|
| 17 |
|
| 18 | import os
|
| 19 | import signal
|
| 20 | import subprocess
|
| 21 | import time
|
| 22 | from pathlib import Path
|
| 23 | from typing import Callable, Optional
|
| 24 |
|
| 25 | from . import HARNESS_ERROR
|
| 26 | from .crash import evaluate
|
| 27 | from .engine import Calibration, Trial, classify, converge_check
|
| 28 | from .faultproxy import GLOBAL_KINDS, MUTATING_METHODS, Fault, FaultProxy
|
| 29 | from .fixtures import Fixture
|
| 30 | from .procs import Result, free_port
|
| 31 | from .readers import worktree_manifest
|
| 32 |
|
| 33 | PROXY_OPS = ("push", "pull_ff", "pull_diverged", "clone")
|
| 34 | PER_REQUEST = ("fail_status", "drop_after_send", "truncate_reply", "kill_before_forward", "kill_after_upstream")
|
| 35 | MAX_POINTS = 8
|
| 36 | DETERMINISM_ENV = {"OAK_UPLOAD_CONCURRENCY": "1", "OAK_DOWNLOAD_CONCURRENCY": "1"}
|
| 37 |
|
| 38 |
|
| 39 | class ProxiedTrial:
|
| 40 | """A Trial whose server listens behind a FaultProxy on the fixture port."""
|
| 41 |
|
| 42 | def __init__(self, fixture: Fixture, subject_bin: str, work: Path, name: str, faults: list[Fault]):
|
| 43 | self.trial = Trial(fixture, subject_bin, work, name)
|
| 44 | self.faults = faults
|
| 45 | self.pid: Optional[int] = None
|
| 46 | self.proxy: Optional[FaultProxy] = None
|
| 47 | self.gitserver = None
|
| 48 |
|
| 49 | def kill_client(self) -> None:
|
| 50 | if self.pid:
|
| 51 | try:
|
| 52 | os.kill(self.pid, signal.SIGKILL)
|
| 53 | except ProcessLookupError:
|
| 54 | pass
|
| 55 |
|
| 56 | def start(self) -> None:
|
| 57 | t = self.trial
|
| 58 | upstream = free_port()
|
| 59 | if t.kind == "oak":
|
| 60 | t.server.start(upstream)
|
| 61 | else:
|
| 62 | from oakbench.gitserver import GitHTTPServer
|
| 63 |
|
| 64 | self.gitserver = GitHTTPServer(t.dir / "srv", port=upstream, git_bin=t.adapter.bin).start()
|
| 65 | t.adapter.must([t.adapter.bin, "remote", "set-url", "origin", self.url + "/remote.git"], t.client)
|
| 66 | self.proxy = FaultProxy(upstream, self.faults, kill_hook=self.kill_client,
|
| 67 | listen_port=t.fixture.port).start()
|
| 68 |
|
| 69 | @property
|
| 70 | def url(self) -> str:
|
| 71 | return f"http://127.0.0.1:{self.trial.fixture.port}"
|
| 72 |
|
| 73 | def clone_url(self) -> str:
|
| 74 | return self.url if self.trial.kind == "oak" else self.url + "/remote.git"
|
| 75 |
|
| 76 | def stop(self) -> None:
|
| 77 | if self.proxy:
|
| 78 | self.proxy.stop()
|
| 79 | if self.gitserver:
|
| 80 | self.gitserver.stop()
|
| 81 | self.trial.stop_server()
|
| 82 |
|
| 83 | def run_op(self, timeout: float = 120) -> Result:
|
| 84 | t = self.trial
|
| 85 | argv = t.op_argv(url=self.clone_url())
|
| 86 | # Per-request faults need a deterministic request sequence; global jitter/slow
|
| 87 | # modes keep the subject's default concurrency so jitter actually reorders replies.
|
| 88 | deterministic = not any(f.kind in GLOBAL_KINDS for f in self.faults)
|
| 89 | env = t.adapter.env(DETERMINISM_ENV if deterministic else None)
|
| 90 | start = time.perf_counter()
|
| 91 | proc = subprocess.Popen(argv, cwd=str(t.op_cwd()), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
| 92 | stdin=subprocess.DEVNULL, start_new_session=True)
|
| 93 | self.pid = proc.pid
|
| 94 | try:
|
| 95 | out, err = proc.communicate(timeout=timeout)
|
| 96 | timed_out = False
|
| 97 | except subprocess.TimeoutExpired:
|
| 98 | os.killpg(proc.pid, signal.SIGKILL)
|
| 99 | out, err = proc.communicate()
|
| 100 | timed_out = True
|
| 101 | self.pid = None
|
| 102 | return Result(argv, None if timed_out else proc.returncode, out.decode("utf-8", "replace"),
|
| 103 | err.decode("utf-8", "replace"), (time.perf_counter() - start) * 1000.0,
|
| 104 | killed_by_harness=timed_out, timed_out=timed_out)
|
| 105 |
|
| 106 |
|
| 107 | def record(fixture: Fixture, subject_bin: str, work: Path) -> tuple[Calibration, list[tuple[str, int]], list]:
|
| 108 | pt = ProxiedTrial(fixture, subject_bin, work, "proxy-record", [])
|
| 109 | try:
|
| 110 | pt.start()
|
| 111 | pt.trial.pre()
|
| 112 | res = pt.run_op()
|
| 113 | pt.trial.settle()
|
| 114 | if res.returncode != 0:
|
| 115 | return Calibration(res.elapsed_ms, None, None, None, None, ok=False,
|
| 116 | detail=f"unfaulted proxied op failed rc={res.returncode}: {res.stderr[-300:]}"), [], []
|
| 117 | repo = pt.trial.target_repo()
|
| 118 | tree, status = pt.trial.adapter.tree(repo)
|
| 119 | calib = Calibration(res.elapsed_ms, tree, worktree_manifest(repo), pt.trial.adapter.head(repo),
|
| 120 | pt.trial.server_head(), detail=f"tree_reader={status}")
|
| 121 | keys = pt.proxy.keys_in_order()
|
| 122 | methods = [(r.key, r.occurrence, r.method) for r in sorted(pt.proxy.records, key=lambda r: r.index)]
|
| 123 | calib.requests = [f"{k}#{o}" for k, o in keys]
|
| 124 | return calib, keys, methods
|
| 125 | finally:
|
| 126 | pt.stop()
|
| 127 | pt.trial.cleanup()
|
| 128 |
|
| 129 |
|
| 130 | def fault_points(keys: list[tuple[str, int]]) -> list[tuple[str, int]]:
|
| 131 | """Every distinct request key's first and last occurrence, capped at MAX_POINTS."""
|
| 132 | last: dict[str, int] = {}
|
| 133 | order: list[str] = []
|
| 134 | for key, occ in keys:
|
| 135 | if key not in last:
|
| 136 | order.append(key)
|
| 137 | last[key] = max(last.get(key, 0), occ)
|
| 138 | points: list[tuple[str, int]] = []
|
| 139 | for key in order:
|
| 140 | points.append((key, 1))
|
| 141 | if last[key] > 1:
|
| 142 | points.append((key, last[key]))
|
| 143 | if len(points) > MAX_POINTS:
|
| 144 | # Mutating requests (unknown-outcome risk) always stay; reads fill the rest evenly.
|
| 145 | mutating = [p for p in points if p[0].split(" ", 1)[0] in MUTATING_METHODS]
|
| 146 | reads = [p for p in points if p not in mutating]
|
| 147 | room = max(0, MAX_POINTS - len(mutating))
|
| 148 | step = len(reads) / room if room else 0
|
| 149 | chosen = set(mutating) | {reads[int(i * step)] for i in range(room)} if reads else set(mutating)
|
| 150 | points = [p for p in points if p in chosen]
|
| 151 | return points
|
| 152 |
|
| 153 |
|
| 154 | def run_proxy_trial(fixture: Fixture, subject_bin: str, work: Path, name: str, calib: Calibration,
|
| 155 | fault: Fault, keep: Callable[[Trial, str], None]) -> dict:
|
| 156 | pt = ProxiedTrial(fixture, subject_bin, work, name, [fault])
|
| 157 | started = time.perf_counter()
|
| 158 | op_res: Optional[Result] = None
|
| 159 | try:
|
| 160 | pt.start()
|
| 161 | pt.trial.pre()
|
| 162 | op_res = pt.run_op()
|
| 163 | fired = [f for f in pt.proxy.fired()]
|
| 164 | # judge with the crash-family evaluator (it also runs the next op through the proxy,
|
| 165 | # whose one-shot fault is already consumed)
|
| 166 | inv, obs = evaluate(pt.trial, calib, op_res, bool(fired))
|
| 167 | obs["requests"] = len(pt.proxy.records)
|
| 168 | faulted = [r for r in pt.proxy.records if r.tag and not r.tag.startswith("SLOW") and "CLIENT_GONE" not in r.tag]
|
| 169 | obs["faulted_requests"] = [f"{r.key}#{r.occurrence}:{r.tag}" for r in faulted][:3]
|
| 170 | if op_res.returncode == 0:
|
| 171 | ok, detail = converge_check(pt.trial, calib, pt.trial.target_repo())
|
| 172 | inv["op_success_is_true"] = ok
|
| 173 | obs["op_converge"] = detail
|
| 174 | unknown = [r for r in faulted if r.method in MUTATING_METHODS
|
| 175 | and r.tag in ("REPLY_DROPPED", "REPLY_TRUNCATED")]
|
| 176 | if unknown and fault.kind not in GLOBAL_KINDS:
|
| 177 | first = min(r.index for r in unknown)
|
| 178 | later = [r for r in pt.proxy.records if r.index > first]
|
| 179 | inv["unknown_not_reported_success"] = bool(later)
|
| 180 | obs["reconciled_after_unknown"] = len(later)
|
| 181 | if op_res.returncode not in (0, None) and fixture.op == "push":
|
| 182 | # Known-failure report for a mutation that was in fact applied: not an
|
| 183 | # invariant (failure is the safe direction) but worth surfacing.
|
| 184 | obs["applied_but_reported_failure"] = pt.trial.server_head() == calib.post_server_head
|
| 185 | classification = classify(inv)
|
| 186 | error = None
|
| 187 | except Exception as exc: # noqa: BLE001
|
| 188 | inv, obs, classification, error = {}, {}, HARNESS_ERROR, f"{type(exc).__name__}: {exc}"
|
| 189 | fired = []
|
| 190 | finally:
|
| 191 | pt.stop()
|
| 192 | row = {
|
| 193 | "fault_point": fault.describe(),
|
| 194 | "fault_fired": bool(fired),
|
| 195 | "invariants": inv,
|
| 196 | "violations": sorted(k for k, v in inv.items() if v is False),
|
| 197 | "observations": obs,
|
| 198 | "classification": classification,
|
| 199 | "elapsed_ms": round(op_res.elapsed_ms if op_res else (time.perf_counter() - started) * 1000.0, 3),
|
| 200 | "command": op_res.argv if op_res else [],
|
| 201 | }
|
| 202 | if error:
|
| 203 | row["harness_error"] = error
|
| 204 | keep(pt.trial, classification)
|
| 205 | return row
|
| 206 |
|
| 207 |
|
| 208 | def plan(keys: list[tuple[str, int]], methods: list, seeds: tuple[int, ...] = (1, 2)) -> list[Fault]:
|
| 209 | faults: list[Fault] = []
|
| 210 | for key, occ in fault_points(keys):
|
| 211 | for kind in PER_REQUEST:
|
| 212 | faults.append(Fault(kind, key=key, occurrence=occ, status=503))
|
| 213 | faults.append(Fault("slow_reply", bytes_per_s=256 * 1024))
|
| 214 | for seed in seeds:
|
| 215 | faults.append(Fault("jitter", max_ms=120, seed=seed))
|
| 216 | return faults
|