Viewing files at commit 7630895676caCommit detailsView latest

suites/robustness/robustness/faultproxy.py 293 lines · 11.2 KB · python
1
"""Fault-injecting HTTP/1.1 reverse proxy for loopback subjects only.
2
3
Idea lineage: the 2026-09-26 W3a QA proxy (drop_reply / fail rules); this is a
4
rewrite with a typed fault plan, per-request records, and client-kill hooks.
5
It refuses non-loopback upstreams: fault injection never targets oak.space.
6
7
Fault kinds (each rule fires once, on the n-th occurrence of an exact
8
"METHOD /path" key, or on every request for the global kinds):
9
10
- fail_status        reply ``status`` without forwarding (typed server error)
11
- drop_after_send    forward, then close without replying (unknown outcome)
12
- truncate_reply     forward, send headers + half the body, close (partial body)
13
- kill_before_forward  run the kill hook (client dies), never forward
14
- kill_after_upstream  forward, run the kill hook, close (server applied it)
15
- slow_reply  (global) relay each reply at ``bytes_per_s``
16
- jitter      (global) random 0..``max_ms`` delay before forwarding (seeded):
17
              concurrent requests complete out of order (reordering)
18
"""
19
20
from __future__ import annotations
21
22
import hashlib
23
import http.client
24
import ipaddress
25
import random
26
import threading
27
import time
28
from dataclasses import dataclass, field
29
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
30
from typing import Callable, Optional
31
32
PER_REQUEST_KINDS = ("fail_status", "drop_after_send", "truncate_reply", "kill_before_forward", "kill_after_upstream")
33
GLOBAL_KINDS = ("slow_reply", "jitter")
34
MUTATING_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"})
35
HOP_HEADERS = frozenset({"host", "transfer-encoding", "content-length", "connection", "keep-alive"})
36
37
38
@dataclass
39
class Fault:
40
    kind: str
41
    key: Optional[str] = None  # "METHOD /path" (query stripped); None for global kinds
42
    occurrence: int = 1
43
    status: int = 503
44
    bytes_per_s: int = 32 * 1024
45
    max_ms: int = 150
46
    seed: int = 0
47
    fired: bool = False
48
49
    def describe(self) -> str:
50
        if self.kind in GLOBAL_KINDS:
51
            detail = f"bytes_per_s={self.bytes_per_s}" if self.kind == "slow_reply" else f"max_ms={self.max_ms},seed={self.seed}"
52
            return f"{self.kind}({detail})"
53
        extra = f",status={self.status}" if self.kind == "fail_status" else ""
54
        return f"{self.kind}@{self.key}#{self.occurrence}{extra}"
55
56
57
@dataclass
58
class Record:
59
    index: int
60
    method: str
61
    path: str
62
    key: str
63
    occurrence: int
64
    request_bytes: int
65
    request_sha256: str
66
    status: Optional[int] = None
67
    response_bytes: Optional[int] = None
68
    tag: str = ""
69
    t_arrive: float = 0.0
70
    t_done: float = 0.0
71
72
73
def request_key(method: str, path: str) -> str:
74
    return f"{method} {path.split('?', 1)[0]}"
75
76
77
def _assert_loopback(host: str) -> None:
78
    try:
79
        if ipaddress.ip_address(host).is_loopback:
80
            return
81
    except ValueError:
82
        if host == "localhost":
83
            return
84
    raise ValueError(f"fault proxy refuses non-loopback upstream {host!r}")
85
86
87
class FaultProxy:
88
    def __init__(self, upstream_port: int, faults: Optional[list[Fault]] = None, upstream_host: str = "127.0.0.1",
89
                 kill_hook: Optional[Callable[[], None]] = None, upstream_timeout_s: float = 120.0,
90
                 listen_port: int = 0):
91
        _assert_loopback(upstream_host)
92
        self.listen_port = listen_port
93
        self.upstream_host = upstream_host
94
        self.upstream_port = upstream_port
95
        self.faults = list(faults or [])
96
        self.kill_hook = kill_hook
97
        self.upstream_timeout_s = upstream_timeout_s
98
        self.records: list[Record] = []
99
        self._occurrences: dict[str, int] = {}
100
        self._lock = threading.Lock()
101
        self._httpd: Optional[ThreadingHTTPServer] = None
102
        self._thread: Optional[threading.Thread] = None
103
        self._rng = random.Random(next((f.seed for f in self.faults if f.kind == "jitter"), 0))
104
105
    # -- lifecycle -----------------------------------------------------------
106
    def start(self) -> "FaultProxy":
107
        proxy = self
108
109
        class Handler(BaseHTTPRequestHandler):
110
            protocol_version = "HTTP/1.1"
111
112
            def log_message(self, *args):  # noqa: D401 - silence
113
                pass
114
115
            def handle_any(self):
116
                proxy._handle(self)
117
118
            do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_HEAD = handle_any
119
120
        class Server(ThreadingHTTPServer):
121
            daemon_threads = True
122
123
            def handle_error(self, request, client_address):  # noqa: D401 - injected faults reset sockets
124
                pass
125
126
        httpd = Server(("127.0.0.1", self.listen_port), Handler)
127
        self._httpd = httpd
128
        self._thread = threading.Thread(target=httpd.serve_forever, name="p4-faultproxy", daemon=True)
129
        self._thread.start()
130
        return self
131
132
    def stop(self) -> None:
133
        if self._httpd is not None:
134
            self._httpd.shutdown()
135
            self._httpd.server_close()
136
            self._httpd = None
137
138
    @property
139
    def port(self) -> int:
140
        assert self._httpd is not None
141
        return int(self._httpd.server_address[1])
142
143
    @property
144
    def url(self) -> str:
145
        return f"http://127.0.0.1:{self.port}"
146
147
    def __enter__(self) -> "FaultProxy":
148
        return self.start()
149
150
    def __exit__(self, *exc) -> None:
151
        self.stop()
152
153
    # -- evidence ------------------------------------------------------------
154
    def fired(self) -> list[Fault]:
155
        return [f for f in self.faults if f.fired]
156
157
    def keys_in_order(self) -> list[tuple[str, int]]:
158
        return [(r.key, r.occurrence) for r in sorted(self.records, key=lambda r: r.index)]
159
160
    def records_after(self, index: int) -> list[Record]:
161
        return [r for r in self.records if r.index > index]
162
163
    # -- core ----------------------------------------------------------------
164
    def _take(self, key: str, occurrence: int) -> Optional[Fault]:
165
        with self._lock:
166
            for fault in self.faults:
167
                if fault.fired or fault.kind in GLOBAL_KINDS:
168
                    continue
169
                if fault.key == key and fault.occurrence == occurrence:
170
                    fault.fired = True
171
                    return fault
172
        return None
173
174
    def _global(self, kind: str) -> Optional[Fault]:
175
        for fault in self.faults:
176
            if fault.kind == kind:
177
                fault.fired = True
178
                return fault
179
        return None
180
181
    @staticmethod
182
    def _read_body(handler: BaseHTTPRequestHandler) -> bytes:
183
        length = handler.headers.get("Content-Length")
184
        if length is not None:
185
            return handler.rfile.read(int(length))
186
        if handler.headers.get("Transfer-Encoding", "").lower() == "chunked":
187
            body = b""
188
            while True:
189
                size = int(handler.rfile.readline().strip().split(b";")[0], 16)
190
                if size == 0:
191
                    handler.rfile.readline()
192
                    return body
193
                body += handler.rfile.read(size)
194
                handler.rfile.readline()
195
        return b""
196
197
    def _handle(self, handler: BaseHTTPRequestHandler) -> None:
198
        body = self._read_body(handler)
199
        key = request_key(handler.command, handler.path)
200
        with self._lock:
201
            occurrence = self._occurrences.get(key, 0) + 1
202
            self._occurrences[key] = occurrence
203
            record = Record(len(self.records), handler.command, handler.path, key, occurrence, len(body),
204
                            hashlib.sha256(body).hexdigest()[:16], t_arrive=time.monotonic())
205
            self.records.append(record)
206
        fault = self._take(key, occurrence)
207
        jitter = self._global("jitter")
208
        if jitter is not None:
209
            with self._lock:
210
                delay = self._rng.uniform(0, jitter.max_ms) / 1000.0
211
            time.sleep(delay)
212
        try:
213
            if fault and fault.kind == "fail_status":
214
                payload = b'{"error":"p4 injected fault"}'
215
                handler.send_response(fault.status)
216
                handler.send_header("Content-Type", "application/json")
217
                handler.send_header("Content-Length", str(len(payload)))
218
                handler.end_headers()
219
                handler.wfile.write(payload)
220
                record.status, record.response_bytes, record.tag = fault.status, len(payload), "INJECTED_STATUS"
221
                return
222
            if fault and fault.kind == "kill_before_forward":
223
                record.tag = "KILLED_BEFORE_FORWARD"
224
                if self.kill_hook:
225
                    self.kill_hook()
226
                self._abort(handler)
227
                return
228
            status, headers, data = self._forward(handler, body)
229
            record.status, record.response_bytes = status, len(data)
230
            if fault and fault.kind == "drop_after_send":
231
                record.tag = "REPLY_DROPPED"
232
                self._abort(handler)
233
                return
234
            if fault and fault.kind == "kill_after_upstream":
235
                record.tag = "KILLED_AFTER_UPSTREAM"
236
                if self.kill_hook:
237
                    self.kill_hook()
238
                self._abort(handler)
239
                return
240
            handler.send_response(status)
241
            for name, value in headers:
242
                if name.lower() not in HOP_HEADERS:
243
                    handler.send_header(name, value)
244
            handler.send_header("Content-Length", str(len(data)))
245
            handler.end_headers()
246
            if fault and fault.kind == "truncate_reply":
247
                record.tag = "REPLY_TRUNCATED"
248
                handler.wfile.write(data[: len(data) // 2])
249
                handler.wfile.flush()
250
                self._abort(handler)
251
                return
252
            slow = self._global("slow_reply")
253
            if slow is not None and data:
254
                record.tag = "SLOW"
255
                step = max(1, slow.bytes_per_s // 20)
256
                for offset in range(0, len(data), step):
257
                    handler.wfile.write(data[offset: offset + step])
258
                    handler.wfile.flush()
259
                    time.sleep(0.05)
260
            else:
261
                handler.wfile.write(data)
262
        except (BrokenPipeError, ConnectionResetError, OSError) as exc:
263
            record.tag = (record.tag + " " if record.tag else "") + f"CLIENT_GONE:{type(exc).__name__}"
264
            handler.close_connection = True
265
        finally:
266
            record.t_done = time.monotonic()
267
268
    def _forward(self, handler: BaseHTTPRequestHandler, body: bytes):
269
        conn = http.client.HTTPConnection(self.upstream_host, self.upstream_port, timeout=self.upstream_timeout_s)
270
        try:
271
            headers = {k: v for k, v in handler.headers.items() if k.lower() not in HOP_HEADERS}
272
            headers["Content-Length"] = str(len(body))
273
            headers["Host"] = f"{self.upstream_host}:{self.upstream_port}"
274
            conn.request(handler.command, handler.path, body=body, headers=headers)
275
            response = conn.getresponse()
276
            data = response.read()
277
            return response.status, response.getheaders(), data
278
        finally:
279
            conn.close()
280
281
    @staticmethod
282
    def _abort(handler: BaseHTTPRequestHandler) -> None:
283
        handler.close_connection = True
284
        try:
285
            handler.wfile.flush()
286
        except OSError:
287
            pass
288
        try:
289
            import socket
290
291
            handler.connection.shutdown(socket.SHUT_RDWR)
292
        except OSError:
293
            pass