| 1 | """Fault-injecting HTTP/1.1 reverse proxy for loopback subjects only.
|
| 2 |
|
| 3 | Idea lineage: the 2026-09-26 W3a QA proxy (drop_reply / fail rules); this is a
|
| 4 | rewrite with a typed fault plan, per-request records, and client-kill hooks.
|
| 5 | It refuses non-loopback upstreams: fault injection never targets oak.space.
|
| 6 |
|
| 7 | Fault kinds (each rule fires once, on the n-th occurrence of an exact
|
| 8 | "METHOD /path" key, or on every request for the global kinds):
|
| 9 |
|
| 10 | - fail_status reply ``status`` without forwarding (typed server error)
|
| 11 | - drop_after_send forward, then close without replying (unknown outcome)
|
| 12 | - truncate_reply forward, send headers + half the body, close (partial body)
|
| 13 | - kill_before_forward run the kill hook (client dies), never forward
|
| 14 | - kill_after_upstream forward, run the kill hook, close (server applied it)
|
| 15 | - slow_reply (global) relay each reply at ``bytes_per_s``
|
| 16 | - jitter (global) random 0..``max_ms`` delay before forwarding (seeded):
|
| 17 | concurrent requests complete out of order (reordering)
|
| 18 | """
|
| 19 |
|
| 20 | from __future__ import annotations
|
| 21 |
|
| 22 | import hashlib
|
| 23 | import http.client
|
| 24 | import ipaddress
|
| 25 | import random
|
| 26 | import threading
|
| 27 | import time
|
| 28 | from dataclasses import dataclass, field
|
| 29 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
| 30 | from typing import Callable, Optional
|
| 31 |
|
| 32 | PER_REQUEST_KINDS = ("fail_status", "drop_after_send", "truncate_reply", "kill_before_forward", "kill_after_upstream")
|
| 33 | GLOBAL_KINDS = ("slow_reply", "jitter")
|
| 34 | MUTATING_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"})
|
| 35 | HOP_HEADERS = frozenset({"host", "transfer-encoding", "content-length", "connection", "keep-alive"})
|
| 36 |
|
| 37 |
|
| 38 | @dataclass
|
| 39 | class Fault:
|
| 40 | kind: str
|
| 41 | key: Optional[str] = None # "METHOD /path" (query stripped); None for global kinds
|
| 42 | occurrence: int = 1
|
| 43 | status: int = 503
|
| 44 | bytes_per_s: int = 32 * 1024
|
| 45 | max_ms: int = 150
|
| 46 | seed: int = 0
|
| 47 | fired: bool = False
|
| 48 |
|
| 49 | def describe(self) -> str:
|
| 50 | if self.kind in GLOBAL_KINDS:
|
| 51 | detail = f"bytes_per_s={self.bytes_per_s}" if self.kind == "slow_reply" else f"max_ms={self.max_ms},seed={self.seed}"
|
| 52 | return f"{self.kind}({detail})"
|
| 53 | extra = f",status={self.status}" if self.kind == "fail_status" else ""
|
| 54 | return f"{self.kind}@{self.key}#{self.occurrence}{extra}"
|
| 55 |
|
| 56 |
|
| 57 | @dataclass
|
| 58 | class Record:
|
| 59 | index: int
|
| 60 | method: str
|
| 61 | path: str
|
| 62 | key: str
|
| 63 | occurrence: int
|
| 64 | request_bytes: int
|
| 65 | request_sha256: str
|
| 66 | status: Optional[int] = None
|
| 67 | response_bytes: Optional[int] = None
|
| 68 | tag: str = ""
|
| 69 | t_arrive: float = 0.0
|
| 70 | t_done: float = 0.0
|
| 71 |
|
| 72 |
|
| 73 | def request_key(method: str, path: str) -> str:
|
| 74 | return f"{method} {path.split('?', 1)[0]}"
|
| 75 |
|
| 76 |
|
| 77 | def _assert_loopback(host: str) -> None:
|
| 78 | try:
|
| 79 | if ipaddress.ip_address(host).is_loopback:
|
| 80 | return
|
| 81 | except ValueError:
|
| 82 | if host == "localhost":
|
| 83 | return
|
| 84 | raise ValueError(f"fault proxy refuses non-loopback upstream {host!r}")
|
| 85 |
|
| 86 |
|
| 87 | class FaultProxy:
|
| 88 | def __init__(self, upstream_port: int, faults: Optional[list[Fault]] = None, upstream_host: str = "127.0.0.1",
|
| 89 | kill_hook: Optional[Callable[[], None]] = None, upstream_timeout_s: float = 120.0,
|
| 90 | listen_port: int = 0):
|
| 91 | _assert_loopback(upstream_host)
|
| 92 | self.listen_port = listen_port
|
| 93 | self.upstream_host = upstream_host
|
| 94 | self.upstream_port = upstream_port
|
| 95 | self.faults = list(faults or [])
|
| 96 | self.kill_hook = kill_hook
|
| 97 | self.upstream_timeout_s = upstream_timeout_s
|
| 98 | self.records: list[Record] = []
|
| 99 | self._occurrences: dict[str, int] = {}
|
| 100 | self._lock = threading.Lock()
|
| 101 | self._httpd: Optional[ThreadingHTTPServer] = None
|
| 102 | self._thread: Optional[threading.Thread] = None
|
| 103 | self._rng = random.Random(next((f.seed for f in self.faults if f.kind == "jitter"), 0))
|
| 104 |
|
| 105 | # -- lifecycle -----------------------------------------------------------
|
| 106 | def start(self) -> "FaultProxy":
|
| 107 | proxy = self
|
| 108 |
|
| 109 | class Handler(BaseHTTPRequestHandler):
|
| 110 | protocol_version = "HTTP/1.1"
|
| 111 |
|
| 112 | def log_message(self, *args): # noqa: D401 - silence
|
| 113 | pass
|
| 114 |
|
| 115 | def handle_any(self):
|
| 116 | proxy._handle(self)
|
| 117 |
|
| 118 | do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_HEAD = handle_any
|
| 119 |
|
| 120 | class Server(ThreadingHTTPServer):
|
| 121 | daemon_threads = True
|
| 122 |
|
| 123 | def handle_error(self, request, client_address): # noqa: D401 - injected faults reset sockets
|
| 124 | pass
|
| 125 |
|
| 126 | httpd = Server(("127.0.0.1", self.listen_port), Handler)
|
| 127 | self._httpd = httpd
|
| 128 | self._thread = threading.Thread(target=httpd.serve_forever, name="p4-faultproxy", daemon=True)
|
| 129 | self._thread.start()
|
| 130 | return self
|
| 131 |
|
| 132 | def stop(self) -> None:
|
| 133 | if self._httpd is not None:
|
| 134 | self._httpd.shutdown()
|
| 135 | self._httpd.server_close()
|
| 136 | self._httpd = None
|
| 137 |
|
| 138 | @property
|
| 139 | def port(self) -> int:
|
| 140 | assert self._httpd is not None
|
| 141 | return int(self._httpd.server_address[1])
|
| 142 |
|
| 143 | @property
|
| 144 | def url(self) -> str:
|
| 145 | return f"http://127.0.0.1:{self.port}"
|
| 146 |
|
| 147 | def __enter__(self) -> "FaultProxy":
|
| 148 | return self.start()
|
| 149 |
|
| 150 | def __exit__(self, *exc) -> None:
|
| 151 | self.stop()
|
| 152 |
|
| 153 | # -- evidence ------------------------------------------------------------
|
| 154 | def fired(self) -> list[Fault]:
|
| 155 | return [f for f in self.faults if f.fired]
|
| 156 |
|
| 157 | def keys_in_order(self) -> list[tuple[str, int]]:
|
| 158 | return [(r.key, r.occurrence) for r in sorted(self.records, key=lambda r: r.index)]
|
| 159 |
|
| 160 | def records_after(self, index: int) -> list[Record]:
|
| 161 | return [r for r in self.records if r.index > index]
|
| 162 |
|
| 163 | # -- core ----------------------------------------------------------------
|
| 164 | def _take(self, key: str, occurrence: int) -> Optional[Fault]:
|
| 165 | with self._lock:
|
| 166 | for fault in self.faults:
|
| 167 | if fault.fired or fault.kind in GLOBAL_KINDS:
|
| 168 | continue
|
| 169 | if fault.key == key and fault.occurrence == occurrence:
|
| 170 | fault.fired = True
|
| 171 | return fault
|
| 172 | return None
|
| 173 |
|
| 174 | def _global(self, kind: str) -> Optional[Fault]:
|
| 175 | for fault in self.faults:
|
| 176 | if fault.kind == kind:
|
| 177 | fault.fired = True
|
| 178 | return fault
|
| 179 | return None
|
| 180 |
|
| 181 | @staticmethod
|
| 182 | def _read_body(handler: BaseHTTPRequestHandler) -> bytes:
|
| 183 | length = handler.headers.get("Content-Length")
|
| 184 | if length is not None:
|
| 185 | return handler.rfile.read(int(length))
|
| 186 | if handler.headers.get("Transfer-Encoding", "").lower() == "chunked":
|
| 187 | body = b""
|
| 188 | while True:
|
| 189 | size = int(handler.rfile.readline().strip().split(b";")[0], 16)
|
| 190 | if size == 0:
|
| 191 | handler.rfile.readline()
|
| 192 | return body
|
| 193 | body += handler.rfile.read(size)
|
| 194 | handler.rfile.readline()
|
| 195 | return b""
|
| 196 |
|
| 197 | def _handle(self, handler: BaseHTTPRequestHandler) -> None:
|
| 198 | body = self._read_body(handler)
|
| 199 | key = request_key(handler.command, handler.path)
|
| 200 | with self._lock:
|
| 201 | occurrence = self._occurrences.get(key, 0) + 1
|
| 202 | self._occurrences[key] = occurrence
|
| 203 | record = Record(len(self.records), handler.command, handler.path, key, occurrence, len(body),
|
| 204 | hashlib.sha256(body).hexdigest()[:16], t_arrive=time.monotonic())
|
| 205 | self.records.append(record)
|
| 206 | fault = self._take(key, occurrence)
|
| 207 | jitter = self._global("jitter")
|
| 208 | if jitter is not None:
|
| 209 | with self._lock:
|
| 210 | delay = self._rng.uniform(0, jitter.max_ms) / 1000.0
|
| 211 | time.sleep(delay)
|
| 212 | try:
|
| 213 | if fault and fault.kind == "fail_status":
|
| 214 | payload = b'{"error":"p4 injected fault"}'
|
| 215 | handler.send_response(fault.status)
|
| 216 | handler.send_header("Content-Type", "application/json")
|
| 217 | handler.send_header("Content-Length", str(len(payload)))
|
| 218 | handler.end_headers()
|
| 219 | handler.wfile.write(payload)
|
| 220 | record.status, record.response_bytes, record.tag = fault.status, len(payload), "INJECTED_STATUS"
|
| 221 | return
|
| 222 | if fault and fault.kind == "kill_before_forward":
|
| 223 | record.tag = "KILLED_BEFORE_FORWARD"
|
| 224 | if self.kill_hook:
|
| 225 | self.kill_hook()
|
| 226 | self._abort(handler)
|
| 227 | return
|
| 228 | status, headers, data = self._forward(handler, body)
|
| 229 | record.status, record.response_bytes = status, len(data)
|
| 230 | if fault and fault.kind == "drop_after_send":
|
| 231 | record.tag = "REPLY_DROPPED"
|
| 232 | self._abort(handler)
|
| 233 | return
|
| 234 | if fault and fault.kind == "kill_after_upstream":
|
| 235 | record.tag = "KILLED_AFTER_UPSTREAM"
|
| 236 | if self.kill_hook:
|
| 237 | self.kill_hook()
|
| 238 | self._abort(handler)
|
| 239 | return
|
| 240 | handler.send_response(status)
|
| 241 | for name, value in headers:
|
| 242 | if name.lower() not in HOP_HEADERS:
|
| 243 | handler.send_header(name, value)
|
| 244 | handler.send_header("Content-Length", str(len(data)))
|
| 245 | handler.end_headers()
|
| 246 | if fault and fault.kind == "truncate_reply":
|
| 247 | record.tag = "REPLY_TRUNCATED"
|
| 248 | handler.wfile.write(data[: len(data) // 2])
|
| 249 | handler.wfile.flush()
|
| 250 | self._abort(handler)
|
| 251 | return
|
| 252 | slow = self._global("slow_reply")
|
| 253 | if slow is not None and data:
|
| 254 | record.tag = "SLOW"
|
| 255 | step = max(1, slow.bytes_per_s // 20)
|
| 256 | for offset in range(0, len(data), step):
|
| 257 | handler.wfile.write(data[offset: offset + step])
|
| 258 | handler.wfile.flush()
|
| 259 | time.sleep(0.05)
|
| 260 | else:
|
| 261 | handler.wfile.write(data)
|
| 262 | except (BrokenPipeError, ConnectionResetError, OSError) as exc:
|
| 263 | record.tag = (record.tag + " " if record.tag else "") + f"CLIENT_GONE:{type(exc).__name__}"
|
| 264 | handler.close_connection = True
|
| 265 | finally:
|
| 266 | record.t_done = time.monotonic()
|
| 267 |
|
| 268 | def _forward(self, handler: BaseHTTPRequestHandler, body: bytes):
|
| 269 | conn = http.client.HTTPConnection(self.upstream_host, self.upstream_port, timeout=self.upstream_timeout_s)
|
| 270 | try:
|
| 271 | headers = {k: v for k, v in handler.headers.items() if k.lower() not in HOP_HEADERS}
|
| 272 | headers["Content-Length"] = str(len(body))
|
| 273 | headers["Host"] = f"{self.upstream_host}:{self.upstream_port}"
|
| 274 | conn.request(handler.command, handler.path, body=body, headers=headers)
|
| 275 | response = conn.getresponse()
|
| 276 | data = response.read()
|
| 277 | return response.status, response.getheaders(), data
|
| 278 | finally:
|
| 279 | conn.close()
|
| 280 |
|
| 281 | @staticmethod
|
| 282 | def _abort(handler: BaseHTTPRequestHandler) -> None:
|
| 283 | handler.close_connection = True
|
| 284 | try:
|
| 285 | handler.wfile.flush()
|
| 286 | except OSError:
|
| 287 | pass
|
| 288 | try:
|
| 289 | import socket
|
| 290 |
|
| 291 | handler.connection.shutdown(socket.SHUT_RDWR)
|
| 292 | except OSError:
|
| 293 | pass
|