| 1 | """Tests for shallow_worker_branch_refresh_v1 instruments: fixture, request counter, oracle."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import os
|
| 6 | import shutil
|
| 7 | import socket
|
| 8 | import sqlite3
|
| 9 | import sys
|
| 10 | import tempfile
|
| 11 | import threading
|
| 12 | import time
|
| 13 | import unittest
|
| 14 | import http.client
|
| 15 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
| 16 | from pathlib import Path
|
| 17 |
|
| 18 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 19 |
|
| 20 | import task_loop as lane
|
| 21 | from oakbench import merge_history_fixture as fixture
|
| 22 | from oakbench import shallow_refresh
|
| 23 | from oakbench.httpcount import HttpFramingError, HttpRequestStreamParser, RequestCountingProxy
|
| 24 | from oakbench.subjects import Subject
|
| 25 |
|
| 26 | GIT = shutil.which("git")
|
| 27 |
|
| 28 |
|
| 29 | def git_env() -> dict[str, str]:
|
| 30 | env = {key: value for key, value in os.environ.items() if key in {"PATH", "HOME", "TMPDIR"}}
|
| 31 | env.update({"GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"})
|
| 32 | return env
|
| 33 |
|
| 34 |
|
| 35 | class MergeHistoryFixtureTest(unittest.TestCase):
|
| 36 | def test_stream_is_deterministic_and_seed_sensitive(self) -> None:
|
| 37 | first, manifest_a = fixture.build_stream(300, 7)
|
| 38 | second, manifest_b = fixture.build_stream(300, 7)
|
| 39 | other, _ = fixture.build_stream(300, 8)
|
| 40 | self.assertEqual(first, second)
|
| 41 | self.assertEqual(manifest_a.tree_content_hash, manifest_b.tree_content_hash)
|
| 42 | self.assertNotEqual(first, other)
|
| 43 |
|
| 44 | def test_exact_commit_count_and_merge_ratio(self) -> None:
|
| 45 | for commits, seed in ((1000, 1), (2000, 2)):
|
| 46 | _stream, manifest = fixture.build_stream(commits, seed)
|
| 47 | self.assertEqual(manifest.commits, commits)
|
| 48 | self.assertGreater(manifest.merge_commits, 0)
|
| 49 | self.assertTrue(0.05 <= manifest.merge_fraction <= 0.10, manifest.merge_fraction)
|
| 50 |
|
| 51 | def test_as_dict_satisfies_row_contract_types(self) -> None:
|
| 52 | _stream, manifest = fixture.build_stream(50, 3)
|
| 53 | fields = manifest.as_dict()
|
| 54 | for key in ("fixture_id", "fixture_version", "fixture_seed", "fixture_generator"):
|
| 55 | self.assertIsInstance(fields[key], str)
|
| 56 | self.assertIsNone(fields["fixture_git_head"]) # unobserved until imported
|
| 57 |
|
| 58 | @unittest.skipUnless(GIT, "git required")
|
| 59 | def test_generated_history_matches_golden_head_and_model_tree(self) -> None:
|
| 60 | with tempfile.TemporaryDirectory() as tmp:
|
| 61 | manifest = fixture.generate(Path(tmp) / "src.git", 1000, 1, git_bin=GIT, env=git_env())
|
| 62 | # Golden identity: changing this is a FIXTURE_SCHEMA bump (ADR-0005).
|
| 63 | self.assertEqual(manifest.head, "ed3b72471b73274cdf80f542e7b5835faf6badc2")
|
| 64 | self.assertEqual(manifest.merge_commits, 66)
|
| 65 | checkout = fixture.tree_content_hash(fixture.worktree_files(Path(tmp) / "src.git"))
|
| 66 | self.assertEqual(checkout, manifest.tree_content_hash)
|
| 67 | merges = lane.subprocess.run(
|
| 68 | [GIT, "rev-list", "--merges", "--count", "main"], cwd=Path(tmp) / "src.git",
|
| 69 | env=git_env(), text=True, stdout=lane.subprocess.PIPE, check=True,
|
| 70 | ).stdout.strip()
|
| 71 | self.assertEqual(int(merges), 66)
|
| 72 |
|
| 73 |
|
| 74 | REQUESTS = (
|
| 75 | b"POST /api/o/r/commits/info HTTP/1.1\r\nContent-Length: 5\r\n\r\nhello"
|
| 76 | b"GET /api/o/r?x=1 HTTP/1.1\r\nHost: h\r\n\r\n"
|
| 77 | b"POST /api/o/r/push HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n0\r\n\r\n"
|
| 78 | )
|
| 79 |
|
| 80 |
|
| 81 | class HttpRequestStreamParserTest(unittest.TestCase):
|
| 82 | def test_pipelined_content_length_and_chunked_requests(self) -> None:
|
| 83 | parser = HttpRequestStreamParser()
|
| 84 | self.assertEqual(parser.feed(REQUESTS), 3)
|
| 85 | self.assertEqual(
|
| 86 | [(item.method, item.path) for item in parser.requests],
|
| 87 | [("POST", "/api/o/r/commits/info"), ("GET", "/api/o/r"), ("POST", "/api/o/r/push")],
|
| 88 | )
|
| 89 | self.assertTrue(parser.at_request_boundary)
|
| 90 |
|
| 91 | def test_byte_at_a_time_framing_is_identical(self) -> None:
|
| 92 | parser = HttpRequestStreamParser()
|
| 93 | started = sum(parser.feed(REQUESTS[index : index + 1]) for index in range(len(REQUESTS)))
|
| 94 | self.assertEqual(started, 3)
|
| 95 | self.assertEqual(len(parser.requests), 3)
|
| 96 |
|
| 97 | def test_body_bytes_resembling_a_request_are_not_counted(self) -> None:
|
| 98 | body = b"GET /fake HTTP/1.1\r\n\r\n"
|
| 99 | parser = HttpRequestStreamParser()
|
| 100 | parser.feed(b"POST /x HTTP/1.1\r\nContent-Length: %d\r\n\r\n" % len(body) + body)
|
| 101 | self.assertEqual([item.path for item in parser.requests], ["/x"])
|
| 102 |
|
| 103 | def test_non_http_stream_is_rejected(self) -> None:
|
| 104 | with self.assertRaises(HttpFramingError):
|
| 105 | HttpRequestStreamParser().feed(b"\x16\x03\x01garbage\r\n\r\n")
|
| 106 |
|
| 107 |
|
| 108 | class _Handler(BaseHTTPRequestHandler):
|
| 109 | protocol_version = "HTTP/1.1"
|
| 110 |
|
| 111 | def _reply(self) -> None:
|
| 112 | length = int(self.headers.get("Content-Length") or 0)
|
| 113 | if length:
|
| 114 | self.rfile.read(length)
|
| 115 | body = b"ok"
|
| 116 | self.send_response(200)
|
| 117 | self.send_header("Content-Length", str(len(body)))
|
| 118 | self.end_headers()
|
| 119 | self.wfile.write(body)
|
| 120 |
|
| 121 | do_GET = _reply
|
| 122 | do_POST = _reply
|
| 123 |
|
| 124 | def log_message(self, format: str, *args) -> None: # noqa: A002
|
| 125 | pass
|
| 126 |
|
| 127 |
|
| 128 | class RequestCountingProxyTest(unittest.TestCase):
|
| 129 | def setUp(self) -> None:
|
| 130 | self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
| 131 | self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
|
| 132 | self.thread.start()
|
| 133 |
|
| 134 | def tearDown(self) -> None:
|
| 135 | self.httpd.shutdown()
|
| 136 | self.httpd.server_close()
|
| 137 |
|
| 138 | def _proxy(self, **kwargs) -> RequestCountingProxy: # type: ignore[no-untyped-def]
|
| 139 | proxy = RequestCountingProxy(
|
| 140 | "127.0.0.1", self.httpd.server_address[1], path_suffixes=("/commits/info",), **kwargs
|
| 141 | ).start()
|
| 142 | self.addCleanup(proxy.stop)
|
| 143 | return proxy
|
| 144 |
|
| 145 | def _wait_for(self, proxy: RequestCountingProxy, count: int) -> None:
|
| 146 | deadline = time.monotonic() + 2.0
|
| 147 | while time.monotonic() < deadline and (proxy.request_stats().request_count or 0) < count:
|
| 148 | time.sleep(0.01)
|
| 149 |
|
| 150 | def test_counts_keepalive_requests_per_path_suffix(self) -> None:
|
| 151 | proxy = self._proxy()
|
| 152 | connection = http.client.HTTPConnection("127.0.0.1", proxy.address[1], timeout=5)
|
| 153 | for path in ("/api/o/r/commits/info", "/api/o/r/commits/info", "/api/o/r"):
|
| 154 | connection.request("POST", path, body=b"{}")
|
| 155 | self.assertEqual(connection.getresponse().read(), b"ok")
|
| 156 | connection.close()
|
| 157 | self._wait_for(proxy, 3)
|
| 158 | stats = proxy.request_stats()
|
| 159 | self.assertEqual(stats.request_count, 3)
|
| 160 | self.assertEqual(stats.count_suffix("/commits/info"), 2)
|
| 161 | self.assertEqual(stats.methods, {"POST": 3})
|
| 162 | self.assertEqual(proxy.snapshot().connections, 1)
|
| 163 | proxy.reset_counts()
|
| 164 | self.assertEqual(proxy.request_stats().request_count, 0) # measured zero after reset
|
| 165 |
|
| 166 | def test_unframable_stream_reports_null_not_zero(self) -> None:
|
| 167 | proxy = self._proxy()
|
| 168 | with socket.create_connection(proxy.address, timeout=5) as sock:
|
| 169 | sock.sendall(b"\x16\x03\x01not http\r\n\r\n")
|
| 170 | sock.shutdown(socket.SHUT_WR)
|
| 171 | sock.settimeout(2)
|
| 172 | try:
|
| 173 | sock.recv(1024)
|
| 174 | except OSError:
|
| 175 | pass
|
| 176 | deadline = time.monotonic() + 2.0
|
| 177 | while time.monotonic() < deadline and proxy.request_stats().framing_errors == 0:
|
| 178 | time.sleep(0.01)
|
| 179 | stats = proxy.request_stats()
|
| 180 | self.assertIsNone(stats.request_count)
|
| 181 | self.assertIsNone(stats.count_suffix("/commits/info"))
|
| 182 | fields = shallow_refresh.request_metric_fields(stats, proxy.snapshot(), "/commits/info")
|
| 183 | self.assertIsNone(fields["http_request_count"])
|
| 184 | self.assertIsNone(fields["http_commits_info_request_count"])
|
| 185 |
|
| 186 | def test_injected_delay_is_applied_per_request_and_labeled(self) -> None:
|
| 187 | proxy = self._proxy(injected_request_delay_ms=50)
|
| 188 | connection = http.client.HTTPConnection("127.0.0.1", proxy.address[1], timeout=5)
|
| 189 | started = time.perf_counter()
|
| 190 | for _ in range(3):
|
| 191 | connection.request("GET", "/x")
|
| 192 | connection.getresponse().read()
|
| 193 | elapsed = time.perf_counter() - started
|
| 194 | connection.close()
|
| 195 | self.assertGreaterEqual(elapsed, 0.15)
|
| 196 | self.assertEqual(proxy.request_stats().injected_request_delay_ms, 50.0)
|
| 197 |
|
| 198 | def test_negative_delay_rejected(self) -> None:
|
| 199 | with self.assertRaises(ValueError):
|
| 200 | RequestCountingProxy("127.0.0.1", 1, injected_request_delay_ms=-1)
|
| 201 |
|
| 202 |
|
| 203 | class OracleTest(unittest.TestCase):
|
| 204 | HEAD = "a" * 64
|
| 205 | TREE = "t" * 64
|
| 206 |
|
| 207 | def evaluate(self, **overrides): # type: ignore[no-untyped-def]
|
| 208 | observed = {
|
| 209 | "observed_branch": "task",
|
| 210 | "observed_branch_head": self.HEAD,
|
| 211 | "observed_head": self.HEAD,
|
| 212 | "worktree_tree_hash": self.TREE,
|
| 213 | }
|
| 214 | observed.update(overrides)
|
| 215 | return shallow_refresh.evaluate_oracle(
|
| 216 | expected_branch="task", expected_head=self.HEAD, expected_tree_hash=self.TREE, **observed
|
| 217 | )
|
| 218 |
|
| 219 | def test_accepts_exact_final_state(self) -> None:
|
| 220 | self.assertTrue(self.evaluate()["oracle_passed"])
|
| 221 |
|
| 222 | def test_rejects_wrong_head(self) -> None:
|
| 223 | result = self.evaluate(observed_branch_head="b" * 64, observed_head="b" * 64)
|
| 224 | self.assertFalse(result["oracle_passed"])
|
| 225 | self.assertFalse(result["oracle_branch_head_is_server_main"])
|
| 226 |
|
| 227 | def test_rejects_wrong_tree(self) -> None:
|
| 228 | result = self.evaluate(worktree_tree_hash="x" * 64)
|
| 229 | self.assertFalse(result["oracle_passed"])
|
| 230 | self.assertFalse(result["oracle_worktree_tree_matches_fixture"])
|
| 231 |
|
| 232 | def test_rejects_wrong_branch(self) -> None:
|
| 233 | self.assertFalse(self.evaluate(observed_branch="main")["oracle_passed"])
|
| 234 |
|
| 235 | def test_unobserved_server_head_never_passes(self) -> None:
|
| 236 | result = shallow_refresh.evaluate_oracle(
|
| 237 | expected_branch="task", expected_head=None, expected_tree_hash=self.TREE,
|
| 238 | observed_branch="task", observed_branch_head=None, observed_head=None,
|
| 239 | worktree_tree_hash=self.TREE,
|
| 240 | )
|
| 241 | self.assertFalse(result["oracle_passed"])
|
| 242 |
|
| 243 | def test_worktree_tree_hash_detects_content_change(self) -> None:
|
| 244 | with tempfile.TemporaryDirectory() as tmp:
|
| 245 | root = Path(tmp)
|
| 246 | (root / "a.txt").write_text("one\n")
|
| 247 | (root / ".oak").mkdir()
|
| 248 | (root / ".oak" / "oak.db").write_text("ignored")
|
| 249 | before = shallow_refresh.worktree_tree_hash(root)
|
| 250 | (root / "a.txt").write_text("two\n")
|
| 251 | self.assertNotEqual(before, shallow_refresh.worktree_tree_hash(root))
|
| 252 | self.assertIsNone(shallow_refresh.worktree_tree_hash(Path(tmp) / "missing"))
|
| 253 |
|
| 254 |
|
| 255 | class OakDiskStateTest(unittest.TestCase):
|
| 256 | def test_missing_database_is_unobserved_not_zero(self) -> None:
|
| 257 | with tempfile.TemporaryDirectory() as tmp:
|
| 258 | state = shallow_refresh.oak_disk_state(Path(tmp), "task")
|
| 259 | self.assertEqual(state["error"], "oak_db_missing")
|
| 260 | self.assertIsNone(state["commit_rows"])
|
| 261 | self.assertIsNone(state["ancestry_missing_edges"])
|
| 262 | self.assertIsNone(shallow_refresh.serve_depth_behavior(state["commit_rows"]))
|
| 263 |
|
| 264 | def test_reads_branch_head_and_ancestry_gaps(self) -> None:
|
| 265 | with tempfile.TemporaryDirectory() as tmp:
|
| 266 | db_dir = Path(tmp) / ".oak"
|
| 267 | db_dir.mkdir()
|
| 268 | connection = sqlite3.connect(str(db_dir / "oak.db"))
|
| 269 | connection.executescript(
|
| 270 | "CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT);"
|
| 271 | "CREATE TABLE branch_heads (branch_name TEXT PRIMARY KEY, head_hash TEXT);"
|
| 272 | "CREATE TABLE commits (hash TEXT PRIMARY KEY, parent_hash TEXT, merge_parent_hash TEXT);"
|
| 273 | "INSERT INTO metadata VALUES ('current_branch','task'),('head','h3');"
|
| 274 | "INSERT INTO branch_heads VALUES ('task','h3');"
|
| 275 | "INSERT INTO commits VALUES ('h3','h2','m1'),('h2','h1',NULL);"
|
| 276 | )
|
| 277 | connection.commit()
|
| 278 | connection.close()
|
| 279 | state = shallow_refresh.oak_disk_state(Path(tmp), "task")
|
| 280 | self.assertEqual((state["current_branch"], state["head"], state["branch_head"]), ("task", "h3", "h3"))
|
| 281 | self.assertEqual(state["commit_rows"], 2)
|
| 282 | self.assertEqual(state["ancestry_missing_edges"], 2) # h1 and m1 absent
|
| 283 |
|
| 284 | def test_serve_depth_classification(self) -> None:
|
| 285 | self.assertEqual(shallow_refresh.serve_depth_behavior(1), shallow_refresh.SERVE_DEPTH_HONORED)
|
| 286 | self.assertEqual(shallow_refresh.serve_depth_behavior(1000), shallow_refresh.SERVE_DEPTH_IGNORED)
|
| 287 |
|
| 288 |
|
| 289 | class ShallowRefreshLaneTest(unittest.TestCase):
|
| 290 | def test_fixture_spec_parsing(self) -> None:
|
| 291 | self.assertEqual(lane.parse_shallow_refresh_fixtures("1000:1, 2000:2"), [(1000, 1), (2000, 2)])
|
| 292 | for bad in ("", "1000", "x:1", "0:1"):
|
| 293 | with self.assertRaises(ValueError):
|
| 294 | lane.parse_shallow_refresh_fixtures(bad)
|
| 295 |
|
| 296 | def test_summed_request_counts_stay_null_when_any_step_unobserved(self) -> None:
|
| 297 | self.assertIsNone(lane._summed_or_none([{"k": 1}, {"k": None}], "k"))
|
| 298 | self.assertEqual(lane._summed_or_none([{"k": 1}, {"k": 0}], "k"), 1)
|
| 299 |
|
| 300 | def test_missing_serve_binary_emits_skip_rows(self) -> None:
|
| 301 | subjects = [Subject("git", "git", "Git", Path("/usr/bin/git"))]
|
| 302 | with tempfile.TemporaryDirectory() as tmp:
|
| 303 | rows = lane.run_shallow_refresh_campaign(
|
| 304 | subjects, None, None, [(10, 1)], Path(tmp), 1, {"bench_id": "t"}, 1000, False, 1, 0.0
|
| 305 | )
|
| 306 | self.assertEqual(len(rows), 1)
|
| 307 | self.assertEqual(rows[0]["returncode"], lane.SKIP_RETURNCODE)
|
| 308 | self.assertEqual(rows[0]["scenario"], lane.SHALLOW_REFRESH_SCENARIO)
|
| 309 | self.assertIn("--oak-serve-bin", rows[0]["skip_reason"])
|
| 310 |
|
| 311 | def test_oak_cells_route_refresh_command_and_repeat_row(self) -> None:
|
| 312 | from unittest.mock import patch
|
| 313 |
|
| 314 | calls: list[tuple[str, list[str]]] = []
|
| 315 |
|
| 316 | def fake_step(subject, proxy, operation, command, *args, **kwargs): # type: ignore[no-untyped-def]
|
| 317 | calls.append((operation, command))
|
| 318 | return {"operation": operation, "command": command, "returncode": 0, "elapsed_ms": 1.0,
|
| 319 | "http_request_count": 1, "http_commits_info_request_count": 0,
|
| 320 | "network_wire_bytes_observed": 10, "network_connections_observed": 1}
|
| 321 |
|
| 322 | def fake_disk(root, branch): # type: ignore[no-untyped-def]
|
| 323 | return {"current_branch": branch, "head": "h", "branch_head": "h", "commit_rows": 1,
|
| 324 | "ancestry_missing_edges": 0, "error": None}
|
| 325 |
|
| 326 | subject = Subject("oak_x", "oak", "Oak", Path("/bin/oak"))
|
| 327 | with tempfile.TemporaryDirectory() as tmp, patch.object(lane, "_shallow_refresh_step", fake_step), \
|
| 328 | patch.object(lane.shallow_refresh, "oak_disk_state", fake_disk), \
|
| 329 | patch.object(lane.shallow_refresh, "worktree_tree_hash", return_value="T"):
|
| 330 | for cell in ("shallow", "shallow_pull"):
|
| 331 | rows = lane._shallow_refresh_oak_trial(
|
| 332 | subject, None, "http://x", "qa/r", "h", "T", Path(tmp) / cell, 0,
|
| 333 | {"bench_id": "t"}, 1000, {}, cell=cell,
|
| 334 | )
|
| 335 | operations = [row["operation"] for row in rows]
|
| 336 | self.assertIn(f"{cell}.total", operations)
|
| 337 | self.assertEqual(
|
| 338 | f"{cell}.branch_create_repeat_local" in operations, cell == "shallow"
|
| 339 | )
|
| 340 | refresh = dict(calls)
|
| 341 | self.assertEqual(refresh["shallow.parent_refresh"][1:], ["fetch"])
|
| 342 | self.assertEqual(refresh["shallow_pull.parent_refresh"][1:], ["pull"])
|
| 343 | self.assertEqual(refresh["shallow_pull.branch_create"][1:], ["switch", "-c", "task"])
|
| 344 |
|
| 345 |
|
| 346 | if __name__ == "__main__":
|
| 347 | unittest.main()
|