Viewing files at commit 085949193d78Commit detailsView latest

Browse files
tests/test_shallow_refresh.py 347 lines · 15.7 KB · python
1
"""Tests for shallow_worker_branch_refresh_v1 instruments: fixture, request counter, oracle."""
2
3
from __future__ import annotations
4
5
import os
6
import shutil
7
import socket
8
import sqlite3
9
import sys
10
import tempfile
11
import threading
12
import time
13
import unittest
14
import http.client
15
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
16
from pathlib import Path
17
18
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
19
20
import task_loop as lane
21
from oakbench import merge_history_fixture as fixture
22
from oakbench import shallow_refresh
23
from oakbench.httpcount import HttpFramingError, HttpRequestStreamParser, RequestCountingProxy
24
from oakbench.subjects import Subject
25
26
GIT = shutil.which("git")
27
28
29
def git_env() -> dict[str, str]:
30
    env = {key: value for key, value in os.environ.items() if key in {"PATH", "HOME", "TMPDIR"}}
31
    env.update({"GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"})
32
    return env
33
34
35
class MergeHistoryFixtureTest(unittest.TestCase):
36
    def test_stream_is_deterministic_and_seed_sensitive(self) -> None:
37
        first, manifest_a = fixture.build_stream(300, 7)
38
        second, manifest_b = fixture.build_stream(300, 7)
39
        other, _ = fixture.build_stream(300, 8)
40
        self.assertEqual(first, second)
41
        self.assertEqual(manifest_a.tree_content_hash, manifest_b.tree_content_hash)
42
        self.assertNotEqual(first, other)
43
44
    def test_exact_commit_count_and_merge_ratio(self) -> None:
45
        for commits, seed in ((1000, 1), (2000, 2)):
46
            _stream, manifest = fixture.build_stream(commits, seed)
47
            self.assertEqual(manifest.commits, commits)
48
            self.assertGreater(manifest.merge_commits, 0)
49
            self.assertTrue(0.05 <= manifest.merge_fraction <= 0.10, manifest.merge_fraction)
50
51
    def test_as_dict_satisfies_row_contract_types(self) -> None:
52
        _stream, manifest = fixture.build_stream(50, 3)
53
        fields = manifest.as_dict()
54
        for key in ("fixture_id", "fixture_version", "fixture_seed", "fixture_generator"):
55
            self.assertIsInstance(fields[key], str)
56
        self.assertIsNone(fields["fixture_git_head"])  # unobserved until imported
57
58
    @unittest.skipUnless(GIT, "git required")
59
    def test_generated_history_matches_golden_head_and_model_tree(self) -> None:
60
        with tempfile.TemporaryDirectory() as tmp:
61
            manifest = fixture.generate(Path(tmp) / "src.git", 1000, 1, git_bin=GIT, env=git_env())
62
            # Golden identity: changing this is a FIXTURE_SCHEMA bump (ADR-0005).
63
            self.assertEqual(manifest.head, "ed3b72471b73274cdf80f542e7b5835faf6badc2")
64
            self.assertEqual(manifest.merge_commits, 66)
65
            checkout = fixture.tree_content_hash(fixture.worktree_files(Path(tmp) / "src.git"))
66
            self.assertEqual(checkout, manifest.tree_content_hash)
67
            merges = lane.subprocess.run(
68
                [GIT, "rev-list", "--merges", "--count", "main"], cwd=Path(tmp) / "src.git",
69
                env=git_env(), text=True, stdout=lane.subprocess.PIPE, check=True,
70
            ).stdout.strip()
71
            self.assertEqual(int(merges), 66)
72
73
74
REQUESTS = (
75
    b"POST /api/o/r/commits/info HTTP/1.1\r\nContent-Length: 5\r\n\r\nhello"
76
    b"GET /api/o/r?x=1 HTTP/1.1\r\nHost: h\r\n\r\n"
77
    b"POST /api/o/r/push HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n0\r\n\r\n"
78
)
79
80
81
class HttpRequestStreamParserTest(unittest.TestCase):
82
    def test_pipelined_content_length_and_chunked_requests(self) -> None:
83
        parser = HttpRequestStreamParser()
84
        self.assertEqual(parser.feed(REQUESTS), 3)
85
        self.assertEqual(
86
            [(item.method, item.path) for item in parser.requests],
87
            [("POST", "/api/o/r/commits/info"), ("GET", "/api/o/r"), ("POST", "/api/o/r/push")],
88
        )
89
        self.assertTrue(parser.at_request_boundary)
90
91
    def test_byte_at_a_time_framing_is_identical(self) -> None:
92
        parser = HttpRequestStreamParser()
93
        started = sum(parser.feed(REQUESTS[index : index + 1]) for index in range(len(REQUESTS)))
94
        self.assertEqual(started, 3)
95
        self.assertEqual(len(parser.requests), 3)
96
97
    def test_body_bytes_resembling_a_request_are_not_counted(self) -> None:
98
        body = b"GET /fake HTTP/1.1\r\n\r\n"
99
        parser = HttpRequestStreamParser()
100
        parser.feed(b"POST /x HTTP/1.1\r\nContent-Length: %d\r\n\r\n" % len(body) + body)
101
        self.assertEqual([item.path for item in parser.requests], ["/x"])
102
103
    def test_non_http_stream_is_rejected(self) -> None:
104
        with self.assertRaises(HttpFramingError):
105
            HttpRequestStreamParser().feed(b"\x16\x03\x01garbage\r\n\r\n")
106
107
108
class _Handler(BaseHTTPRequestHandler):
109
    protocol_version = "HTTP/1.1"
110
111
    def _reply(self) -> None:
112
        length = int(self.headers.get("Content-Length") or 0)
113
        if length:
114
            self.rfile.read(length)
115
        body = b"ok"
116
        self.send_response(200)
117
        self.send_header("Content-Length", str(len(body)))
118
        self.end_headers()
119
        self.wfile.write(body)
120
121
    do_GET = _reply
122
    do_POST = _reply
123
124
    def log_message(self, format: str, *args) -> None:  # noqa: A002
125
        pass
126
127
128
class RequestCountingProxyTest(unittest.TestCase):
129
    def setUp(self) -> None:
130
        self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
131
        self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
132
        self.thread.start()
133
134
    def tearDown(self) -> None:
135
        self.httpd.shutdown()
136
        self.httpd.server_close()
137
138
    def _proxy(self, **kwargs) -> RequestCountingProxy:  # type: ignore[no-untyped-def]
139
        proxy = RequestCountingProxy(
140
            "127.0.0.1", self.httpd.server_address[1], path_suffixes=("/commits/info",), **kwargs
141
        ).start()
142
        self.addCleanup(proxy.stop)
143
        return proxy
144
145
    def _wait_for(self, proxy: RequestCountingProxy, count: int) -> None:
146
        deadline = time.monotonic() + 2.0
147
        while time.monotonic() < deadline and (proxy.request_stats().request_count or 0) < count:
148
            time.sleep(0.01)
149
150
    def test_counts_keepalive_requests_per_path_suffix(self) -> None:
151
        proxy = self._proxy()
152
        connection = http.client.HTTPConnection("127.0.0.1", proxy.address[1], timeout=5)
153
        for path in ("/api/o/r/commits/info", "/api/o/r/commits/info", "/api/o/r"):
154
            connection.request("POST", path, body=b"{}")
155
            self.assertEqual(connection.getresponse().read(), b"ok")
156
        connection.close()
157
        self._wait_for(proxy, 3)
158
        stats = proxy.request_stats()
159
        self.assertEqual(stats.request_count, 3)
160
        self.assertEqual(stats.count_suffix("/commits/info"), 2)
161
        self.assertEqual(stats.methods, {"POST": 3})
162
        self.assertEqual(proxy.snapshot().connections, 1)
163
        proxy.reset_counts()
164
        self.assertEqual(proxy.request_stats().request_count, 0)  # measured zero after reset
165
166
    def test_unframable_stream_reports_null_not_zero(self) -> None:
167
        proxy = self._proxy()
168
        with socket.create_connection(proxy.address, timeout=5) as sock:
169
            sock.sendall(b"\x16\x03\x01not http\r\n\r\n")
170
            sock.shutdown(socket.SHUT_WR)
171
            sock.settimeout(2)
172
            try:
173
                sock.recv(1024)
174
            except OSError:
175
                pass
176
        deadline = time.monotonic() + 2.0
177
        while time.monotonic() < deadline and proxy.request_stats().framing_errors == 0:
178
            time.sleep(0.01)
179
        stats = proxy.request_stats()
180
        self.assertIsNone(stats.request_count)
181
        self.assertIsNone(stats.count_suffix("/commits/info"))
182
        fields = shallow_refresh.request_metric_fields(stats, proxy.snapshot(), "/commits/info")
183
        self.assertIsNone(fields["http_request_count"])
184
        self.assertIsNone(fields["http_commits_info_request_count"])
185
186
    def test_injected_delay_is_applied_per_request_and_labeled(self) -> None:
187
        proxy = self._proxy(injected_request_delay_ms=50)
188
        connection = http.client.HTTPConnection("127.0.0.1", proxy.address[1], timeout=5)
189
        started = time.perf_counter()
190
        for _ in range(3):
191
            connection.request("GET", "/x")
192
            connection.getresponse().read()
193
        elapsed = time.perf_counter() - started
194
        connection.close()
195
        self.assertGreaterEqual(elapsed, 0.15)
196
        self.assertEqual(proxy.request_stats().injected_request_delay_ms, 50.0)
197
198
    def test_negative_delay_rejected(self) -> None:
199
        with self.assertRaises(ValueError):
200
            RequestCountingProxy("127.0.0.1", 1, injected_request_delay_ms=-1)
201
202
203
class OracleTest(unittest.TestCase):
204
    HEAD = "a" * 64
205
    TREE = "t" * 64
206
207
    def evaluate(self, **overrides):  # type: ignore[no-untyped-def]
208
        observed = {
209
            "observed_branch": "task",
210
            "observed_branch_head": self.HEAD,
211
            "observed_head": self.HEAD,
212
            "worktree_tree_hash": self.TREE,
213
        }
214
        observed.update(overrides)
215
        return shallow_refresh.evaluate_oracle(
216
            expected_branch="task", expected_head=self.HEAD, expected_tree_hash=self.TREE, **observed
217
        )
218
219
    def test_accepts_exact_final_state(self) -> None:
220
        self.assertTrue(self.evaluate()["oracle_passed"])
221
222
    def test_rejects_wrong_head(self) -> None:
223
        result = self.evaluate(observed_branch_head="b" * 64, observed_head="b" * 64)
224
        self.assertFalse(result["oracle_passed"])
225
        self.assertFalse(result["oracle_branch_head_is_server_main"])
226
227
    def test_rejects_wrong_tree(self) -> None:
228
        result = self.evaluate(worktree_tree_hash="x" * 64)
229
        self.assertFalse(result["oracle_passed"])
230
        self.assertFalse(result["oracle_worktree_tree_matches_fixture"])
231
232
    def test_rejects_wrong_branch(self) -> None:
233
        self.assertFalse(self.evaluate(observed_branch="main")["oracle_passed"])
234
235
    def test_unobserved_server_head_never_passes(self) -> None:
236
        result = shallow_refresh.evaluate_oracle(
237
            expected_branch="task", expected_head=None, expected_tree_hash=self.TREE,
238
            observed_branch="task", observed_branch_head=None, observed_head=None,
239
            worktree_tree_hash=self.TREE,
240
        )
241
        self.assertFalse(result["oracle_passed"])
242
243
    def test_worktree_tree_hash_detects_content_change(self) -> None:
244
        with tempfile.TemporaryDirectory() as tmp:
245
            root = Path(tmp)
246
            (root / "a.txt").write_text("one\n")
247
            (root / ".oak").mkdir()
248
            (root / ".oak" / "oak.db").write_text("ignored")
249
            before = shallow_refresh.worktree_tree_hash(root)
250
            (root / "a.txt").write_text("two\n")
251
            self.assertNotEqual(before, shallow_refresh.worktree_tree_hash(root))
252
        self.assertIsNone(shallow_refresh.worktree_tree_hash(Path(tmp) / "missing"))
253
254
255
class OakDiskStateTest(unittest.TestCase):
256
    def test_missing_database_is_unobserved_not_zero(self) -> None:
257
        with tempfile.TemporaryDirectory() as tmp:
258
            state = shallow_refresh.oak_disk_state(Path(tmp), "task")
259
        self.assertEqual(state["error"], "oak_db_missing")
260
        self.assertIsNone(state["commit_rows"])
261
        self.assertIsNone(state["ancestry_missing_edges"])
262
        self.assertIsNone(shallow_refresh.serve_depth_behavior(state["commit_rows"]))
263
264
    def test_reads_branch_head_and_ancestry_gaps(self) -> None:
265
        with tempfile.TemporaryDirectory() as tmp:
266
            db_dir = Path(tmp) / ".oak"
267
            db_dir.mkdir()
268
            connection = sqlite3.connect(str(db_dir / "oak.db"))
269
            connection.executescript(
270
                "CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT);"
271
                "CREATE TABLE branch_heads (branch_name TEXT PRIMARY KEY, head_hash TEXT);"
272
                "CREATE TABLE commits (hash TEXT PRIMARY KEY, parent_hash TEXT, merge_parent_hash TEXT);"
273
                "INSERT INTO metadata VALUES ('current_branch','task'),('head','h3');"
274
                "INSERT INTO branch_heads VALUES ('task','h3');"
275
                "INSERT INTO commits VALUES ('h3','h2','m1'),('h2','h1',NULL);"
276
            )
277
            connection.commit()
278
            connection.close()
279
            state = shallow_refresh.oak_disk_state(Path(tmp), "task")
280
        self.assertEqual((state["current_branch"], state["head"], state["branch_head"]), ("task", "h3", "h3"))
281
        self.assertEqual(state["commit_rows"], 2)
282
        self.assertEqual(state["ancestry_missing_edges"], 2)  # h1 and m1 absent
283
284
    def test_serve_depth_classification(self) -> None:
285
        self.assertEqual(shallow_refresh.serve_depth_behavior(1), shallow_refresh.SERVE_DEPTH_HONORED)
286
        self.assertEqual(shallow_refresh.serve_depth_behavior(1000), shallow_refresh.SERVE_DEPTH_IGNORED)
287
288
289
class ShallowRefreshLaneTest(unittest.TestCase):
290
    def test_fixture_spec_parsing(self) -> None:
291
        self.assertEqual(lane.parse_shallow_refresh_fixtures("1000:1, 2000:2"), [(1000, 1), (2000, 2)])
292
        for bad in ("", "1000", "x:1", "0:1"):
293
            with self.assertRaises(ValueError):
294
                lane.parse_shallow_refresh_fixtures(bad)
295
296
    def test_summed_request_counts_stay_null_when_any_step_unobserved(self) -> None:
297
        self.assertIsNone(lane._summed_or_none([{"k": 1}, {"k": None}], "k"))
298
        self.assertEqual(lane._summed_or_none([{"k": 1}, {"k": 0}], "k"), 1)
299
300
    def test_missing_serve_binary_emits_skip_rows(self) -> None:
301
        subjects = [Subject("git", "git", "Git", Path("/usr/bin/git"))]
302
        with tempfile.TemporaryDirectory() as tmp:
303
            rows = lane.run_shallow_refresh_campaign(
304
                subjects, None, None, [(10, 1)], Path(tmp), 1, {"bench_id": "t"}, 1000, False, 1, 0.0
305
            )
306
        self.assertEqual(len(rows), 1)
307
        self.assertEqual(rows[0]["returncode"], lane.SKIP_RETURNCODE)
308
        self.assertEqual(rows[0]["scenario"], lane.SHALLOW_REFRESH_SCENARIO)
309
        self.assertIn("--oak-serve-bin", rows[0]["skip_reason"])
310
311
    def test_oak_cells_route_refresh_command_and_repeat_row(self) -> None:
312
        from unittest.mock import patch
313
314
        calls: list[tuple[str, list[str]]] = []
315
316
        def fake_step(subject, proxy, operation, command, *args, **kwargs):  # type: ignore[no-untyped-def]
317
            calls.append((operation, command))
318
            return {"operation": operation, "command": command, "returncode": 0, "elapsed_ms": 1.0,
319
                    "http_request_count": 1, "http_commits_info_request_count": 0,
320
                    "network_wire_bytes_observed": 10, "network_connections_observed": 1}
321
322
        def fake_disk(root, branch):  # type: ignore[no-untyped-def]
323
            return {"current_branch": branch, "head": "h", "branch_head": "h", "commit_rows": 1,
324
                    "ancestry_missing_edges": 0, "error": None}
325
326
        subject = Subject("oak_x", "oak", "Oak", Path("/bin/oak"))
327
        with tempfile.TemporaryDirectory() as tmp, patch.object(lane, "_shallow_refresh_step", fake_step), \
328
                patch.object(lane.shallow_refresh, "oak_disk_state", fake_disk), \
329
                patch.object(lane.shallow_refresh, "worktree_tree_hash", return_value="T"):
330
            for cell in ("shallow", "shallow_pull"):
331
                rows = lane._shallow_refresh_oak_trial(
332
                    subject, None, "http://x", "qa/r", "h", "T", Path(tmp) / cell, 0,
333
                    {"bench_id": "t"}, 1000, {}, cell=cell,
334
                )
335
                operations = [row["operation"] for row in rows]
336
                self.assertIn(f"{cell}.total", operations)
337
                self.assertEqual(
338
                    f"{cell}.branch_create_repeat_local" in operations, cell == "shallow"
339
                )
340
        refresh = dict(calls)
341
        self.assertEqual(refresh["shallow.parent_refresh"][1:], ["fetch"])
342
        self.assertEqual(refresh["shallow_pull.parent_refresh"][1:], ["pull"])
343
        self.assertEqual(refresh["shallow_pull.branch_create"][1:], ["switch", "-c", "task"])
344
345
346
if __name__ == "__main__":
347
    unittest.main()