Viewing files at commit 13de5e5480b3Commit detailsView latest

Browse files
tests/test_task_loop.py 508 lines · 20.4 KB · python
1
"""Tests for the task-loop lane's derived fields and lane registration."""
2
3
from __future__ import annotations
4
5
import sys
6
import os
7
import shutil
8
import socket
9
import tempfile
10
import time
11
import unittest
12
import http.client
13
from pathlib import Path
14
from unittest.mock import patch
15
16
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
17
18
import task_loop as lane
19
from oakbench.execution import TimedCapture
20
from oakbench.rows import LANE_REQUIRED_KEYS
21
from oakbench.subjects import Subject
22
from oakbench.remotes import RemoteResolution
23
24
25
class LoopSummaryFieldsTest(unittest.TestCase):
26
    def test_warm_delta_from_acquire_pair(self) -> None:
27
        rows = [
28
            {"operation": "task.acquire", "elapsed_ms": 1000.0},
29
            {"operation": "task.edit", "elapsed_ms": 5.0},
30
            {"operation": "followup.acquire", "elapsed_ms": 600.0},
31
            {"operation": "followup.edit", "elapsed_ms": 5.0},
32
        ]
33
        fields = lane.loop_summary_fields(rows)
34
        self.assertEqual(fields["followup_warm_delta_ms"], -400.0)
35
        self.assertEqual(fields["task_phase_ms"], 1005.0)
36
        self.assertEqual(fields["followup_phase_ms"], 605.0)
37
38
    def test_missing_acquire_yields_null_not_zero(self) -> None:
39
        fields = lane.loop_summary_fields([{"operation": "task.edit", "elapsed_ms": 5.0}])
40
        self.assertIsNone(fields["followup_warm_delta_ms"])
41
42
43
class LaneRegistrationTest(unittest.TestCase):
44
    def test_task_loop_lane_registered(self) -> None:
45
        self.assertIn("task-loop", LANE_REQUIRED_KEYS)
46
        self.assertIn("command", LANE_REQUIRED_KEYS["task-loop"])
47
48
49
class VirtualBranchReTest(unittest.TestCase):
50
    def test_parses_mount_status(self) -> None:
51
        text = "Mount: oak/bench-sync-tmp@abc (virtual branch tl-x-r0-task--9f00aa11)"
52
        match = lane.VIRTUAL_BRANCH_RE.search(text)
53
        self.assertEqual(match.group(1), "tl-x-r0-task--9f00aa11")
54
55
56
class FailurePolicyTest(unittest.TestCase):
57
    def test_step_row_never_treats_a_process_timeout_as_an_expected_error(self) -> None:
58
        subject = Subject("oak", "oak", "Oak", Path("/usr/bin/oak"))
59
        capture = TimedCapture(
60
            elapsed_ms=100.0,
61
            returncode=124,
62
            stdout_text="",
63
            stdout_bytes=0,
64
            stdout_truncated=False,
65
            stderr_text="",
66
            stderr_bytes=0,
67
            stderr_truncated=False,
68
            timed_out=True,
69
        )
70
        with tempfile.TemporaryDirectory() as tmp, patch.object(
71
            lane, "run_timed", return_value=capture
72
        ) as timed:
73
            row = lane.step_row(
74
                subject,
75
                "interrupted_publication_after_request_v1",
76
                "publication.attempt",
77
                ["oak", "push"],
78
                Path(tmp),
79
                0,
80
                {"bench_id": "test"},
81
                100,
82
                expected_returncodes=tuple(range(1, 128)),
83
                timeout_seconds=0.1,
84
            )
85
86
        self.assertEqual(timed.call_args.kwargs["timeout_seconds"], 0.1)
87
        self.assertTrue(row["timed_out"])
88
        self.assertEqual(row["returncode"], 124)
89
90
    def test_step_row_marks_unexpected_zero_exit_as_failure(self) -> None:
91
        subject = Subject("git", "git", "Git", Path("/usr/bin/git"))
92
        capture = TimedCapture(
93
            elapsed_ms=1.0,
94
            returncode=0,
95
            stdout_text="",
96
            stdout_bytes=0,
97
            stdout_truncated=False,
98
            stderr_text="",
99
            stderr_bytes=0,
100
            stderr_truncated=False,
101
        )
102
        with tempfile.TemporaryDirectory() as tmp, patch.object(lane, "run_timed", return_value=capture):
103
            row = lane.step_row(
104
                subject,
105
                "task_loop",
106
                "expect.fail",
107
                ["sh", "-c", "true"],
108
                Path(tmp),
109
                0,
110
                {
111
                    "bench_id": "test",
112
                    "profile": "task-loop",
113
                    "timestamp_utc": "test",
114
                    "benchmark_track": "agent-default",
115
                },
116
                100,
117
                expected_returncodes=(1,),
118
            )
119
120
        self.assertEqual(row["process_returncode"], 0)
121
        self.assertEqual(row["returncode"], 1)
122
123
    def test_oracle_false_marks_total_failed(self) -> None:
124
        total = {
125
            **lane.phase_total_row(
126
                [{"operation": "task.acquire", "elapsed_ms": 1.0, "returncode": 0, "command": []}],
127
                Subject("oak", "oak", "Oak", Path("/usr/bin/oak")),
128
                "task_loop",
129
                0,
130
                {
131
                    "bench_id": "test",
132
                    "profile": "task-loop",
133
                    "timestamp_utc": "test",
134
                    "benchmark_track": "agent-default",
135
                },
136
                "task_loop.total",
137
            ),
138
            "oracle_payload_on_remote_branch": False,
139
        }
140
        lane.enforce_oracle_result(total, False)
141
        self.assertEqual(total["returncode"], 1)
142
        self.assertTrue(lane.has_lane_failures([total]))
143
144
    def test_lane_failures_ignore_skips_but_fail_on_real_errors(self) -> None:
145
        self.assertFalse(lane.has_lane_failures([{"returncode": lane.SKIP_RETURNCODE, "skipped": True}]))
146
        self.assertTrue(lane.has_lane_failures([{"returncode": 1}]))
147
148
149
class LocalGitSeedTest(unittest.TestCase):
150
    def test_seeded_local_remote_advertises_main_as_head(self) -> None:
151
        subject = Subject("git", "git", "Git", Path("/usr/bin/git"))
152
        with tempfile.TemporaryDirectory() as tmp:
153
            root = Path(tmp)
154
            remote_path = root / "remote.git"
155
            init = lane.run_untimed(
156
                [str(subject.bin), "init", "--bare", "-q", str(remote_path)], root
157
            )
158
            self.assertEqual(init.returncode, 0, init.stderr)
159
            remote = RemoteResolution(
160
                repo=str(remote_path),
161
                transport="local_file",
162
                server="local-bare",
163
                env_var="generated",
164
            )
165
166
            error = lane.ensure_git_remote_seeded(subject, remote, root)
167
168
            self.assertIsNone(error)
169
            advertised = lane.run_untimed(
170
                [str(subject.bin), "symbolic-ref", "HEAD"], remote_path
171
            )
172
            self.assertEqual(advertised.returncode, 0, advertised.stderr)
173
            self.assertEqual(advertised.stdout.strip(), "refs/heads/main")
174
175
176
class AgentTaskEvidenceTest(unittest.TestCase):
177
    @unittest.skipUnless(
178
        os.environ.get("OAK_BENCH_TEST_OAK_BIN"),
179
        "set OAK_BENCH_TEST_OAK_BIN to exercise the real local Oak server",
180
    )
181
    def test_local_oak_agent_task_uses_fresh_checkouts_and_remote_oracles(self) -> None:
182
        oak_bin = Path(os.environ["OAK_BENCH_TEST_OAK_BIN"])
183
        subject = Subject("oak_local", "oak", "Oak local", oak_bin)
184
        with tempfile.TemporaryDirectory() as tmp:
185
            rows = lane.run_oak_local_agent_task_baseline(
186
                subject,
187
                Path(tmp),
188
                0,
189
                {
190
                    "bench_id": "test",
191
                    "profile": "task-loop",
192
                    "timestamp_utc": "test",
193
                    "benchmark_track": "agent-default",
194
                },
195
                20_000,
196
            )
197
198
        total = next(row for row in rows if row["operation"] == "agent_task_oak_local.total")
199
        self.assertEqual(total["returncode"], 0)
200
        self.assertFalse(total["coverage_complete"])
201
        self.assertTrue(total["oracle_payload_on_remote_branch"])
202
        self.assertTrue(total["oracle_branch_head_matches"])
203
        self.assertTrue(total["server_process_stopped"])
204
        self.assertEqual(
205
            total["acquisition_models"],
206
            {"cold": "fresh_full_clone", "followup": "second_fresh_full_clone"},
207
        )
208
        self.assertEqual(total["repository_cache_state"]["followup"], "no_shared_object_store")
209
        self.assertIn("cold.ci.observe", total["unmeasured_operations"])
210
        self.assertIn("followup.review_handoff", total["unmeasured_operations"])
211
212
    def test_total_keeps_skips_unmeasured_and_aggregates_costs(self) -> None:
213
        subject = Subject("git", "git", "Git", Path("/usr/bin/git"))
214
        rows = [
215
            {
216
                "operation": "task.acquire",
217
                "elapsed_ms": 10.0,
218
                "returncode": 0,
219
                "command": ["git", "clone"],
220
                "tool_call_count": 1,
221
                "stdout_bytes": 7,
222
                "stderr_bytes": 0,
223
                "retry_count": 0,
224
                "human_action_count": 0,
225
            },
226
            {
227
                "operation": "task.ci.observe",
228
                "elapsed_ms": 0.0,
229
                "returncode": lane.SKIP_RETURNCODE,
230
                "command": [],
231
                "skipped": True,
232
                "skip_reason": "local transport has no hosted CI provider",
233
                "tool_call_count": 0,
234
                "stdout_bytes": None,
235
                "stderr_bytes": None,
236
                "retry_count": 0,
237
                "human_action_count": 0,
238
            },
239
        ]
240
241
        total = lane.phase_total_row(
242
            rows,
243
            subject,
244
            "agent_task_baseline_v1",
245
            0,
246
            {
247
                "bench_id": "test",
248
                "profile": "task-loop",
249
                "timestamp_utc": "test",
250
                "benchmark_track": "agent-default",
251
            },
252
            "agent_task.total",
253
        )
254
255
        self.assertEqual(total["returncode"], 0)
256
        self.assertFalse(total["coverage_complete"])
257
        self.assertEqual(total["unmeasured_operations"], ["task.ci.observe"])
258
        self.assertEqual(total["raw_output_bytes"], 7)
259
        self.assertEqual(total["retry_count"], 0)
260
        self.assertEqual(total["human_action_count"], 0)
261
        self.assertEqual(total["phase_elapsed_ms"], {"task": 10.0})
262
263
    def test_interrupted_publication_summary_never_claims_retry(self) -> None:
264
        fields = lane.interrupted_publication_fields(
265
            request_count=1,
266
            request_bytes=4096,
267
            declared_request_bytes=4096,
268
            request_framing_valid=True,
269
            request_body_timed_out=False,
270
            local_head_before="a" * 64,
271
            local_head_after="a" * 64,
272
            status_after={"needs_push": True, "dirty": False},
273
            worktree_payload_before="payload-hash",
274
            worktree_payload_after="payload-hash",
275
            cli_observation={"reported_outcome": "unknown", "observation": "json_error_unknown"},
276
        )
277
278
        self.assertEqual(fields["publication_outcome"], "unknown")
279
        self.assertEqual(fields["mutation_requests_observed"], 1)
280
        self.assertEqual(fields["blind_replay_requests"], 0)
281
        self.assertTrue(fields["local_head_preserved"])
282
        self.assertTrue(fields["local_work_preserved"])
283
        self.assertTrue(fields["worktree_payload_preserved"])
284
        self.assertEqual(fields["network_request_bytes_observed"], 4096)
285
286
    def test_interruption_separates_cli_report_from_fixture_delivery_knowledge(self) -> None:
287
        generic_error = {
288
            "stdout": '{"schema_version":1,"error":{"code":"http","message":"connection closed"}}',
289
            "stderr": "",
290
            "process_returncode": 6,
291
            "timed_out": False,
292
        }
293
        cli = lane.cli_publication_observation(generic_error, json_requested=True)
294
        self.assertEqual(cli["reported_outcome"], "unreported")
295
        self.assertEqual(cli["observation"], "json_error_without_publication_outcome")
296
297
        fields = lane.interrupted_publication_fields(
298
            request_count=1,
299
            request_bytes=3,
300
            declared_request_bytes=100,
301
            request_framing_valid=True,
302
            request_body_timed_out=False,
303
            local_head_before="a" * 64,
304
            local_head_after="a" * 64,
305
            status_after={"needs_push": True, "dirty": False},
306
            worktree_payload_before="payload",
307
            worktree_payload_after="payload",
308
            cli_observation=cli,
309
        )
310
311
        self.assertEqual(fields["publication_outcome"], "unreported")
312
        self.assertEqual(fields["cli_observation"], "json_error_without_publication_outcome")
313
        self.assertEqual(fields["fixture_delivery_state"], "partial_request_observed")
314
        self.assertEqual(
315
            fields["fixture_outcome_knowledge"],
316
            "indeterminate_after_partial_request_disconnect",
317
        )
318
319
    def test_interruption_requires_exact_payload_and_unpublished_state(self) -> None:
320
        fields = lane.interrupted_publication_fields(
321
            request_count=1,
322
            request_bytes=100,
323
            declared_request_bytes=100,
324
            request_framing_valid=True,
325
            request_body_timed_out=False,
326
            local_head_before="a" * 64,
327
            local_head_after="a" * 64,
328
            status_after={"needs_push": True, "dirty": False},
329
            worktree_payload_before="before-hash",
330
            worktree_payload_after="after-hash",
331
            cli_observation={"reported_outcome": "unknown", "observation": "json_error_unknown"},
332
        )
333
334
        self.assertTrue(fields["unpublished_state_preserved"])
335
        self.assertFalse(fields["worktree_payload_preserved"])
336
        self.assertFalse(fields["local_work_preserved"])
337
338
    def test_interruption_server_reads_one_complete_push_then_drops_reply(self) -> None:
339
        server, thread = lane.start_interrupted_publication_server()
340
        host, port = server.server_address[:2]
341
        try:
342
            connection = http.client.HTTPConnection(host, port, timeout=2)
343
            connection.request("POST", "/api/repos", body=b"{}", headers={"Content-Type": "application/json"})
344
            response = connection.getresponse()
345
            self.assertEqual(response.status, 200)
346
            response.read()
347
            connection.close()
348
349
            connection = http.client.HTTPConnection(host, port, timeout=2)
350
            connection.request(
351
                "POST",
352
                "/api/oak/repo/push",
353
                body=b"fixture-payload",
354
                headers={"Content-Type": "application/json"},
355
            )
356
            with self.assertRaises(http.client.RemoteDisconnected):
357
                connection.getresponse()
358
            connection.close()
359
        finally:
360
            server.shutdown()
361
            server.server_close()
362
            thread.join(timeout=2)
363
364
        self.assertEqual(server.push_request_count, 1)
365
        self.assertEqual(server.push_request_bytes, len(b"fixture-payload"))
366
        self.assertTrue(server.request_complete_before_disconnect)
367
368
    def test_interruption_server_never_certifies_a_truncated_request(self) -> None:
369
        server, thread = lane.start_interrupted_publication_server()
370
        host, port = server.server_address[:2]
371
        try:
372
            with socket.create_connection((host, port), timeout=2) as connection:
373
                connection.sendall(
374
                    b"POST /api/oak/repo/push HTTP/1.1\r\n"
375
                    b"Host: local\r\n"
376
                    b"Content-Length: 100\r\n"
377
                    b"Content-Type: application/json\r\n\r\n"
378
                    b"abc"
379
                )
380
                connection.shutdown(socket.SHUT_WR)
381
                connection.recv(1024)
382
        finally:
383
            server.shutdown()
384
            server.server_close()
385
            thread.join(timeout=2)
386
387
        self.assertEqual(server.push_request_count, 1)
388
        self.assertEqual(server.push_request_bytes, 3)
389
        self.assertEqual(server.push_declared_request_bytes, 100)
390
        self.assertTrue(server.push_request_framing_valid)
391
        self.assertFalse(server.request_complete_before_disconnect)
392
393
    def test_interruption_server_rejects_absent_and_invalid_framing(self) -> None:
394
        for framing in (
395
            b"",
396
            b"Content-Length: nope\r\n",
397
            b"Content-Length: +3\r\n",
398
            b"Content-Length: 3\r\nContent-Length: 3\r\n",
399
            f"Content-Length: {lane.INTERRUPTED_PUBLICATION_MAX_REQUEST_BYTES + 1}\r\n".encode(),
400
        ):
401
            with self.subTest(framing=framing):
402
                server, thread = lane.start_interrupted_publication_server()
403
                host, port = server.server_address[:2]
404
                try:
405
                    with socket.create_connection((host, port), timeout=2) as connection:
406
                        connection.sendall(
407
                            b"POST /api/oak/repo/push HTTP/1.1\r\n"
408
                            b"Host: local\r\n"
409
                            + framing
410
                            + b"Content-Type: application/json\r\n\r\nabc"
411
                        )
412
                        connection.shutdown(socket.SHUT_WR)
413
                        connection.recv(1024)
414
                finally:
415
                    server.shutdown()
416
                    server.server_close()
417
                    thread.join(timeout=2)
418
419
                self.assertEqual(server.push_request_count, 1)
420
                self.assertIsNone(server.push_declared_request_bytes)
421
                self.assertFalse(server.push_request_framing_valid)
422
                self.assertFalse(server.request_complete_before_disconnect)
423
424
    def test_interruption_server_accepts_http_ows_around_content_length(self) -> None:
425
        server, thread = lane.start_interrupted_publication_server()
426
        host, port = server.server_address[:2]
427
        try:
428
            with socket.create_connection((host, port), timeout=2) as connection:
429
                connection.sendall(
430
                    b"POST /api/oak/repo/push HTTP/1.1\r\n"
431
                    b"Host: local\r\n"
432
                    b"Content-Length:\t3 \t\r\n\r\nabc"
433
                )
434
                connection.shutdown(socket.SHUT_WR)
435
                connection.recv(1024)
436
        finally:
437
            server.shutdown()
438
            server.server_close()
439
            thread.join(timeout=2)
440
441
        self.assertEqual(server.push_declared_request_bytes, 3)
442
        self.assertTrue(server.push_request_framing_valid)
443
        self.assertTrue(server.request_complete_before_disconnect)
444
445
    def test_interruption_server_bounds_a_partial_request_while_sender_stays_open(self) -> None:
446
        server, thread = lane.start_interrupted_publication_server()
447
        host, port = server.server_address[:2]
448
        started = time.monotonic()
449
        try:
450
            with socket.create_connection((host, port), timeout=4) as connection:
451
                connection.sendall(
452
                    b"POST /api/oak/repo/push HTTP/1.1\r\n"
453
                    b"Host: local\r\n"
454
                    b"Content-Length: 100\r\n\r\nabc"
455
                )
456
                connection.recv(1024)
457
        finally:
458
            server.shutdown()
459
            server.server_close()
460
            thread.join(timeout=2)
461
462
        self.assertLess(time.monotonic() - started, 4)
463
        self.assertEqual(server.push_request_bytes, 3)
464
        self.assertTrue(server.push_request_framing_valid)
465
        self.assertTrue(server.push_request_body_timed_out)
466
        self.assertFalse(server.request_complete_before_disconnect)
467
468
    @unittest.skipIf(shutil.which("git") is None, "git not installed")
469
    def test_local_git_agent_task_records_cold_warm_and_unmeasured_ci(self) -> None:
470
        subject = Subject("git", "git", "Git", Path(shutil.which("git") or "git"))
471
        with tempfile.TemporaryDirectory() as tmp:
472
            root = Path(tmp)
473
            remote = lane.oakbench_remotes.make_git_bare_remote(
474
                subject.bin, root, "agent-task.git"
475
            )
476
            rows = lane.run_git_agent_task_baseline(
477
                subject,
478
                remote,
479
                root,
480
                0,
481
                {
482
                    "bench_id": "test",
483
                    "profile": "task-loop",
484
                    "timestamp_utc": "test",
485
                    "benchmark_track": "agent-default",
486
                },
487
                20_000,
488
            )
489
490
        operations = [row["operation"] for row in rows]
491
        self.assertIn("cold.acquire", operations)
492
        self.assertIn("warm.acquire", operations)
493
        self.assertIn("cold.ci.observe", operations)
494
        total = next(row for row in rows if row["operation"] == "agent_task.total")
495
        self.assertEqual(total["returncode"], 0)
496
        self.assertFalse(total["coverage_complete"])
497
        self.assertEqual(total["oracle_payload_on_remote_branch"], True)
498
        self.assertEqual(total["acquisition_models"], {
499
            "cold": "fresh_full_clone",
500
            "warm": "shared_object_worktree",
501
        })
502
        ci = next(row for row in rows if row["operation"] == "cold.ci.observe")
503
        self.assertEqual(ci["returncode"], lane.SKIP_RETURNCODE)
504
        self.assertIn("no hosted CI", ci["skip_reason"])
505
506
507
if __name__ == "__main__":
508
    unittest.main()