Log in
benchmarksPublic

Viewing files at commit e9b767a51662Commit detailsView latest

Browse files
tests/test_devloop.py 603 lines · 28.2 KB · python
1
"""Focused regression tests for devloop verdict gates."""
2
3
from __future__ import annotations
4
5
import importlib.util
6
import sys
7
import tempfile
8
import unittest
9
from pathlib import Path
10
from unittest import mock
11
12
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
13
14
15
def load_devloop():
16
    path = Path(__file__).resolve().parents[1] / "scripts" / "devloop.py"
17
    spec = importlib.util.spec_from_file_location("devloop", path)
18
    module = importlib.util.module_from_spec(spec)
19
    sys.modules["devloop"] = module
20
    spec.loader.exec_module(module)
21
    return module
22
23
24
def row(subject: str, operation: str, *, returncode: int = 0, elapsed_ms: float = 10.0) -> dict:
25
    return {
26
        "subject": subject,
27
        "scenario": "tiny_text",
28
        "operation": operation,
29
        "run": 0,
30
        "returncode": returncode,
31
        "elapsed_ms": elapsed_ms,
32
        "vcs_tool_call_count": 1,
33
        "tool_call_count": 1,
34
        "raw_output_bytes": 10,
35
        "estimated_tokens_total": 3,
36
    }
37
38
39
class DevloopVerdictTest(unittest.TestCase):
40
    def setUp(self) -> None:
41
        self.devloop = load_devloop()
42
43
    def compare(self, rows: list[dict]) -> list[str]:
44
        regressions, _ = self.devloop.compare_subjects(
45
            rows,
46
            "oak_local",
47
            "oak_installed",
48
            "Oak baseline",
49
            latency_threshold_pct=10.0,
50
            min_delta_ms=5.0,
51
            efficiency_gates=True,
52
            output_bytes_threshold_pct=25.0,
53
        )
54
        return regressions
55
56
    def test_all_failing_target_is_regression_even_without_target_latency_median(self) -> None:
57
        rows = [
58
            row("oak_installed", "status.dirty", returncode=0, elapsed_ms=10.0),
59
            row("oak_installed", "status.dirty", returncode=0, elapsed_ms=11.0),
60
            row("oak_local", "status.dirty", returncode=1, elapsed_ms=100.0),
61
            row("oak_local", "status.dirty", returncode=1, elapsed_ms=101.0),
62
        ]
63
64
        regressions = self.compare(rows)
65
66
        self.assertTrue(any("NO SUCCESS" in item and "status.dirty" in item for item in regressions))
67
68
    def test_baseline_flake_does_not_hide_higher_target_failure_rate(self) -> None:
69
        rows = [
70
            row("oak_installed", "status.dirty", returncode=0, elapsed_ms=10.0),
71
            row("oak_installed", "status.dirty", returncode=0, elapsed_ms=11.0),
72
            row("oak_installed", "status.dirty", returncode=1, elapsed_ms=12.0),
73
            row("oak_local", "status.dirty", returncode=0, elapsed_ms=10.0),
74
            row("oak_local", "status.dirty", returncode=1, elapsed_ms=12.0),
75
            row("oak_local", "status.dirty", returncode=1, elapsed_ms=13.0),
76
        ]
77
78
        regressions = self.compare(rows)
79
80
        self.assertTrue(any("FAIL-RATE" in item and "2/3" in item for item in regressions))
81
82
    def test_embedded_null_is_symmetric_like_aa_without_changing_directional_delta(self):
83
        for a, b in ((12, 150), (150, 12)):
84
            rows = [{**row("oak_installed", "read.file", elapsed_ms=a), "vcs_tool_call_count": 0},
85
                    {**row("oak_local", "read.file", elapsed_ms=b), "vcs_tool_call_count": 0}]
86
            self.assertEqual(self.devloop.embedded_null_p95(rows, "oak_local", "oak_installed"), (1150, 1))
87
        self.assertLess(self.devloop.slower_pct(150, 12), 0)
88
89
    def test_remote_transports_are_not_false_git_guardrail_gaps(self):
90
        rows = [{**row("git", "remote.clone.cold"), "subject_kind": "git", "remote_transport": "local_file"},
91
                {**row("oak_local", "remote.net.clone.cold"), "subject_kind": "oak", "remote_transport": "network"}]
92
        diagnostics = []
93
        failures, _ = self.devloop.compare_subjects(rows, "oak_local", "git", "Git", 100, 100,
94
            efficiency_gates=False, output_bytes_threshold_pct=100, diagnostics=diagnostics)
95
        self.assertFalse(failures)
96
        self.assertTrue(any("TRANSPORT" in text for text in diagnostics))
97
        rows[1]["operation"] = "status.clean"
98
        rows[0].update(subject="oak_installed", subject_kind="oak")
99
        self.assertTrue(self.compare(rows))
100
101
    def test_embedded_null_excludes_probe_and_failed_or_skipped_groups(self) -> None:
102
        rows = [
103
            {**row("oak_installed", "read.file", elapsed_ms=100.0), "vcs_tool_call_count": 0},
104
            {**row("oak_local", "read.file", elapsed_ms=100.0), "vcs_tool_call_count": 0},
105
            {**row("oak_installed", "diff.dirty.inforecall", elapsed_ms=100.0), "vcs_tool_call_count": 0},
106
            {**row("oak_local", "diff.dirty.inforecall", elapsed_ms=900.0), "vcs_tool_call_count": 0},
107
            {**row("oak_installed", "test.step", elapsed_ms=100.0), "vcs_tool_call_count": 0},
108
            {**row("oak_local", "test.step", elapsed_ms=900.0), "vcs_tool_call_count": 0},
109
            {**row("oak_local", "test.step", returncode=1, elapsed_ms=901.0), "vcs_tool_call_count": 0},
110
            {**row("oak_installed", "skipped.step", elapsed_ms=100.0), "vcs_tool_call_count": 0},
111
            {**row("oak_local", "skipped.step", elapsed_ms=900.0), "vcs_tool_call_count": 0},
112
            {
113
                **row("oak_local", "skipped.step", returncode=self.devloop.SKIP_RETURNCODE, elapsed_ms=0.0),
114
                "vcs_tool_call_count": 0,
115
                "skipped": True,
116
            },
117
        ]
118
119
        null_pct, null_ops = self.devloop.embedded_null_p95(rows, "oak_local", "oak_installed")
120
121
        self.assertEqual(null_pct, 0.0)
122
        self.assertEqual(null_ops, 1)
123
124
    def test_noise_floor_is_invariant_to_identical_subject_label_order(self) -> None:
125
        for left, right in ((150.0, 12.0), (12.0, 150.0)):
126
            with mock.patch.object(self.devloop, "run_lane"), mock.patch.object(self.devloop, "load_jsonl", return_value=[
127
                row("oak_installed", "status.clean", elapsed_ms=left),
128
                row("oak_local", "status.clean", elapsed_ms=right),
129
            ]):
130
                spread, operations = self.devloop.measure_noise_floor(Path("/tmp/aa"), Path("/bin/oak"), 2)
131
            self.assertAlmostEqual(spread, 1150.0)
132
            self.assertEqual(operations, 1)
133
134
    def test_noise_floor_equal_medians_have_zero_spread(self) -> None:
135
        with mock.patch.object(self.devloop, "run_lane"), mock.patch.object(self.devloop, "NOISE_FLOOR_MIN_PCT", 0.0), \
136
             mock.patch.object(self.devloop, "load_jsonl", return_value=[
137
                 row("oak_installed", "status.clean"), row("oak_local", "status.clean")]):
138
            self.assertEqual(self.devloop.measure_noise_floor(Path("/tmp/aa"), Path("/bin/oak"), 2), (0.0, 1))
139
140
    def test_noise_floor_excludes_nonpositive_and_missing_medians(self) -> None:
141
        for left, right in ((0, 12), (12, 0), (-1, 12), (12, -1), (12, None), (None, 12)):
142
            with mock.patch.object(self.devloop, "run_lane"), mock.patch.object(self.devloop, "load_jsonl", return_value=[
143
                row("oak_installed", "status.clean", elapsed_ms=left),
144
                row("oak_local", "status.clean", elapsed_ms=right),
145
            ]):
146
                self.assertEqual(self.devloop.measure_noise_floor(Path("/tmp/aa"), Path("/bin/oak"), 2),
147
                                 (self.devloop.NOISE_FLOOR_MIN_PCT, 0))
148
149
    def test_noise_floor_command_skips_remote_operations(self) -> None:
150
        captured: list[list[str]] = []
151
152
        def fake_run_lane(command: list[str], label: str) -> None:
153
            captured.append(command)
154
155
        with (
156
            mock.patch.object(self.devloop, "run_lane", side_effect=fake_run_lane),
157
            mock.patch.object(self.devloop, "load_jsonl", return_value=[]),
158
        ):
159
            self.devloop.measure_noise_floor(Path("/tmp/aa"), Path("/bin/oak"), 3)
160
161
        self.assertIn("--skip-remote", captured[0])
162
163
    def test_contention_preflight_includes_workspace_and_clone_push_gate(self) -> None:
164
        requirements = self.devloop.remote_preflight_requirements(["contention"])
165
166
        self.assertEqual(len(requirements), 1)
167
        operations = set(requirements[0].operations)
168
        self.assertIn("contention_workspace_per_task_w2/parallel.total", operations)
169
        self.assertIn("contention_workspace_per_task_w8/parallel.total", operations)
170
        self.assertIn("contention_clone_push_storm_w2/parallel.total", operations)
171
        self.assertIn("contention_clone_push_storm_w8/parallel.total", operations)
172
173
    def test_contention_preflight_uses_custom_launch_worker_tiers(self) -> None:
174
        requirements = self.devloop.remote_preflight_requirements(
175
            ["contention"],
176
            contention_modes=["clone_push_storm"],
177
            contention_workers=["16", "128"],
178
        )
179
180
        operations = set(requirements[0].operations)
181
        self.assertEqual(
182
            operations,
183
            {
184
                "contention_clone_push_storm_w16/parallel.total",
185
                "contention_clone_push_storm_w128/parallel.total",
186
            },
187
        )
188
189
    def test_contention_command_runs_clone_push_storm_by_default(self) -> None:
190
        import tempfile
191
192
        captured: list[list[str]] = []
193
194
        def fake_run_lane(command: list[str], label: str) -> None:
195
            captured.append(command)
196
197
        with tempfile.TemporaryDirectory() as tmp:
198
            argv = [
199
                "devloop.py",
200
                "--lanes",
201
                "contention",
202
                "--skip-noise-floor",
203
                "--skip-build",
204
                "--oak-baseline-bin",
205
                "/bin/echo",
206
                "--oak-local-bin",
207
                "/bin/echo",
208
                "--results",
209
                tmp,
210
            ]
211
            with (
212
                mock.patch.object(sys, "argv", argv),
213
                mock.patch.object(self.devloop, "run_lane", side_effect=fake_run_lane),
214
                mock.patch.object(self.devloop, "load_jsonl", return_value=[]),
215
                mock.patch.object(self.devloop.oakbench_remotes, "print_remote_preflight_warnings"),
216
            ):
217
                rc = self.devloop.main()
218
219
        self.assertEqual(rc, 0)
220
        self.assertEqual(len(captured), 1)
221
        command = captured[0]
222
        self.assertIn("--modes", command)
223
        self.assertEqual(
224
            command[command.index("--modes") + 1],
225
            self.devloop.DEFAULT_CONTENTION_MODES,
226
        )
227
        self.assertIn("clone_push_storm", command[command.index("--modes") + 1])
228
229
    def test_diagnostic_probe_failure_does_not_trip_regression_gate(self) -> None:
230
        rows = [
231
            row("oak_installed", "status.dirty.determinism", returncode=0, elapsed_ms=10.0),
232
            row("oak_local", "status.dirty.determinism", returncode=1, elapsed_ms=11.0),
233
            row("oak_installed", "status.dirty", returncode=0, elapsed_ms=10.0),
234
            row("oak_local", "status.dirty", returncode=0, elapsed_ms=11.0),
235
        ]
236
237
        regressions = self.compare(rows)
238
239
        self.assertFalse(any("status.dirty.determinism" in item for item in regressions))
240
241
    def test_load_jsonl_reports_controlled_input_errors(self) -> None:
242
        with tempfile.TemporaryDirectory() as tmp:
243
            root = Path(tmp)
244
            with self.assertRaises(SystemExit) as missing:
245
                self.devloop.load_jsonl(root / "missing.jsonl")
246
            self.assertIn("result JSONL missing", str(missing.exception))
247
248
            empty = root / "empty.jsonl"
249
            empty.write_text("\n")
250
            with self.assertRaises(SystemExit) as empty_error:
251
                self.devloop.load_jsonl(empty)
252
            self.assertIn("contains no rows", str(empty_error.exception))
253
254
            malformed = root / "malformed.jsonl"
255
            malformed.write_text('{"ok": true}\nnot json\n')
256
            with self.assertRaises(SystemExit) as malformed_error:
257
                self.devloop.load_jsonl(malformed)
258
            self.assertIn("malformed result JSONL", str(malformed_error.exception))
259
            self.assertIn(":2:", str(malformed_error.exception))
260
261
            not_object = root / "not-object.jsonl"
262
            not_object.write_text("[1, 2, 3]\n")
263
            with self.assertRaises(SystemExit) as not_object_error:
264
                self.devloop.load_jsonl(not_object)
265
            self.assertIn("row is not an object", str(not_object_error.exception))
266
267
            valid = root / "valid.jsonl"
268
            valid.write_text('{"ok": true}\n\n')
269
            self.assertEqual(self.devloop.load_jsonl(valid), [{"ok": True}])
270
271
272
class DevloopScorecardTest(unittest.TestCase):
273
    def setUp(self) -> None:
274
        self.devloop = load_devloop()
275
276
    def test_scorecard_lines_explain_missing_book_and_targets(self) -> None:
277
        import json
278
        import tempfile
279
280
        with tempfile.TemporaryDirectory() as tmp:
281
            root = Path(tmp)
282
            self.assertIn("config/targets.json missing", self.devloop.scorecard_lines([], root)[0])
283
284
            (root / "config").mkdir()
285
            (root / "config" / "targets.json").write_text(json.dumps({"schema_version": 1, "targets": []}))
286
            self.assertIn("baselines/git/CURRENT missing", self.devloop.scorecard_lines([], root)[0])
287
288
    def test_scorecard_lines_render_verdicts_from_current_book(self) -> None:
289
        import json
290
        import tempfile
291
292
        with tempfile.TemporaryDirectory() as tmp:
293
            root = Path(tmp)
294
            (root / "config").mkdir()
295
            (root / "config" / "targets.json").write_text(
296
                json.dumps(
297
                    {
298
                        "schema_version": 1,
299
                        "targets": [
300
                            {
301
                                "id": "t",
302
                                "lane": "core",
303
                                "profile": "smoke",
304
                                "scenario": "tiny_text",
305
                                "operation": "status.clean",
306
                                "subject": "oak_local",
307
                                "comparator": "git",
308
                                "metric": "elapsed_ms",
309
                                "goal": {"type": "ratio_at_most", "value": 0.8},
310
                                "guards": {},
311
                            }
312
                        ],
313
                    }
314
                )
315
            )
316
            book_dir = root / "baselines" / "git" / "test-book"
317
            book_dir.mkdir(parents=True)
318
            (root / "baselines" / "git" / "CURRENT").write_text("test-book\n")
319
            (book_dir / "book.json").write_text(
320
                json.dumps(
321
                    {
322
                        "schema_version": 1,
323
                        "cells": [
324
                            {
325
                                "lane": "core",
326
                                "profile": "smoke",
327
                                "scenario": "tiny_text",
328
                                "operation": "status.clean",
329
                                "subject": "git",
330
                                "metric": "elapsed_ms",
331
                                "statistic": "median",
332
                                "value": 10.0,
333
                                "n": 50,
334
                            }
335
                        ],
336
                    }
337
                )
338
            )
339
            core_row = {
340
                "profile": "smoke",
341
                "scenario": "tiny_text",
342
                "operation": "status.clean",
343
                "subject": "oak_local",
344
                "subject_kind": "oak",
345
                "returncode": 0,
346
                "elapsed_ms": 5.0,
347
                "command": ["oak", "status"],
348
            }
349
            lines = self.devloop.scorecard_lines([core_row, core_row], root)
350
            self.assertEqual(len(lines), 1)
351
            self.assertIn("GREEN", lines[0])
352
353
    def test_scorecard_lines_applies_live_noise_floor_to_loaded_book(self) -> None:
354
        import json
355
        import tempfile
356
357
        with tempfile.TemporaryDirectory() as tmp:
358
            root = Path(tmp)
359
            (root / "config").mkdir()
360
            (root / "config" / "targets.json").write_text(
361
                json.dumps(
362
                    {
363
                        "schema_version": 1,
364
                        "targets": [
365
                            {
366
                                "id": "t",
367
                                "lane": "core",
368
                                "profile": "smoke",
369
                                "scenario": "tiny_text",
370
                                "operation": "status.clean",
371
                                "subject": "oak_local",
372
                                "comparator": "git",
373
                                "metric": "elapsed_ms",
374
                                "goal": {"type": "ratio_at_most", "value": 0.99},
375
                                "guards": {},
376
                            }
377
                        ],
378
                    }
379
                )
380
            )
381
            book_dir = root / "baselines" / "git" / "test-book"
382
            book_dir.mkdir(parents=True)
383
            (root / "baselines" / "git" / "CURRENT").write_text("test-book\n")
384
            (book_dir / "book.json").write_text(
385
                json.dumps(
386
                    {
387
                        "schema_version": 1,
388
                        "cells": [
389
                            {
390
                                "lane": "core",
391
                                "profile": "smoke",
392
                                "scenario": "tiny_text",
393
                                "operation": "status.clean",
394
                                "subject": "git",
395
                                "metric": "elapsed_ms",
396
                                "statistic": "median",
397
                                "value": 10.0,
398
                                "n": 50,
399
                                "noise_floor_pct": None,
400
                            }
401
                        ],
402
                    }
403
                )
404
            )
405
            core_row = {
406
                "profile": "smoke",
407
                "scenario": "tiny_text",
408
                "operation": "status.clean",
409
                "subject": "oak_local",
410
                "subject_kind": "oak",
411
                "returncode": 0,
412
                "elapsed_ms": 9.7,
413
                "command": ["oak", "status"],
414
            }
415
            no_noise = self.devloop.scorecard_lines([core_row, core_row], root, noise_floor_pct=0.0)
416
            self.assertIn("GREEN", no_noise[0])
417
            self.assertIn("`t`", no_noise[0])
418
            self.assertIn(
419
                "UNDECIDABLE",
420
                self.devloop.scorecard_lines([core_row, core_row], root, noise_floor_pct=5.0)[0],
421
            )
422
423
424
class OutputOracleGateTest(unittest.TestCase):
425
    def setUp(self):
426
        self.devloop = load_devloop()
427
428
    compare = DevloopVerdictTest.compare
429
430
    def pair(self):
431
        result = [row(subject, "diff.full.inforecall") for subject in ("oak_installed", "oak_local")]
432
        for item in result:
433
            item.update(output_oracle_version="exact-paths-v2", information_recall=1.0,
434
                        information_precision=1.0, patch_oracle_version="apply-tree-v1",
435
                        patch_apply_ok=True, patch_application_required=True, patch_input_sha256="a" * 64,
436
                        patch_expected_tree_sha256="b" * 64, patch_apply_returncode=0,
437
                        patch_instrument_sha256="d" * 64, patch_instrument_version="git version control")
438
        return result
439
440
    def test_mixed_recall_versions_are_not_compared(self):
441
        rows = self.pair()
442
        rows[0].pop("output_oracle_version")
443
        rows[1]["information_recall"] = 0.2
444
        failures = self.compare(rows)
445
        self.assertTrue(any("OUTPUT-ORACLE" in item for item in failures))
446
        self.assertFalse(any(item.startswith("RECALL") for item in failures))
447
448
    def test_exact_precision_regression_is_gated(self):
449
        rows = self.pair()
450
        rows[1]["information_precision"] = 0.5
451
        self.assertTrue(any("PRECISION" in item for item in self.compare(rows)))
452
453
    def test_patch_false_and_lost_measurement_cannot_pass(self):
454
        for value in (False,):
455
            rows = self.pair()
456
            rows[1]["patch_apply_ok"] = value
457
            self.assertTrue(any("PATCH-APPLY" in item for item in self.compare(rows)))
458
459
    def test_preexisting_patch_defect_and_unknown_instrument_are_separate(self):
460
        rows = self.pair()
461
        for item in rows:
462
            item.update(patch_apply_ok=False, patch_failure_reason="patch_result_tree_mismatch")
463
        diagnostics = []
464
        failures, _ = self.devloop.compare_subjects(rows, "oak_local", "oak_installed", "baseline", 100, 100,
465
            efficiency_gates=False, output_bytes_threshold_pct=100, diagnostics=diagnostics)
466
        self.assertFalse(failures)
467
        self.assertTrue(any("PREEXISTING" in text for text in diagnostics))
468
        rows[1].update(patch_apply_ok=None, patch_skip_reason="git_apply_instrument_timeout")
469
        diagnostics.clear()
470
        failures, _ = self.devloop.compare_subjects(rows, "oak_local", "oak_installed", "baseline", 100, 100,
471
            efficiency_gates=False, output_bytes_threshold_pct=100, diagnostics=diagnostics)
472
        self.assertFalse(failures)
473
        self.assertTrue(any("UNMEASURED" in text for text in diagnostics))
474
475
    def test_different_failed_patch_is_not_preexisting(self):
476
        for difference in ({"patch_input_sha256": "c" * 64}, {"patch_expected_tree_sha256": "c" * 64},
477
                           {"patch_input_sha256": None}, {"patch_apply_returncode": 2},
478
                           {"patch_instrument_sha256": "c" * 64}, {"patch_instrument_version": "changed"}):
479
            rows = self.pair()
480
            for item in rows:
481
                item.update(patch_apply_ok=False, patch_failure_reason="patch_application_failed", patch_apply_returncode=1)
482
            rows[1].update(difference)
483
            self.assertTrue(any("PATCH-APPLY" in text for text in self.compare(rows)))
484
485
    def test_skipped_instrument_is_not_generic_no_success_regression(self):
486
        rows = [row("oak_installed", "workflow.total"),
487
                {**row("oak_local", "workflow.total", returncode=77), "skip_reason": "git_unavailable"}]
488
        diagnostics = []
489
        failures, _ = self.devloop.compare_subjects(rows, "oak_local", "oak_installed", "baseline", 100, 100,
490
            efficiency_gates=False, output_bytes_threshold_pct=100, diagnostics=diagnostics)
491
        self.assertFalse(failures)
492
        self.assertTrue(any("UNMEASURED" in text for text in diagnostics))
493
        rows[1]["returncode"] = 1
494
        self.assertTrue(self.compare(rows))
495
496
    def test_optional_patch_unknown_is_informational(self):
497
        rows = self.pair()
498
        rows[1].update(patch_apply_ok=None, patch_application_required=False, patch_skip_reason="patch_output_exceeds_capture_limit")
499
        diagnostics = []
500
        failures, _ = self.devloop.compare_subjects(rows, "oak_local", "oak_installed", "baseline", 100, 100,
501
            efficiency_gates=False, output_bytes_threshold_pct=100, diagnostics=diagnostics)
502
        self.assertFalse(failures)
503
        self.assertTrue(any(text.startswith("INFO PATCH-APPLY") for text in diagnostics))
504
        self.assertFalse(any(text.startswith("UNMEASURED") for text in diagnostics))
505
506
    def test_optional_failed_or_omitted_patch_is_reported(self):
507
        for omit in (False, True):
508
            rows = self.pair()
509
            for item in rows:
510
                item["patch_application_required"] = False
511
            if omit:
512
                rows.pop()
513
            else:
514
                rows[1]["patch_apply_ok"] = False
515
            diagnostics = []
516
            failures, _ = self.devloop.compare_subjects(rows, "oak_local", "oak_installed", "baseline", 100, 100,
517
                efficiency_gates=False, output_bytes_threshold_pct=100, diagnostics=diagnostics)
518
            self.assertFalse(failures)
519
            self.assertTrue(any(text.startswith("INFO PATCH-APPLY") for text in diagnostics))
520
521
    def test_unmeasured_baseline_needs_no_patch_instrument_identity(self):
522
        rows = self.pair()
523
        rows[0].update(patch_apply_ok=None, patch_skip_reason="subject_diff_command_failed")
524
        rows[0].pop("patch_instrument_sha256"); rows[0].pop("patch_instrument_version")
525
        diagnostics = []
526
        failures, _ = self.devloop.compare_subjects(rows, "oak_local", "oak_installed", "baseline", 100, 100,
527
            efficiency_gates=False, output_bytes_threshold_pct=100, diagnostics=diagnostics)
528
        self.assertFalse(failures)
529
        self.assertEqual(diagnostics, [])
530
531
    def test_absent_target_operation_remains_a_regression(self):
532
        self.assertTrue(any("NO SUCCESS" in text for text in self.compare([row("oak_installed", "workflow.total")])))
533
534
    def test_git_guardrail_different_patch_failure_is_not_prefix_suppressed(self):
535
        rows = self.pair()
536
        for item in rows:
537
            item.update(patch_apply_ok=False, patch_failure_reason="patch_application_failed", patch_apply_returncode=1)
538
        breach = f"PATCH-APPLY `{rows[0]['scenario']}/diff.full.inforecall`: exact patch application failed"
539
        self.assertTrue(self.devloop.preexisting_patch_breach(breach, rows))
540
        rows[1]["patch_input_sha256"] = "c" * 64
541
        self.assertFalse(self.devloop.preexisting_patch_breach(breach, rows))
542
543
    def test_repeated_failures_compare_matching_run_inputs(self):
544
        rows = []
545
        for run in (0, 1):
546
            for item in self.pair():
547
                item.update(run=run, patch_apply_ok=False, patch_failure_reason="patch_application_failed",
548
                            patch_apply_returncode=1, patch_input_sha256=str(run) * 64)
549
                rows.append(item)
550
        self.assertFalse(self.compare(rows))
551
        rows[-1]["patch_input_sha256"] = "f" * 64
552
        self.assertTrue(self.compare(rows))
553
554
    def test_cli_reports_preexisting_and_unmeasured_distinctly(self):
555
        import contextlib
556
        import io
557
        for value, expected_exit, expected_note in ((False, 0, "PREEXISTING"), (None, 2, "UNMEASURED")):
558
            rows = self.pair()
559
            for item in rows:
560
                item.update(patch_apply_ok=False, patch_failure_reason="patch_result_tree_mismatch", returncode=1)
561
            rows[1]["patch_apply_ok"] = value
562
            if value is None:
563
                rows[1].update(returncode=0, patch_skip_reason="git_apply_instrument_timeout")
564
            with tempfile.TemporaryDirectory() as tmp:
565
                argv = ["devloop.py", "--lanes", "core", "--skip-noise-floor", "--skip-build",
566
                        "--oak-baseline-bin", "/bin/echo", "--oak-local-bin", "/bin/echo", "--results", tmp]
567
                output = io.StringIO()
568
                with mock.patch.object(sys, "argv", argv), mock.patch.object(self.devloop, "run_lane"), mock.patch.object(self.devloop, "load_jsonl", return_value=rows), mock.patch.object(self.devloop.oakbench_remotes, "print_remote_preflight_warnings"), contextlib.redirect_stdout(output):
569
                    result = self.devloop.main()
570
                self.assertEqual(result, expected_exit, output.getvalue())
571
                self.assertIn(expected_note, output.getvalue())
572
                self.assertNotIn("**REGRESSED**", output.getvalue())
573
574
    def test_cli_new_bad_patch_is_a_new_git_guardrail_breach(self):
575
        import contextlib
576
        import io
577
        rows = self.pair()
578
        rows.append({**rows[0], "subject": "git"})
579
        for item in rows[:2]:
580
            item.update(patch_apply_ok=False, patch_failure_reason="patch_application_failed", patch_apply_returncode=1)
581
        rows[1]["patch_input_sha256"] = "c" * 64
582
        with tempfile.TemporaryDirectory() as tmp:
583
            argv = ["devloop.py", "--lanes", "core", "--skip-noise-floor", "--skip-build",
584
                    "--oak-baseline-bin", "/bin/echo", "--oak-local-bin", "/bin/echo", "--results", tmp]
585
            output = io.StringIO()
586
            with mock.patch.object(sys, "argv", argv), mock.patch.object(self.devloop, "run_lane"), mock.patch.object(self.devloop, "load_jsonl", return_value=rows), mock.patch.object(self.devloop.oakbench_remotes, "print_remote_preflight_warnings"), contextlib.redirect_stdout(output):
587
                self.assertEqual(self.devloop.main(), 1)
588
            self.assertIn("New Git-guardrail breaches introduced by this changeset (1)", output.getvalue())
589
            self.assertIn("Known Git gaps, pre-existing in the baseline (0)", output.getvalue())
590
591
    def test_patch_oracle_versions_must_match(self):
592
        rows = self.pair()
593
        rows[1]["patch_oracle_version"] = "different"
594
        self.assertTrue(any("PATCH-ORACLE" in item for item in self.compare(rows)))
595
596
    def test_omitting_target_patch_row_cannot_hide_lost_proof(self):
597
        rows = self.pair()[:1]
598
        rows.append(row("oak_local", "status.clean"))
599
        self.assertTrue(any("PATCH-APPLY" in item for item in self.compare(rows)))
600
601
602
if __name__ == "__main__":
603
    unittest.main()