Viewing files at commit 5bd54e7b7233Commit detailsView latest

Browse files
tests/test_integration_lanes.py 212 lines · 7.7 KB · python
1
"""Fast end-to-end checks for the core, contention, and mock-agent lanes.
2
3
Together with test_integration_workflow.py, every lane has a tiny-scale wire
4
test from fixture to contract-valid rows, so agents can refactor any harness
5
and know within seconds whether the lane still measures.
6
"""
7
8
from __future__ import annotations
9
10
import sys
11
from pathlib import Path
12
13
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
14
15
import shutil
16
import tempfile
17
import unittest
18
import hashlib
19
import re
20
21
import agent_workflow
22
import bench
23
import parallel_contention
24
import workflow_ab
25
from agent_metrics_schema import validate_row
26
from oakbench.rows import validate_rows
27
from oakbench.runner import runner_fields
28
from oakbench.subjects import subject_details
29
from oakbench.subjects import Subject
30
31
GIT = shutil.which("git")
32
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
33
34
35
def git_subject() -> Subject:
36
    return Subject(name="git", kind="git", label="Git", bin=Path(GIT))
37
38
39
@unittest.skipIf(GIT is None, "git not installed")
40
class CoreLaneIntegrationTest(unittest.TestCase):
41
    def setUp(self) -> None:
42
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-core-test-"))
43
44
    def tearDown(self) -> None:
45
        shutil.rmtree(self.tmp, ignore_errors=True)
46
47
    def test_micro_scenario_end_to_end(self) -> None:
48
        scenario = bench.Scenario("tiny_text", 8, 256, 2, False, 1)
49
        fixture = bench.ensure_fixture(self.tmp, scenario)
50
        metadata = {
51
            "bench_id": "test",
52
            "profile": "micro",
53
            "benchmark_track": "agent-default",
54
            "command_semantics_version": bench.COMMAND_SEMANTICS_VERSION,
55
            "timestamp_utc": "test",
56
            "subject_details": subject_details([git_subject()]),
57
            **runner_fields(
58
                {
59
                    "system": "Linux",
60
                    "release": "6.8",
61
                    "machine": "x86_64",
62
                    "processor": "x86_64",
63
                    "cpu_model": "Test CPU",
64
                    "cpu_count_logical": 8,
65
                    "filesystem_type": "ext4",
66
                },
67
                {"classes": [{"id": "unknown", "default": True}]},
68
            ),
69
        }
70
        rows = bench.run_subject_scenario(
71
            git_subject(),
72
            scenario,
73
            fixture,
74
            run_index=0,
75
            run_root=self.tmp / "runs",
76
            metadata=metadata,
77
            skip_diff=False,
78
            admitted_output_chars=20_000,
79
            track="agent-default",
80
        )
81
82
        self.assertEqual(validate_rows(rows, "core"), [])
83
        self.assertTrue(rows)
84
        for row in rows:
85
            self.assertIsInstance(row.get("runner_id"), str)
86
            self.assertIsInstance(row.get("runner_class"), str)
87
            self.assertIsInstance(row.get("runner_profile"), dict)
88
            self.assertIn("binary_sha256", row)
89
            digest = row.get("binary_sha256")
90
            self.assertTrue(digest is None or SHA256_RE.fullmatch(digest), digest)
91
        operations = [row["operation"] for row in rows]
92
        for expected in (
93
            "proc.spawn",
94
            "repo.init",
95
            "snapshot.initial.add",
96
            "snapshot.initial.commit",
97
            "snapshot.initial",
98
            "status.clean",
99
            "status.dirty",
100
            "status.dirty.determinism",
101
            "diff.dirty",
102
            "diff.dirty.determinism",
103
            "snapshot.dirty",
104
            "branch.create",
105
            "task.snapshot",
106
        ):
107
            self.assertIn(expected, operations)
108
109
        failures = [row for row in rows if row["returncode"] != 0]
110
        self.assertEqual(failures, [])
111
112
        probe = next(row for row in rows if row["operation"] == "status.dirty.determinism")
113
        self.assertIn("output_stable", probe)
114
        self.assertIn("ansi_escape_count_non_tty", probe)
115
116
    def test_subject_provenance_uses_subject_specific_binary_hashes(self) -> None:
117
        git_bin = self.tmp / "fake-git"
118
        oak_bin = self.tmp / "fake-oak"
119
        git_bin.write_bytes(b"git-binary")
120
        oak_bin.write_bytes(b"oak-binary")
121
        git_subject = Subject(name="git", kind="git", label="Git", bin=git_bin)
122
        oak_subject = Subject(name="oak_local", kind="oak", label="Oak", bin=oak_bin)
123
        metadata = {"subject_details": subject_details([git_subject, oak_subject])}
124
125
        git_hash = bench.subject_provenance(git_subject, metadata)["binary_sha256"]
126
        oak_hash = bench.subject_provenance(oak_subject, metadata)["binary_sha256"]
127
128
        self.assertEqual(git_hash, hashlib.sha256(b"git-binary").hexdigest())
129
        self.assertEqual(oak_hash, hashlib.sha256(b"oak-binary").hexdigest())
130
        self.assertNotEqual(git_hash, oak_hash)
131
132
133
@unittest.skipIf(GIT is None, "git not installed")
134
class ContentionLaneIntegrationTest(unittest.TestCase):
135
    def setUp(self) -> None:
136
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-contention-test-"))
137
138
    def tearDown(self) -> None:
139
        shutil.rmtree(self.tmp, ignore_errors=True)
140
141
    def test_shared_checkout_two_workers(self) -> None:
142
        fixture = self.tmp / "fixture"
143
        parallel_contention.make_fixture(fixture)
144
        metadata = {"bench_id": "test", "profile": "parallel-contention", "timestamp_utc": "test"}
145
        rows = parallel_contention.run_shared_checkout(
146
            git_subject(),
147
            fixture,
148
            workers=2,
149
            commits=1,
150
            run_root=self.tmp / "runs",
151
            run_index=0,
152
            metadata=metadata,
153
        )
154
155
        self.assertEqual(validate_rows(rows, "contention"), [])
156
        total = next(row for row in rows if row["operation"] == "parallel.total")
157
        metrics = total["parallel_metrics"]
158
        self.assertEqual(metrics["tasks_total"], 2)
159
        self.assertEqual(metrics["tasks_completed"], 2)
160
        self.assertEqual(metrics["lost_updates_detected"], 0)
161
        self.assertTrue(metrics["integrity_check_passed"])
162
        self.assertIsNotNone(metrics["commit_throughput_per_s"])
163
164
165
@unittest.skipIf(GIT is None, "git not installed")
166
class MockAgentLaneIntegrationTest(unittest.TestCase):
167
    def setUp(self) -> None:
168
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-agent-test-"))
169
170
    def tearDown(self) -> None:
171
        shutil.rmtree(self.tmp, ignore_errors=True)
172
173
    def test_mock_bugfix_end_to_end(self) -> None:
174
        agent = agent_workflow.AgentConfig(
175
            name="mock",
176
            adapter="deterministic_mock",
177
            provider="local",
178
            bin=None,
179
            model=None,
180
            version="test",
181
        )
182
        workflow = workflow_ab.WORKFLOWS["bugfix_test_loop"]
183
        fixture = workflow_ab.prepare_fixture(self.tmp, workflow)
184
        row = agent_workflow.run_one(
185
            agent=agent,
186
            subject=git_subject(),
187
            workflow=workflow,
188
            fixture=fixture,
189
            run_index=0,
190
            run_root=self.tmp / "runs",
191
            artifacts_root=self.tmp / "artifacts",
192
            timeout_seconds=300.0,
193
            track="agent-default",
194
            agent_environment="minimal",
195
            instruction_level="full-docs",
196
            bench_id="test",
197
            timestamp_utc="2026-06-10T00:00:00Z",
198
            source={"oak_hash": None, "oak_status": None},
199
            host_meta={"host": "test", "platform": "test", "machine": "test"},
200
        )
201
202
        self.assertEqual(validate_row(row), [])
203
        self.assertTrue(row["success"], row.get("failure_reason"))
204
        # Null-honesty: the mock adapter exposes no stream, so turn metrics
205
        # must be unmeasured, never zero (ADR-0002).
206
        self.assertEqual(row["turn_metrics"]["turn_source"], "unmeasured")
207
        self.assertIsNone(row["turn_metrics"]["assistant_turns_total"])
208
        self.assertIsNone(row["bytes_metrics"]["bytes_hydrated"])
209
210
211
if __name__ == "__main__":
212
    unittest.main()