| 1 | """Fast end-to-end checks for the core, contention, and mock-agent lanes.
|
| 2 |
|
| 3 | Together with test_integration_workflow.py, every lane has a tiny-scale wire
|
| 4 | test from fixture to contract-valid rows, so agents can refactor any harness
|
| 5 | and know within seconds whether the lane still measures.
|
| 6 | """
|
| 7 |
|
| 8 | from __future__ import annotations
|
| 9 |
|
| 10 | import sys
|
| 11 | from pathlib import Path
|
| 12 |
|
| 13 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 14 |
|
| 15 | import shutil
|
| 16 | import tempfile
|
| 17 | import unittest
|
| 18 | import hashlib
|
| 19 | import re
|
| 20 |
|
| 21 | import agent_workflow
|
| 22 | import bench
|
| 23 | import parallel_contention
|
| 24 | import workflow_ab
|
| 25 | from agent_metrics_schema import validate_row
|
| 26 | from oakbench.rows import validate_rows
|
| 27 | from oakbench.runner import runner_fields
|
| 28 | from oakbench.subjects import subject_details
|
| 29 | from oakbench.subjects import Subject
|
| 30 |
|
| 31 | GIT = shutil.which("git")
|
| 32 | SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
| 33 |
|
| 34 |
|
| 35 | def git_subject() -> Subject:
|
| 36 | return Subject(name="git", kind="git", label="Git", bin=Path(GIT))
|
| 37 |
|
| 38 |
|
| 39 | @unittest.skipIf(GIT is None, "git not installed")
|
| 40 | class CoreLaneIntegrationTest(unittest.TestCase):
|
| 41 | def setUp(self) -> None:
|
| 42 | self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-core-test-"))
|
| 43 |
|
| 44 | def tearDown(self) -> None:
|
| 45 | shutil.rmtree(self.tmp, ignore_errors=True)
|
| 46 |
|
| 47 | def test_micro_scenario_end_to_end(self) -> None:
|
| 48 | scenario = bench.Scenario("tiny_text", 8, 256, 2, False, 1)
|
| 49 | fixture = bench.ensure_fixture(self.tmp, scenario)
|
| 50 | metadata = {
|
| 51 | "bench_id": "test",
|
| 52 | "profile": "micro",
|
| 53 | "benchmark_track": "agent-default",
|
| 54 | "command_semantics_version": bench.COMMAND_SEMANTICS_VERSION,
|
| 55 | "timestamp_utc": "test",
|
| 56 | "subject_details": subject_details([git_subject()]),
|
| 57 | **runner_fields(
|
| 58 | {
|
| 59 | "system": "Linux",
|
| 60 | "release": "6.8",
|
| 61 | "machine": "x86_64",
|
| 62 | "processor": "x86_64",
|
| 63 | "cpu_model": "Test CPU",
|
| 64 | "cpu_count_logical": 8,
|
| 65 | "filesystem_type": "ext4",
|
| 66 | },
|
| 67 | {"classes": [{"id": "unknown", "default": True}]},
|
| 68 | ),
|
| 69 | }
|
| 70 | rows = bench.run_subject_scenario(
|
| 71 | git_subject(),
|
| 72 | scenario,
|
| 73 | fixture,
|
| 74 | run_index=0,
|
| 75 | run_root=self.tmp / "runs",
|
| 76 | metadata=metadata,
|
| 77 | skip_diff=False,
|
| 78 | admitted_output_chars=20_000,
|
| 79 | track="agent-default",
|
| 80 | )
|
| 81 |
|
| 82 | self.assertEqual(validate_rows(rows, "core"), [])
|
| 83 | self.assertTrue(rows)
|
| 84 | for row in rows:
|
| 85 | self.assertIsInstance(row.get("runner_id"), str)
|
| 86 | self.assertIsInstance(row.get("runner_class"), str)
|
| 87 | self.assertIsInstance(row.get("runner_profile"), dict)
|
| 88 | self.assertIn("binary_sha256", row)
|
| 89 | digest = row.get("binary_sha256")
|
| 90 | self.assertTrue(digest is None or SHA256_RE.fullmatch(digest), digest)
|
| 91 | operations = [row["operation"] for row in rows]
|
| 92 | for expected in (
|
| 93 | "proc.spawn",
|
| 94 | "repo.init",
|
| 95 | "snapshot.initial.add",
|
| 96 | "snapshot.initial.commit",
|
| 97 | "snapshot.initial",
|
| 98 | "status.clean",
|
| 99 | "status.dirty",
|
| 100 | "status.dirty.determinism",
|
| 101 | "diff.dirty",
|
| 102 | "diff.dirty.determinism",
|
| 103 | "snapshot.dirty",
|
| 104 | "branch.create",
|
| 105 | "task.snapshot",
|
| 106 | ):
|
| 107 | self.assertIn(expected, operations)
|
| 108 |
|
| 109 | failures = [row for row in rows if row["returncode"] != 0]
|
| 110 | self.assertEqual(failures, [])
|
| 111 |
|
| 112 | probe = next(row for row in rows if row["operation"] == "status.dirty.determinism")
|
| 113 | self.assertIn("output_stable", probe)
|
| 114 | self.assertIn("ansi_escape_count_non_tty", probe)
|
| 115 |
|
| 116 | def test_subject_provenance_uses_subject_specific_binary_hashes(self) -> None:
|
| 117 | git_bin = self.tmp / "fake-git"
|
| 118 | oak_bin = self.tmp / "fake-oak"
|
| 119 | git_bin.write_bytes(b"git-binary")
|
| 120 | oak_bin.write_bytes(b"oak-binary")
|
| 121 | git_subject = Subject(name="git", kind="git", label="Git", bin=git_bin)
|
| 122 | oak_subject = Subject(name="oak_local", kind="oak", label="Oak", bin=oak_bin)
|
| 123 | metadata = {"subject_details": subject_details([git_subject, oak_subject])}
|
| 124 |
|
| 125 | git_hash = bench.subject_provenance(git_subject, metadata)["binary_sha256"]
|
| 126 | oak_hash = bench.subject_provenance(oak_subject, metadata)["binary_sha256"]
|
| 127 |
|
| 128 | self.assertEqual(git_hash, hashlib.sha256(b"git-binary").hexdigest())
|
| 129 | self.assertEqual(oak_hash, hashlib.sha256(b"oak-binary").hexdigest())
|
| 130 | self.assertNotEqual(git_hash, oak_hash)
|
| 131 |
|
| 132 |
|
| 133 | @unittest.skipIf(GIT is None, "git not installed")
|
| 134 | class ContentionLaneIntegrationTest(unittest.TestCase):
|
| 135 | def setUp(self) -> None:
|
| 136 | self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-contention-test-"))
|
| 137 |
|
| 138 | def tearDown(self) -> None:
|
| 139 | shutil.rmtree(self.tmp, ignore_errors=True)
|
| 140 |
|
| 141 | def test_shared_checkout_two_workers(self) -> None:
|
| 142 | fixture = self.tmp / "fixture"
|
| 143 | parallel_contention.make_fixture(fixture)
|
| 144 | metadata = {"bench_id": "test", "profile": "parallel-contention", "timestamp_utc": "test"}
|
| 145 | rows = parallel_contention.run_shared_checkout(
|
| 146 | git_subject(),
|
| 147 | fixture,
|
| 148 | workers=2,
|
| 149 | commits=1,
|
| 150 | run_root=self.tmp / "runs",
|
| 151 | run_index=0,
|
| 152 | metadata=metadata,
|
| 153 | )
|
| 154 |
|
| 155 | self.assertEqual(validate_rows(rows, "contention"), [])
|
| 156 | total = next(row for row in rows if row["operation"] == "parallel.total")
|
| 157 | metrics = total["parallel_metrics"]
|
| 158 | self.assertEqual(metrics["tasks_total"], 2)
|
| 159 | self.assertEqual(metrics["tasks_completed"], 2)
|
| 160 | self.assertEqual(metrics["lost_updates_detected"], 0)
|
| 161 | self.assertTrue(metrics["integrity_check_passed"])
|
| 162 | self.assertIsNotNone(metrics["commit_throughput_per_s"])
|
| 163 |
|
| 164 |
|
| 165 | @unittest.skipIf(GIT is None, "git not installed")
|
| 166 | class MockAgentLaneIntegrationTest(unittest.TestCase):
|
| 167 | def setUp(self) -> None:
|
| 168 | self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-agent-test-"))
|
| 169 |
|
| 170 | def tearDown(self) -> None:
|
| 171 | shutil.rmtree(self.tmp, ignore_errors=True)
|
| 172 |
|
| 173 | def test_mock_bugfix_end_to_end(self) -> None:
|
| 174 | agent = agent_workflow.AgentConfig(
|
| 175 | name="mock",
|
| 176 | adapter="deterministic_mock",
|
| 177 | provider="local",
|
| 178 | bin=None,
|
| 179 | model=None,
|
| 180 | version="test",
|
| 181 | )
|
| 182 | workflow = workflow_ab.WORKFLOWS["bugfix_test_loop"]
|
| 183 | fixture = workflow_ab.prepare_fixture(self.tmp, workflow)
|
| 184 | row = agent_workflow.run_one(
|
| 185 | agent=agent,
|
| 186 | subject=git_subject(),
|
| 187 | workflow=workflow,
|
| 188 | fixture=fixture,
|
| 189 | run_index=0,
|
| 190 | run_root=self.tmp / "runs",
|
| 191 | artifacts_root=self.tmp / "artifacts",
|
| 192 | timeout_seconds=300.0,
|
| 193 | track="agent-default",
|
| 194 | agent_environment="minimal",
|
| 195 | instruction_level="full-docs",
|
| 196 | bench_id="test",
|
| 197 | timestamp_utc="2026-06-10T00:00:00Z",
|
| 198 | source={"oak_hash": None, "oak_status": None},
|
| 199 | host_meta={"host": "test", "platform": "test", "machine": "test"},
|
| 200 | )
|
| 201 |
|
| 202 | self.assertEqual(validate_row(row), [])
|
| 203 | self.assertTrue(row["success"], row.get("failure_reason"))
|
| 204 | # Null-honesty: the mock adapter exposes no stream, so turn metrics
|
| 205 | # must be unmeasured, never zero (ADR-0002).
|
| 206 | self.assertEqual(row["turn_metrics"]["turn_source"], "unmeasured")
|
| 207 | self.assertIsNone(row["turn_metrics"]["assistant_turns_total"])
|
| 208 | self.assertIsNone(row["bytes_metrics"]["bytes_hydrated"])
|
| 209 |
|
| 210 |
|
| 211 | if __name__ == "__main__":
|
| 212 | unittest.main()
|