| 1 | """Tests for the VCS PATH shim instrument.
|
| 2 |
|
| 3 | The shim must be behavior-transparent (same stdout, same exit code as the
|
| 4 | real binary) while producing sidecar rows directly consumable by
|
| 5 | oakbench.thrash. Tests run the real git binary and skip when absent.
|
| 6 | """
|
| 7 |
|
| 8 | from __future__ import annotations
|
| 9 |
|
| 10 | import json
|
| 11 | import os
|
| 12 | import shutil
|
| 13 | import subprocess
|
| 14 | import sys
|
| 15 | import tempfile
|
| 16 | import unittest
|
| 17 | from pathlib import Path
|
| 18 |
|
| 19 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 20 |
|
| 21 | from oakbench.thrash import agent_blocked_on_vcs_ms
|
| 22 | from oakbench.vcs_shim import create_shim_dir, measure_shim_overhead, read_sidecar_rows
|
| 23 |
|
| 24 | GIT = shutil.which("git")
|
| 25 |
|
| 26 |
|
| 27 | def shim_env(shim_dir: Path) -> dict[str, str]:
|
| 28 | env = dict(os.environ)
|
| 29 | env["PATH"] = str(shim_dir) + os.pathsep + env.get("PATH", "")
|
| 30 | return env
|
| 31 |
|
| 32 |
|
| 33 | @unittest.skipIf(GIT is None, "git is not installed")
|
| 34 | class VcsShimTest(unittest.TestCase):
|
| 35 | def setUp(self) -> None:
|
| 36 | self.tmp = Path(tempfile.mkdtemp(prefix="vcs-shim-test-"))
|
| 37 | self.addCleanup(shutil.rmtree, self.tmp, True)
|
| 38 | self.sidecar = self.tmp / "sidecar.jsonl"
|
| 39 | self.shim_dir = create_shim_dir(
|
| 40 | self.tmp / "shim", real_binaries={"git": GIT}, sidecar_path=self.sidecar
|
| 41 | )
|
| 42 |
|
| 43 | def run_shimmed(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
|
| 44 | return subprocess.run(
|
| 45 | ["git", *args],
|
| 46 | env=shim_env(self.shim_dir),
|
| 47 | cwd=cwd or self.tmp,
|
| 48 | text=True,
|
| 49 | capture_output=True,
|
| 50 | check=False,
|
| 51 | )
|
| 52 |
|
| 53 | def test_only_listed_binaries_get_wrappers(self) -> None:
|
| 54 | self.assertTrue((self.shim_dir / "git").exists())
|
| 55 | self.assertFalse((self.shim_dir / "oak").exists())
|
| 56 |
|
| 57 | def test_wrapper_uses_full_buffer_write_loop(self) -> None:
|
| 58 | script = (self.shim_dir / "git").read_text()
|
| 59 | self.assertIn("def write_all", script)
|
| 60 | self.assertIn("while written_total < len(data)", script)
|
| 61 | self.assertIn("write_all(fd, data)", script)
|
| 62 |
|
| 63 | def test_shim_is_transparent_and_records_one_row(self) -> None:
|
| 64 | direct = subprocess.run([GIT, "--version"], text=True, capture_output=True, check=False)
|
| 65 | shimmed = self.run_shimmed("--version")
|
| 66 |
|
| 67 | self.assertEqual(shimmed.returncode, 0)
|
| 68 | self.assertEqual(shimmed.stdout, direct.stdout)
|
| 69 |
|
| 70 | rows = read_sidecar_rows(self.sidecar)
|
| 71 | self.assertEqual(len(rows), 1)
|
| 72 | row = rows[0]
|
| 73 | for key in ("command", "argv", "start_ms", "end_ms", "elapsed_ms", "returncode", "shim"):
|
| 74 | self.assertIn(key, row)
|
| 75 | self.assertEqual(row["command"], "git --version")
|
| 76 | self.assertTrue(row["command"].startswith("git --version"))
|
| 77 | self.assertEqual(row["argv"], ["--version"])
|
| 78 | self.assertEqual(row["returncode"], 0)
|
| 79 | self.assertIs(row["shim"], True)
|
| 80 | self.assertGreater(row["elapsed_ms"], 0)
|
| 81 | self.assertGreaterEqual(row["end_ms"], row["start_ms"])
|
| 82 | self.assertAlmostEqual(row["elapsed_ms"], row["end_ms"] - row["start_ms"], places=3)
|
| 83 |
|
| 84 | def test_concurrent_shim_calls_produce_two_valid_json_lines(self) -> None:
|
| 85 | env = shim_env(self.shim_dir)
|
| 86 | procs = [
|
| 87 | subprocess.Popen(
|
| 88 | ["git", "--version"],
|
| 89 | env=env,
|
| 90 | cwd=self.tmp,
|
| 91 | stdout=subprocess.DEVNULL,
|
| 92 | stderr=subprocess.DEVNULL,
|
| 93 | )
|
| 94 | for _ in range(2)
|
| 95 | ]
|
| 96 | for proc in procs:
|
| 97 | self.assertEqual(proc.wait(timeout=30), 0)
|
| 98 |
|
| 99 | lines = [line for line in self.sidecar.read_text().splitlines() if line.strip()]
|
| 100 | self.assertEqual(len(lines), 2)
|
| 101 | for line in lines:
|
| 102 | row = json.loads(line) # raises on interleaved/corrupt lines
|
| 103 | self.assertEqual(row["command"], "git --version")
|
| 104 | self.assertEqual(row["returncode"], 0)
|
| 105 |
|
| 106 | def test_measure_shim_overhead_is_non_negative_float(self) -> None:
|
| 107 | overhead = measure_shim_overhead(self.shim_dir, self.sidecar, GIT, n=3)
|
| 108 | self.assertIsInstance(overhead, float)
|
| 109 | self.assertGreaterEqual(overhead, 0.0)
|
| 110 |
|
| 111 | def test_nonzero_exit_passthrough(self) -> None:
|
| 112 | direct = subprocess.run(
|
| 113 | [GIT, "definitely-not-a-command"],
|
| 114 | cwd=self.tmp,
|
| 115 | text=True,
|
| 116 | capture_output=True,
|
| 117 | check=False,
|
| 118 | )
|
| 119 | self.assertNotEqual(direct.returncode, 0)
|
| 120 | shimmed = self.run_shimmed("definitely-not-a-command")
|
| 121 | self.assertEqual(shimmed.returncode, direct.returncode)
|
| 122 | rows = read_sidecar_rows(self.sidecar)
|
| 123 | self.assertEqual(rows[-1]["returncode"], direct.returncode)
|
| 124 |
|
| 125 | def test_sidecar_rows_feed_thrash_metrics(self) -> None:
|
| 126 | repo = self.tmp / "repo"
|
| 127 | repo.mkdir()
|
| 128 | init = self.run_shimmed("init", cwd=repo)
|
| 129 | self.assertEqual(init.returncode, 0)
|
| 130 | status = self.run_shimmed("status", cwd=repo)
|
| 131 | self.assertEqual(status.returncode, 0)
|
| 132 |
|
| 133 | rows = read_sidecar_rows(self.sidecar)
|
| 134 | self.assertGreaterEqual(len(rows), 2)
|
| 135 | blocked = agent_blocked_on_vcs_ms(rows)
|
| 136 | self.assertIsInstance(blocked, float)
|
| 137 | self.assertGreater(blocked, 0.0)
|
| 138 |
|
| 139 |
|
| 140 | if __name__ == "__main__":
|
| 141 | unittest.main()
|