Viewing files at commit 3354584a5d91Commit detailsView latest

Browse files
tests/test_vcs_shim.py 141 lines · 5.2 KB · python
1
"""Tests for the VCS PATH shim instrument.
2
3
The shim must be behavior-transparent (same stdout, same exit code as the
4
real binary) while producing sidecar rows directly consumable by
5
oakbench.thrash. Tests run the real git binary and skip when absent.
6
"""
7
8
from __future__ import annotations
9
10
import json
11
import os
12
import shutil
13
import subprocess
14
import sys
15
import tempfile
16
import unittest
17
from pathlib import Path
18
19
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
20
21
from oakbench.thrash import agent_blocked_on_vcs_ms
22
from oakbench.vcs_shim import create_shim_dir, measure_shim_overhead, read_sidecar_rows
23
24
GIT = shutil.which("git")
25
26
27
def shim_env(shim_dir: Path) -> dict[str, str]:
28
    env = dict(os.environ)
29
    env["PATH"] = str(shim_dir) + os.pathsep + env.get("PATH", "")
30
    return env
31
32
33
@unittest.skipIf(GIT is None, "git is not installed")
34
class VcsShimTest(unittest.TestCase):
35
    def setUp(self) -> None:
36
        self.tmp = Path(tempfile.mkdtemp(prefix="vcs-shim-test-"))
37
        self.addCleanup(shutil.rmtree, self.tmp, True)
38
        self.sidecar = self.tmp / "sidecar.jsonl"
39
        self.shim_dir = create_shim_dir(
40
            self.tmp / "shim", real_binaries={"git": GIT}, sidecar_path=self.sidecar
41
        )
42
43
    def run_shimmed(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
44
        return subprocess.run(
45
            ["git", *args],
46
            env=shim_env(self.shim_dir),
47
            cwd=cwd or self.tmp,
48
            text=True,
49
            capture_output=True,
50
            check=False,
51
        )
52
53
    def test_only_listed_binaries_get_wrappers(self) -> None:
54
        self.assertTrue((self.shim_dir / "git").exists())
55
        self.assertFalse((self.shim_dir / "oak").exists())
56
57
    def test_wrapper_uses_full_buffer_write_loop(self) -> None:
58
        script = (self.shim_dir / "git").read_text()
59
        self.assertIn("def write_all", script)
60
        self.assertIn("while written_total < len(data)", script)
61
        self.assertIn("write_all(fd, data)", script)
62
63
    def test_shim_is_transparent_and_records_one_row(self) -> None:
64
        direct = subprocess.run([GIT, "--version"], text=True, capture_output=True, check=False)
65
        shimmed = self.run_shimmed("--version")
66
67
        self.assertEqual(shimmed.returncode, 0)
68
        self.assertEqual(shimmed.stdout, direct.stdout)
69
70
        rows = read_sidecar_rows(self.sidecar)
71
        self.assertEqual(len(rows), 1)
72
        row = rows[0]
73
        for key in ("command", "argv", "start_ms", "end_ms", "elapsed_ms", "returncode", "shim"):
74
            self.assertIn(key, row)
75
        self.assertEqual(row["command"], "git --version")
76
        self.assertTrue(row["command"].startswith("git --version"))
77
        self.assertEqual(row["argv"], ["--version"])
78
        self.assertEqual(row["returncode"], 0)
79
        self.assertIs(row["shim"], True)
80
        self.assertGreater(row["elapsed_ms"], 0)
81
        self.assertGreaterEqual(row["end_ms"], row["start_ms"])
82
        self.assertAlmostEqual(row["elapsed_ms"], row["end_ms"] - row["start_ms"], places=3)
83
84
    def test_concurrent_shim_calls_produce_two_valid_json_lines(self) -> None:
85
        env = shim_env(self.shim_dir)
86
        procs = [
87
            subprocess.Popen(
88
                ["git", "--version"],
89
                env=env,
90
                cwd=self.tmp,
91
                stdout=subprocess.DEVNULL,
92
                stderr=subprocess.DEVNULL,
93
            )
94
            for _ in range(2)
95
        ]
96
        for proc in procs:
97
            self.assertEqual(proc.wait(timeout=30), 0)
98
99
        lines = [line for line in self.sidecar.read_text().splitlines() if line.strip()]
100
        self.assertEqual(len(lines), 2)
101
        for line in lines:
102
            row = json.loads(line)  # raises on interleaved/corrupt lines
103
            self.assertEqual(row["command"], "git --version")
104
            self.assertEqual(row["returncode"], 0)
105
106
    def test_measure_shim_overhead_is_non_negative_float(self) -> None:
107
        overhead = measure_shim_overhead(self.shim_dir, self.sidecar, GIT, n=3)
108
        self.assertIsInstance(overhead, float)
109
        self.assertGreaterEqual(overhead, 0.0)
110
111
    def test_nonzero_exit_passthrough(self) -> None:
112
        direct = subprocess.run(
113
            [GIT, "definitely-not-a-command"],
114
            cwd=self.tmp,
115
            text=True,
116
            capture_output=True,
117
            check=False,
118
        )
119
        self.assertNotEqual(direct.returncode, 0)
120
        shimmed = self.run_shimmed("definitely-not-a-command")
121
        self.assertEqual(shimmed.returncode, direct.returncode)
122
        rows = read_sidecar_rows(self.sidecar)
123
        self.assertEqual(rows[-1]["returncode"], direct.returncode)
124
125
    def test_sidecar_rows_feed_thrash_metrics(self) -> None:
126
        repo = self.tmp / "repo"
127
        repo.mkdir()
128
        init = self.run_shimmed("init", cwd=repo)
129
        self.assertEqual(init.returncode, 0)
130
        status = self.run_shimmed("status", cwd=repo)
131
        self.assertEqual(status.returncode, 0)
132
133
        rows = read_sidecar_rows(self.sidecar)
134
        self.assertGreaterEqual(len(rows), 2)
135
        blocked = agent_blocked_on_vcs_ms(rows)
136
        self.assertIsInstance(blocked, float)
137
        self.assertGreater(blocked, 0.0)
138
139
140
if __name__ == "__main__":
141
    unittest.main()