Viewing files at commit f2acb751aa96Commit detailsView latest

scripts/oakbench/vcs_shim.py 185 lines · 6.4 KB · python
1
"""PATH shim for VCS-call attribution in real-agent runs.
2
3
Real agent CLIs do not always expose per-tool-call timestamps in their
4
streams. The shim makes VCS timing observable regardless of adapter: wrapper
5
executables named after the VCS binaries (git/oak) are written into a
6
directory that the harness prepends to the agent subprocess PATH. Each
7
wrapper runs the real binary with inherited stdio (agent behavior unchanged),
8
then appends one JSON line to a sidecar JSONL with keys directly consumable
9
by oakbench.thrash: command, start_ms, end_ms, elapsed_ms, returncode.
10
11
Shim overhead is measured per run via measure_shim_overhead and recorded as a
12
field. It is never subtracted from measurements (ADR-0002: record the
13
instrument's cost, do not fabricate corrected numbers).
14
"""
15
16
from __future__ import annotations
17
18
import json
19
import statistics
20
import subprocess
21
import sys
22
import time
23
from pathlib import Path
24
25
# The wrapper intentionally uses subprocess.run rather than os.exec*: exec
26
# replaces the wrapper process, so there would be nobody left to record
27
# end_ns/returncode. stdio is inherited (no capture) and the child shares the
28
# wrapper's process-group defaults, so interactive prompts and Ctrl-C reach
29
# the real binary unchanged; a KeyboardInterrupt in the wrapper itself exits
30
# 130 like a shell would.
31
_WRAPPER_TEMPLATE = '''#!{python}
32
"""Auto-generated VCS shim wrapper. Records call timing, then runs the real binary."""
33
import fcntl
34
import json
35
import os
36
import subprocess
37
import sys
38
import time
39
40
REAL = {real}
41
SIDECAR = {sidecar}
42
TOOL = {tool}
43
44
45
def write_all(fd, data):
46
    written_total = 0
47
    while written_total < len(data):
48
        written = os.write(fd, data[written_total:])
49
        if written == 0:
50
            raise OSError("os.write returned 0 bytes")
51
        written_total += written
52
53
54
def main() -> int:
55
    argv = sys.argv[1:]
56
    start_ns = time.time_ns()
57
    try:
58
        rc = subprocess.run([REAL, *argv]).returncode
59
    except KeyboardInterrupt:
60
        rc = 130
61
    except FileNotFoundError:
62
        rc = 127
63
    end_ns = time.time_ns()
64
    row = {{
65
        "command": " ".join([TOOL, *argv]) if argv else TOOL,
66
        "argv": argv,
67
        "start_ms": start_ns / 1e6,
68
        "end_ms": end_ns / 1e6,
69
        "elapsed_ms": (end_ns - start_ns) / 1e6,
70
        "returncode": rc,
71
        "shim": True,
72
    }}
73
    data = (json.dumps(row, sort_keys=True) + "\\n").encode("utf-8")
74
    try:
75
        fd = os.open(SIDECAR, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o644)
76
        try:
77
            fcntl.flock(fd, fcntl.LOCK_EX)
78
            # With O_APPEND plus the exclusive lock, parallel shim calls
79
            # cannot interleave rows even if the OS accepts only part of the
80
            # buffer per syscall.
81
            write_all(fd, data)
82
        finally:
83
            try:
84
                fcntl.flock(fd, fcntl.LOCK_UN)
85
            finally:
86
                os.close(fd)
87
    except OSError:
88
        # Recording must never change agent-visible behavior; a lost row is
89
        # an unmeasured call, not a failed agent command.
90
        pass
91
    return rc
92
93
94
if __name__ == "__main__":
95
    raise SystemExit(main())
96
'''
97
98
99
def create_shim_dir(target_dir: Path, *, real_binaries: dict[str, str], sidecar_path: Path) -> Path:
100
    """Write executable wrapper scripts into target_dir and return it.
101
102
    One wrapper per entry in real_binaries (e.g. {"git": "/usr/bin/git",
103
    "oak": "/path/to/oak"}); names without a real binary get no wrapper, so
104
    PATH lookups for them behave exactly as without the shim. The real-binary
105
    absolute path and the sidecar path are baked into the wrapper at creation
106
    time, so the wrapper never re-resolves PATH (no recursion risk).
107
    """
108
    target_dir = Path(target_dir)
109
    target_dir.mkdir(parents=True, exist_ok=True)
110
    sidecar = Path(sidecar_path).expanduser()
111
    sidecar.parent.mkdir(parents=True, exist_ok=True)
112
    sidecar.touch(exist_ok=True)
113
    for tool, real in real_binaries.items():
114
        if not real:
115
            continue
116
        real_path = Path(real).expanduser()
117
        script = _WRAPPER_TEMPLATE.format(
118
            python=sys.executable,
119
            real=json.dumps(str(real_path)),
120
            sidecar=json.dumps(str(sidecar)),
121
            tool=json.dumps(str(tool)),
122
        )
123
        wrapper = target_dir / str(tool)
124
        wrapper.write_text(script)
125
        wrapper.chmod(0o755)
126
    return target_dir
127
128
129
def measure_shim_overhead(shim_dir: Path, sidecar: Path, real_git: str, n: int = 5) -> float | None:
130
    """Median extra ms of `shim git --version` over direct `git --version`.
131
132
    Calibration rows for the shimmed runs land in `sidecar` (the path baked
133
    into the wrappers); callers that reuse the sidecar for agent attribution
134
    should measure overhead first and reset the file. The result is recorded
135
    alongside measurements, never subtracted from them, and is clamped at 0
136
    (timer noise can make the shim appear faster; negative overhead is
137
    fabrication). Returns None when overhead could not be measured.
138
    """
139
    shim_git = Path(shim_dir) / "git"
140
    if not real_git or not shim_git.exists():
141
        return None
142
143
    def median_ms(command: list[str]) -> float | None:
144
        samples: list[float] = []
145
        for _ in range(max(1, n)):
146
            start = time.perf_counter()
147
            try:
148
                subprocess.run(
149
                    command,
150
                    stdout=subprocess.DEVNULL,
151
                    stderr=subprocess.DEVNULL,
152
                    stdin=subprocess.DEVNULL,
153
                    check=False,
154
                    timeout=30.0,
155
                )
156
            except (OSError, subprocess.SubprocessError):
157
                return None
158
            samples.append((time.perf_counter() - start) * 1000.0)
159
        return statistics.median(samples)
160
161
    direct = median_ms([str(real_git), "--version"])
162
    shimmed = median_ms([str(shim_git), "--version"])
163
    if direct is None or shimmed is None:
164
        return None
165
    return round(max(0.0, shimmed - direct), 3)
166
167
168
def read_sidecar_rows(sidecar: Path) -> list[dict]:
169
    """Parse sidecar JSONL rows, skipping blank or malformed lines."""
170
    try:
171
        text = Path(sidecar).read_text(encoding="utf-8")
172
    except (FileNotFoundError, OSError):
173
        return []
174
    rows: list[dict] = []
175
    for line in text.splitlines():
176
        stripped = line.strip()
177
        if not stripped:
178
            continue
179
        try:
180
            row = json.loads(stripped)
181
        except json.JSONDecodeError:
182
            continue
183
        if isinstance(row, dict):
184
            rows.append(row)
185
    return rows