| 1 | """PATH shim for VCS-call attribution in real-agent runs.
|
| 2 |
|
| 3 | Real agent CLIs do not always expose per-tool-call timestamps in their
|
| 4 | streams. The shim makes VCS timing observable regardless of adapter: wrapper
|
| 5 | executables named after the VCS binaries (git/oak) are written into a
|
| 6 | directory that the harness prepends to the agent subprocess PATH. Each
|
| 7 | wrapper runs the real binary with inherited stdio (agent behavior unchanged),
|
| 8 | then appends one JSON line to a sidecar JSONL with keys directly consumable
|
| 9 | by oakbench.thrash: command, start_ms, end_ms, elapsed_ms, returncode.
|
| 10 |
|
| 11 | Shim overhead is measured per run via measure_shim_overhead and recorded as a
|
| 12 | field. It is never subtracted from measurements (ADR-0002: record the
|
| 13 | instrument's cost, do not fabricate corrected numbers).
|
| 14 | """
|
| 15 |
|
| 16 | from __future__ import annotations
|
| 17 |
|
| 18 | import json
|
| 19 | import statistics
|
| 20 | import subprocess
|
| 21 | import sys
|
| 22 | import time
|
| 23 | from pathlib import Path
|
| 24 |
|
| 25 | # The wrapper intentionally uses subprocess.run rather than os.exec*: exec
|
| 26 | # replaces the wrapper process, so there would be nobody left to record
|
| 27 | # end_ns/returncode. stdio is inherited (no capture) and the child shares the
|
| 28 | # wrapper's process-group defaults, so interactive prompts and Ctrl-C reach
|
| 29 | # the real binary unchanged; a KeyboardInterrupt in the wrapper itself exits
|
| 30 | # 130 like a shell would.
|
| 31 | _WRAPPER_TEMPLATE = '''#!{python}
|
| 32 | """Auto-generated VCS shim wrapper. Records call timing, then runs the real binary."""
|
| 33 | import fcntl
|
| 34 | import json
|
| 35 | import os
|
| 36 | import subprocess
|
| 37 | import sys
|
| 38 | import time
|
| 39 |
|
| 40 | REAL = {real}
|
| 41 | SIDECAR = {sidecar}
|
| 42 | TOOL = {tool}
|
| 43 |
|
| 44 |
|
| 45 | def write_all(fd, data):
|
| 46 | written_total = 0
|
| 47 | while written_total < len(data):
|
| 48 | written = os.write(fd, data[written_total:])
|
| 49 | if written == 0:
|
| 50 | raise OSError("os.write returned 0 bytes")
|
| 51 | written_total += written
|
| 52 |
|
| 53 |
|
| 54 | def main() -> int:
|
| 55 | argv = sys.argv[1:]
|
| 56 | start_ns = time.time_ns()
|
| 57 | try:
|
| 58 | rc = subprocess.run([REAL, *argv]).returncode
|
| 59 | except KeyboardInterrupt:
|
| 60 | rc = 130
|
| 61 | except FileNotFoundError:
|
| 62 | rc = 127
|
| 63 | end_ns = time.time_ns()
|
| 64 | row = {{
|
| 65 | "command": " ".join([TOOL, *argv]) if argv else TOOL,
|
| 66 | "argv": argv,
|
| 67 | "start_ms": start_ns / 1e6,
|
| 68 | "end_ms": end_ns / 1e6,
|
| 69 | "elapsed_ms": (end_ns - start_ns) / 1e6,
|
| 70 | "returncode": rc,
|
| 71 | "shim": True,
|
| 72 | }}
|
| 73 | data = (json.dumps(row, sort_keys=True) + "\\n").encode("utf-8")
|
| 74 | try:
|
| 75 | fd = os.open(SIDECAR, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o644)
|
| 76 | try:
|
| 77 | fcntl.flock(fd, fcntl.LOCK_EX)
|
| 78 | # With O_APPEND plus the exclusive lock, parallel shim calls
|
| 79 | # cannot interleave rows even if the OS accepts only part of the
|
| 80 | # buffer per syscall.
|
| 81 | write_all(fd, data)
|
| 82 | finally:
|
| 83 | try:
|
| 84 | fcntl.flock(fd, fcntl.LOCK_UN)
|
| 85 | finally:
|
| 86 | os.close(fd)
|
| 87 | except OSError:
|
| 88 | # Recording must never change agent-visible behavior; a lost row is
|
| 89 | # an unmeasured call, not a failed agent command.
|
| 90 | pass
|
| 91 | return rc
|
| 92 |
|
| 93 |
|
| 94 | if __name__ == "__main__":
|
| 95 | raise SystemExit(main())
|
| 96 | '''
|
| 97 |
|
| 98 |
|
| 99 | def create_shim_dir(target_dir: Path, *, real_binaries: dict[str, str], sidecar_path: Path) -> Path:
|
| 100 | """Write executable wrapper scripts into target_dir and return it.
|
| 101 |
|
| 102 | One wrapper per entry in real_binaries (e.g. {"git": "/usr/bin/git",
|
| 103 | "oak": "/path/to/oak"}); names without a real binary get no wrapper, so
|
| 104 | PATH lookups for them behave exactly as without the shim. The real-binary
|
| 105 | absolute path and the sidecar path are baked into the wrapper at creation
|
| 106 | time, so the wrapper never re-resolves PATH (no recursion risk).
|
| 107 | """
|
| 108 | target_dir = Path(target_dir)
|
| 109 | target_dir.mkdir(parents=True, exist_ok=True)
|
| 110 | sidecar = Path(sidecar_path).expanduser()
|
| 111 | sidecar.parent.mkdir(parents=True, exist_ok=True)
|
| 112 | sidecar.touch(exist_ok=True)
|
| 113 | for tool, real in real_binaries.items():
|
| 114 | if not real:
|
| 115 | continue
|
| 116 | real_path = Path(real).expanduser()
|
| 117 | script = _WRAPPER_TEMPLATE.format(
|
| 118 | python=sys.executable,
|
| 119 | real=json.dumps(str(real_path)),
|
| 120 | sidecar=json.dumps(str(sidecar)),
|
| 121 | tool=json.dumps(str(tool)),
|
| 122 | )
|
| 123 | wrapper = target_dir / str(tool)
|
| 124 | wrapper.write_text(script)
|
| 125 | wrapper.chmod(0o755)
|
| 126 | return target_dir
|
| 127 |
|
| 128 |
|
| 129 | def measure_shim_overhead(shim_dir: Path, sidecar: Path, real_git: str, n: int = 5) -> float | None:
|
| 130 | """Median extra ms of `shim git --version` over direct `git --version`.
|
| 131 |
|
| 132 | Calibration rows for the shimmed runs land in `sidecar` (the path baked
|
| 133 | into the wrappers); callers that reuse the sidecar for agent attribution
|
| 134 | should measure overhead first and reset the file. The result is recorded
|
| 135 | alongside measurements, never subtracted from them, and is clamped at 0
|
| 136 | (timer noise can make the shim appear faster; negative overhead is
|
| 137 | fabrication). Returns None when overhead could not be measured.
|
| 138 | """
|
| 139 | shim_git = Path(shim_dir) / "git"
|
| 140 | if not real_git or not shim_git.exists():
|
| 141 | return None
|
| 142 |
|
| 143 | def median_ms(command: list[str]) -> float | None:
|
| 144 | samples: list[float] = []
|
| 145 | for _ in range(max(1, n)):
|
| 146 | start = time.perf_counter()
|
| 147 | try:
|
| 148 | subprocess.run(
|
| 149 | command,
|
| 150 | stdout=subprocess.DEVNULL,
|
| 151 | stderr=subprocess.DEVNULL,
|
| 152 | stdin=subprocess.DEVNULL,
|
| 153 | check=False,
|
| 154 | timeout=30.0,
|
| 155 | )
|
| 156 | except (OSError, subprocess.SubprocessError):
|
| 157 | return None
|
| 158 | samples.append((time.perf_counter() - start) * 1000.0)
|
| 159 | return statistics.median(samples)
|
| 160 |
|
| 161 | direct = median_ms([str(real_git), "--version"])
|
| 162 | shimmed = median_ms([str(shim_git), "--version"])
|
| 163 | if direct is None or shimmed is None:
|
| 164 | return None
|
| 165 | return round(max(0.0, shimmed - direct), 3)
|
| 166 |
|
| 167 |
|
| 168 | def read_sidecar_rows(sidecar: Path) -> list[dict]:
|
| 169 | """Parse sidecar JSONL rows, skipping blank or malformed lines."""
|
| 170 | try:
|
| 171 | text = Path(sidecar).read_text(encoding="utf-8")
|
| 172 | except (FileNotFoundError, OSError):
|
| 173 | return []
|
| 174 | rows: list[dict] = []
|
| 175 | for line in text.splitlines():
|
| 176 | stripped = line.strip()
|
| 177 | if not stripped:
|
| 178 | continue
|
| 179 | try:
|
| 180 | row = json.loads(stripped)
|
| 181 | except json.JSONDecodeError:
|
| 182 | continue
|
| 183 | if isinstance(row, dict):
|
| 184 | rows.append(row)
|
| 185 | return rows
|