Viewing files at commit 54b9920be8dfCommit detailsView latest

scripts/oakbench/execution.py 185 lines · 6.1 KB · python
1
"""Timed command execution with admitted-output capture.
2
3
This is the measurement primitive every CLI lane shares: run a command under a
4
monotonic clock, capture full output bytes to disk, and admit only a capped
5
character window into token estimates (full byte counts are always recorded).
6
7
Memory: the child's peak RSS is captured at reap time via os.wait4 (rusage),
8
so no wrapper binary or sampling thread perturbs the timing. ru_maxrss units
9
differ by platform (bytes on macOS, KiB on Linux); peak_rss_bytes is always
10
bytes. None means unmeasured (ADR-0002), never zero — e.g. on platforms
11
without wait4.
12
"""
13
14
from __future__ import annotations
15
16
import os
17
import signal
18
import subprocess
19
import sys
20
import tempfile
21
import time
22
from dataclasses import dataclass
23
from pathlib import Path
24
from typing import Any, Optional
25
26
_HAS_WAIT4 = hasattr(os, "wait4")
27
PEAK_RSS_SOURCE = "os_wait4_ru_maxrss" if _HAS_WAIT4 else None
28
29
30
@dataclass(frozen=True)
31
class TimedCapture:
32
    elapsed_ms: float
33
    returncode: int
34
    stdout_text: str
35
    stdout_bytes: int
36
    stdout_truncated: bool
37
    stderr_text: str
38
    stderr_bytes: int
39
    stderr_truncated: bool
40
    peak_rss_bytes: Optional[int] = None
41
    stdout_full_text: Optional[str] = None
42
    stdout_full_text_truncated: Optional[bool] = None
43
    stderr_full_text: Optional[str] = None
44
    stderr_full_text_truncated: Optional[bool] = None
45
    timed_out: bool = False
46
47
48
def _stop_owned_process(proc: subprocess.Popen[Any]) -> None:
49
    """Stop the timed child and its owned process group, then always reap it."""
50
    try:
51
        if os.name == "posix":
52
            os.killpg(proc.pid, signal.SIGTERM)
53
        else:
54
            proc.terminate()
55
    except (OSError, ProcessLookupError):
56
        pass
57
    try:
58
        proc.wait(timeout=1.0)
59
        return
60
    except subprocess.TimeoutExpired:
61
        pass
62
    try:
63
        if os.name == "posix":
64
            os.killpg(proc.pid, signal.SIGKILL)
65
        else:
66
            proc.kill()
67
    except (OSError, ProcessLookupError):
68
        pass
69
    proc.wait()
70
71
72
class _RusagePopen(subprocess.Popen):
73
    """Popen that records the child's rusage when it is reaped.
74
75
    Overriding _try_wait keeps a single wait4 syscall as both the reap and the
76
    measurement; there is no race against the kernel discarding rusage and no
77
    extra process in the timed path.
78
    """
79
80
    rusage = None
81
82
    def _try_wait(self, wait_flags):  # type: ignore[override]
83
        try:
84
            pid, sts, rusage = os.wait4(self.pid, wait_flags)
85
        except ChildProcessError:
86
            pid = self.pid
87
            sts = 0
88
        else:
89
            if pid == self.pid:
90
                self.rusage = rusage
91
        return (pid, sts)
92
93
94
def _peak_rss_bytes(rusage: Any) -> Optional[int]:
95
    if rusage is None:
96
        return None
97
    maxrss = getattr(rusage, "ru_maxrss", None)
98
    if not maxrss or maxrss < 0:
99
        return None
100
    # macOS reports bytes; Linux (and most other Unixes) report KiB.
101
    return int(maxrss) if sys.platform == "darwin" else int(maxrss) * 1024
102
103
104
def read_admitted_text(handle: Any, char_limit: int) -> tuple[str, int, bool]:
105
    handle.flush()
106
    handle.seek(0, os.SEEK_END)
107
    byte_count = handle.tell()
108
    handle.seek(0)
109
    if char_limit <= 0:
110
        return "", byte_count, byte_count > 0
111
    raw = handle.read(char_limit * 4 + 16)
112
    text = raw.decode("utf-8", "replace")[:char_limit]
113
    return text, byte_count, byte_count > len(text.encode("utf-8", "replace"))
114
115
116
def read_full_text_with_byte_limit(handle: Any, byte_limit: int) -> tuple[str | None, bool]:
117
    handle.flush()
118
    handle.seek(0, os.SEEK_END)
119
    byte_count = handle.tell()
120
    if byte_limit < 0:
121
        return None, byte_count > 0
122
    if byte_count > byte_limit:
123
        return None, True
124
    handle.seek(0)
125
    return handle.read().decode("utf-8", "replace"), False
126
127
128
def run_timed(
129
    command: list[str],
130
    cwd: Path,
131
    env: dict[str, str],
132
    admitted_output_chars: int,
133
    full_output_bytes: int | None = None,
134
    timeout_seconds: float | None = None,
135
) -> TimedCapture:
136
    if timeout_seconds is not None and timeout_seconds <= 0:
137
        raise ValueError("timeout_seconds must be positive")
138
    popen_cls = _RusagePopen if _HAS_WAIT4 else subprocess.Popen
139
    start = time.perf_counter()
140
    with tempfile.TemporaryFile() as stdout_fh, tempfile.TemporaryFile() as stderr_fh:
141
        proc = popen_cls(
142
            command,
143
            cwd=cwd,
144
            env=env,
145
            stdout=stdout_fh,
146
            stderr=stderr_fh,
147
            start_new_session=timeout_seconds is not None and os.name == "posix",
148
        )
149
        timed_out = False
150
        try:
151
            returncode = proc.wait(timeout=timeout_seconds)
152
        except subprocess.TimeoutExpired:
153
            timed_out = True
154
            _stop_owned_process(proc)
155
            returncode = 124
156
        elapsed_ms = (time.perf_counter() - start) * 1000
157
        stdout_text, stdout_bytes, stdout_truncated = read_admitted_text(stdout_fh, admitted_output_chars)
158
        stderr_text, stderr_bytes, stderr_truncated = read_admitted_text(stderr_fh, admitted_output_chars)
159
        stdout_full_text = None
160
        stdout_full_text_truncated = None
161
        stderr_full_text = None
162
        stderr_full_text_truncated = None
163
        if full_output_bytes is not None:
164
            stdout_full_text, stdout_full_text_truncated = read_full_text_with_byte_limit(
165
                stdout_fh, full_output_bytes
166
            )
167
            stderr_full_text, stderr_full_text_truncated = read_full_text_with_byte_limit(
168
                stderr_fh, full_output_bytes
169
            )
170
    return TimedCapture(
171
        elapsed_ms=elapsed_ms,
172
        returncode=returncode,
173
        stdout_text=stdout_text,
174
        stdout_bytes=stdout_bytes,
175
        stdout_truncated=stdout_truncated,
176
        stderr_text=stderr_text,
177
        stderr_bytes=stderr_bytes,
178
        stderr_truncated=stderr_truncated,
179
        peak_rss_bytes=_peak_rss_bytes(getattr(proc, "rusage", None)),
180
        stdout_full_text=stdout_full_text,
181
        stdout_full_text_truncated=stdout_full_text_truncated,
182
        stderr_full_text=stderr_full_text,
183
        stderr_full_text_truncated=stderr_full_text_truncated,
184
        timed_out=timed_out,
185
    )