Viewing files at commit c10cefe7abe5Commit detailsView latest

scripts/oakbench/execution.py 148 lines · 5.0 KB · python
1
"""Timed command execution with admitted-output capture.
2
3
This is the measurement primitive every CLI lane shares: run a command under a
4
monotonic clock, capture full output bytes to disk, and admit only a capped
5
character window into token estimates (full byte counts are always recorded).
6
7
Memory: the child's peak RSS is captured at reap time via os.wait4 (rusage),
8
so no wrapper binary or sampling thread perturbs the timing. ru_maxrss units
9
differ by platform (bytes on macOS, KiB on Linux); peak_rss_bytes is always
10
bytes. None means unmeasured (ADR-0002), never zero — e.g. on platforms
11
without wait4.
12
"""
13
14
from __future__ import annotations
15
16
import os
17
import subprocess
18
import sys
19
import tempfile
20
import time
21
from dataclasses import dataclass
22
from pathlib import Path
23
from typing import Any, Optional
24
25
_HAS_WAIT4 = hasattr(os, "wait4")
26
PEAK_RSS_SOURCE = "os_wait4_ru_maxrss" if _HAS_WAIT4 else None
27
28
29
@dataclass(frozen=True)
30
class TimedCapture:
31
    elapsed_ms: float
32
    returncode: int
33
    stdout_text: str
34
    stdout_bytes: int
35
    stdout_truncated: bool
36
    stderr_text: str
37
    stderr_bytes: int
38
    stderr_truncated: bool
39
    peak_rss_bytes: Optional[int] = None
40
    stdout_full_text: Optional[str] = None
41
    stdout_full_text_truncated: Optional[bool] = None
42
    stderr_full_text: Optional[str] = None
43
    stderr_full_text_truncated: Optional[bool] = None
44
45
46
class _RusagePopen(subprocess.Popen):
47
    """Popen that records the child's rusage when it is reaped.
48
49
    Overriding _try_wait keeps a single wait4 syscall as both the reap and the
50
    measurement; there is no race against the kernel discarding rusage and no
51
    extra process in the timed path.
52
    """
53
54
    rusage = None
55
56
    def _try_wait(self, wait_flags):  # type: ignore[override]
57
        try:
58
            pid, sts, rusage = os.wait4(self.pid, wait_flags)
59
        except ChildProcessError:
60
            pid = self.pid
61
            sts = 0
62
        else:
63
            if pid == self.pid:
64
                self.rusage = rusage
65
        return (pid, sts)
66
67
68
def _peak_rss_bytes(rusage: Any) -> Optional[int]:
69
    if rusage is None:
70
        return None
71
    maxrss = getattr(rusage, "ru_maxrss", None)
72
    if not maxrss or maxrss < 0:
73
        return None
74
    # macOS reports bytes; Linux (and most other Unixes) report KiB.
75
    return int(maxrss) if sys.platform == "darwin" else int(maxrss) * 1024
76
77
78
def read_admitted_text(handle: Any, char_limit: int) -> tuple[str, int, bool]:
79
    handle.flush()
80
    handle.seek(0, os.SEEK_END)
81
    byte_count = handle.tell()
82
    handle.seek(0)
83
    if char_limit <= 0:
84
        return "", byte_count, byte_count > 0
85
    raw = handle.read(char_limit * 4 + 16)
86
    text = raw.decode("utf-8", "replace")[:char_limit]
87
    return text, byte_count, byte_count > len(text.encode("utf-8", "replace"))
88
89
90
def read_full_text_with_byte_limit(handle: Any, byte_limit: int) -> tuple[str | None, bool]:
91
    handle.flush()
92
    handle.seek(0, os.SEEK_END)
93
    byte_count = handle.tell()
94
    if byte_limit < 0:
95
        return None, byte_count > 0
96
    if byte_count > byte_limit:
97
        return None, True
98
    handle.seek(0)
99
    return handle.read().decode("utf-8", "replace"), False
100
101
102
def run_timed(
103
    command: list[str],
104
    cwd: Path,
105
    env: dict[str, str],
106
    admitted_output_chars: int,
107
    full_output_bytes: int | None = None,
108
) -> TimedCapture:
109
    popen_cls = _RusagePopen if _HAS_WAIT4 else subprocess.Popen
110
    start = time.perf_counter()
111
    with tempfile.TemporaryFile() as stdout_fh, tempfile.TemporaryFile() as stderr_fh:
112
        proc = popen_cls(
113
            command,
114
            cwd=cwd,
115
            env=env,
116
            stdout=stdout_fh,
117
            stderr=stderr_fh,
118
        )
119
        returncode = proc.wait()
120
        elapsed_ms = (time.perf_counter() - start) * 1000
121
        stdout_text, stdout_bytes, stdout_truncated = read_admitted_text(stdout_fh, admitted_output_chars)
122
        stderr_text, stderr_bytes, stderr_truncated = read_admitted_text(stderr_fh, admitted_output_chars)
123
        stdout_full_text = None
124
        stdout_full_text_truncated = None
125
        stderr_full_text = None
126
        stderr_full_text_truncated = None
127
        if full_output_bytes is not None:
128
            stdout_full_text, stdout_full_text_truncated = read_full_text_with_byte_limit(
129
                stdout_fh, full_output_bytes
130
            )
131
            stderr_full_text, stderr_full_text_truncated = read_full_text_with_byte_limit(
132
                stderr_fh, full_output_bytes
133
            )
134
    return TimedCapture(
135
        elapsed_ms=elapsed_ms,
136
        returncode=returncode,
137
        stdout_text=stdout_text,
138
        stdout_bytes=stdout_bytes,
139
        stdout_truncated=stdout_truncated,
140
        stderr_text=stderr_text,
141
        stderr_bytes=stderr_bytes,
142
        stderr_truncated=stderr_truncated,
143
        peak_rss_bytes=_peak_rss_bytes(getattr(proc, "rusage", None)),
144
        stdout_full_text=stdout_full_text,
145
        stdout_full_text_truncated=stdout_full_text_truncated,
146
        stderr_full_text=stderr_full_text,
147
        stderr_full_text_truncated=stderr_full_text_truncated,
148
    )