Viewing files at commit ada5e175e5bfCommit detailsView latest

scripts/oakbench/environment.py 139 lines · 4.9 KB · python
1
"""Process environment policy for measured commands.
2
3
All measured subprocesses run through these environments so that author
4
identity, color suppression, and locale-ish noise are controlled in one place.
5
"""
6
7
from __future__ import annotations
8
9
import atexit
10
import os
11
import shutil
12
import tempfile
13
from pathlib import Path, PurePath
14
from typing import Any, Mapping
15
16
ENV_ISOLATION_VERSION = "3"
17
HOST_HOME_ENV_ISOLATION_VERSION = "3-host-home"
18
19
GIT_IDENTITY_KEYS = frozenset({"GIT_AUTHOR_NAME", "GIT_AUTHOR_EMAIL", "GIT_COMMITTER_NAME", "GIT_COMMITTER_EMAIL"})
20
21
22
def sanitize_git_environment(values: Mapping[str, str]) -> dict[str, str]:
23
    """Prevent host plumbing/configuration from redirecting measured commands.
24
25
    Explicit per-command scenario overrides may be applied after this boundary.
26
    Agent credential filtering is a separate, stricter policy.
27
    """
28
    env = {key: value for key, value in values.items()
29
           if not key.startswith("GIT_") or key in GIT_IDENTITY_KEYS}
30
    env.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1", GIT_NO_REPLACE_OBJECTS="1")
31
    return env
32
33
def benchmark_process_environment(values: Mapping[str, str]) -> dict[str, str]:
34
    """Deterministic engine measurement policy, also enforced after agent filtering.
35
36
    Oak's documented updater otherwise performs a network request after the
37
    first successful command in a fresh HOME, biasing randomized subject order.
38
    Natural first-use updater UX requires a separate measurement contract.
39
    """
40
    env = sanitize_git_environment(values)
41
    env["OAK_NO_UPDATE_CHECK"] = "1"
42
    return env
43
44
45
def host_home_environment(values: Mapping[str, str] | None = None) -> dict[str, str]:
46
    """Sanitize engine subprocesses while retaining caller credentials/config.
47
48
    Mount and external-command probes need the caller's HOME. This is not the
49
    isolated-HOME contract implemented by base_env, and must not share its ID.
50
    """
51
    env = benchmark_process_environment(os.environ if values is None else values)
52
    env["OAK_BENCH_ENV_ISOLATION_VERSION"] = HOST_HOME_ENV_ISOLATION_VERSION
53
    return env
54
55
56
_AUTO_BENCH_HOME: Path | None = None
57
_CLEANUP_REGISTERED = False
58
59
60
def _cleanup_auto_bench_home() -> None:
61
    if _AUTO_BENCH_HOME is not None:
62
        shutil.rmtree(_AUTO_BENCH_HOME, ignore_errors=True)
63
64
65
def _bench_home(home: Path | None = None) -> str:
66
    global _AUTO_BENCH_HOME, _CLEANUP_REGISTERED
67
    if home is None:
68
        path = Path(tempfile.gettempdir()) / f"oak-bench-home-{os.getpid()}"
69
        _AUTO_BENCH_HOME = path
70
        if not _CLEANUP_REGISTERED:
71
            atexit.register(_cleanup_auto_bench_home)
72
            _CLEANUP_REGISTERED = True
73
    else:
74
        path = home
75
    path.mkdir(mode=0o700, parents=True, exist_ok=True)
76
    (path / ".config").mkdir(exist_ok=True)
77
    (path / ".cache").mkdir(exist_ok=True)
78
    return str(path)
79
80
81
def base_env(
82
    author_name: str = "Oak Bench",
83
    author_email: str = "[email protected]",
84
    oak_author: str = "oakbench",
85
    home: Path | None = None,
86
) -> dict[str, str]:
87
    env = benchmark_process_environment(os.environ)
88
    bench_home = _bench_home(home)
89
    env.update(
90
        {
91
            "GIT_AUTHOR_NAME": author_name,
92
            "GIT_AUTHOR_EMAIL": author_email,
93
            "GIT_COMMITTER_NAME": author_name,
94
            "GIT_COMMITTER_EMAIL": author_email,
95
            "OAK_AUTHOR": oak_author,
96
            "NO_COLOR": "1",
97
            "CLICOLOR": "0",
98
            "HOME": bench_home,
99
            "XDG_CONFIG_HOME": str(Path(bench_home) / ".config"),
100
            "XDG_CACHE_HOME": str(Path(bench_home) / ".cache"),
101
            "ZDOTDIR": bench_home,
102
            "BASH_ENV": os.devnull,
103
            "ENV": os.devnull,
104
            "GIT_CONFIG_GLOBAL": os.devnull,
105
            "GIT_CONFIG_NOSYSTEM": "1",
106
            "LC_ALL": "C",
107
            "LANG": "C",
108
            "TZ": "UTC",
109
            "OAK_BENCH_ENV_ISOLATION_VERSION": ENV_ISOLATION_VERSION,
110
        }
111
    )
112
    return env
113
114
115
def probe_env(**kwargs: str) -> dict[str, str]:
116
    # The determinism probe intentionally does NOT suppress color: it measures
117
    # the tool's own non-TTY behavior. ANSI bytes in piped output are token
118
    # waste and break provider prompt caching.
119
    env = base_env(**kwargs)
120
    env.pop("NO_COLOR", None)
121
    env.pop("CLICOLOR", None)
122
    return env
123
124
125
def command_display(command: Any) -> str:
126
    """Human/token-countable form of a command or command sequence."""
127
    if (
128
        isinstance(command, list)
129
        and len(command) >= 3
130
        and command[0] in {"/bin/zsh", "zsh", "/bin/bash", "bash", "sh"}
131
        and command[1] in {"-c", "-lc"}
132
    ):
133
        return command[2]
134
    if isinstance(command, list) and command and all(isinstance(item, str) for item in command):
135
        executable = PurePath(command[0]).name if os.sep in command[0] else command[0]
136
        return " ".join([executable, *command[1:]])
137
    if isinstance(command, list):
138
        return " && ".join(command_display(item) for item in command)
139
    return str(command)