Viewing files at commit 3cd2661fdbe9Commit detailsView latest

scripts/oakbench/subjects.py 251 lines · 9.3 KB · python
1
"""Subject identity: which VCS binaries are under measurement.
2
3
Subjects have two identity layers (CONTEXT.md): the logical name used for
4
trend grouping, and binary+version+source-hash provenance recorded per run.
5
"""
6
7
from __future__ import annotations
8
9
import argparse
10
import hashlib
11
import os
12
import platform
13
import shutil
14
import subprocess
15
from dataclasses import dataclass
16
from pathlib import Path
17
from typing import Any
18
19
from oakbench.configio import load_subject_config
20
21
ROOT = Path(__file__).resolve().parents[2]
22
DEFAULT_OAK_REPO = Path(os.environ.get("OAK_REPO", ROOT.parent / "oak"))
23
24
25
@dataclass(frozen=True)
26
class Subject:
27
    name: str
28
    kind: str
29
    label: str
30
    bin: Path
31
    git_mode: str | None = None
32
    # Provenance of shim resolution: the configured path when it was the
33
    # macOS xcrun trampoline and was resolved to the real binary (or kept,
34
    # explicitly allowed, in which case git_shim is True).
35
    requested_bin: Path | None = None
36
    git_shim: bool = False
37
38
39
# macOS ships /usr/bin/git as an xcrun trampoline: every call re-resolves the
40
# developer directory before exec'ing the real binary (~3 ms per call on the
41
# benchmark workstation), and hashing it identifies the stub, not the Git
42
# that ran. Only the canonical stub location is treated as a shim.
43
XCRUN_SHIM_PATHS = frozenset({"/usr/bin/git"})
44
ALLOW_GIT_SHIM_ENV = "OAK_BENCH_ALLOW_GIT_SHIM"
45
46
47
def is_xcrun_git_shim(path: Path | str, system: str | None = None) -> bool:
48
    if (system or platform.system()) != "Darwin":
49
        return False
50
    try:
51
        resolved = str(Path(path).resolve())
52
    except OSError:
53
        resolved = str(path)
54
    return resolved in XCRUN_SHIM_PATHS or str(path) in XCRUN_SHIM_PATHS
55
56
57
def resolve_real_git(runner=subprocess.run) -> Path | None:
58
    """The real Git behind the xcrun shim, or None when it cannot be found.
59
60
    Order: ``xcrun --find git`` (honours DEVELOPER_DIR / xcode-select), then
61
    the selected developer directory, then Command Line Tools.
62
    """
63
    candidates: list[str] = []
64
    for command in (["/usr/bin/xcrun", "--find", "git"], ["/usr/bin/xcode-select", "-p"]):
65
        try:
66
            result = runner(command, capture_output=True, text=True, timeout=10, check=False)
67
        except (OSError, subprocess.TimeoutExpired):
68
            continue
69
        text = (result.stdout or "").strip()
70
        if result.returncode == 0 and text:
71
            candidates.append(text if command[-1] == "git" else str(Path(text) / "usr" / "bin" / "git"))
72
    candidates.append("/Library/Developer/CommandLineTools/usr/bin/git")
73
    for candidate in candidates:
74
        path = Path(candidate)
75
        if path.is_file() and os.access(path, os.X_OK) and not is_xcrun_git_shim(path, "Darwin"):
76
            return path.resolve()
77
    return None
78
79
80
def which_measured_git() -> str | None:
81
    """``shutil.which("git")`` with the xcrun shim resolved to the real Git.
82
83
    For auxiliary resolution sites; timed subjects go through load_subjects,
84
    which refuses an unresolvable shim instead of silently measuring it.
85
    """
86
    found = shutil.which("git")
87
    if found and is_xcrun_git_shim(found) and not allow_git_shim():
88
        real = resolve_real_git()
89
        return str(real) if real else found
90
    return found
91
92
93
def allow_git_shim(args: argparse.Namespace | None = None) -> bool:
94
    return bool(getattr(args, "allow_git_shim", False)) or os.environ.get(ALLOW_GIT_SHIM_ENV) == "1"
95
96
97
def resolve_git_subject_bin(path: Path, *, allow_shim: bool, system: str | None = None,
98
                            resolver=resolve_real_git) -> tuple[Path, Path | None, bool]:
99
    """Return (bin to execute, requested bin if resolved/kept, shim_measured).
100
101
    Refuses (SystemExit) to measure the shim unless explicitly allowed.
102
    """
103
    if not is_xcrun_git_shim(path, system):
104
        return path, None, False
105
    if allow_shim:
106
        return path, path, True
107
    real = resolver()
108
    if real is None:
109
        raise SystemExit(
110
            f"{path} is the macOS xcrun git shim and the real Git could not be resolved "
111
            "(xcrun --find git failed). Pass --git-bin <real git>, or set "
112
            f"{ALLOW_GIT_SHIM_ENV}=1 / --allow-git-shim to measure the shim deliberately.")
113
    return real, path, False
114
115
116
def resolve_subject_bin(raw: str | Path, override: Path | None = None) -> Path:
117
    if override is not None:
118
        path = override.expanduser()
119
        return path if path.is_absolute() else (ROOT / path).resolve()
120
121
    raw_text = str(raw)
122
    path = Path(raw_text).expanduser()
123
    if path.is_absolute():
124
        return path
125
126
    if "/" in raw_text or "\\" in raw_text:
127
        return (ROOT / path).resolve()
128
129
    found = shutil.which(raw_text)
130
    return Path(found) if found else path
131
132
133
def load_subjects(args: argparse.Namespace) -> list[Subject]:
134
    """Load subjects from config/subjects.toml with the standard CLI overrides.
135
136
    Expects args to carry: config, subjects, git_bin, oak_installed_bin,
137
    oak_local_bin — the shared flag set every lane defines.
138
    """
139
    data = load_subject_config(args.config)
140
    subjects: list[Subject] = []
141
    wanted = {item.strip() for item in args.subjects.split(",") if item.strip()} if args.subjects else None
142
    for name, raw in data["subjects"].items():
143
        if not raw.get("enabled", False) and (wanted is None or name not in wanted):
144
            continue
145
        override = None
146
        if name in ("git", "git_aa"):
147
            override = args.git_bin
148
        elif name == "oak_installed":
149
            override = args.oak_installed_bin
150
        elif name == "oak_local":
151
            override = args.oak_local_bin
152
        elif name == "oak_installed_aa":
153
            override = getattr(args, "oak_aa_bin", None)
154
        path = resolve_subject_bin(raw["bin"], override)
155
        requested, shim = None, False
156
        if raw["kind"] == "git":
157
            path, requested, shim = resolve_git_subject_bin(path, allow_shim=allow_git_shim(args))
158
        subjects.append(Subject(name=name, kind=raw["kind"], label=raw.get("label", name), bin=path,
159
                                requested_bin=requested, git_shim=shim))
160
161
    if wanted is not None:
162
        subjects = [subject for subject in subjects if subject.name in wanted]
163
164
    missing = [str(subject.bin) for subject in subjects if not subject.bin.exists()]
165
    if missing:
166
        raise SystemExit("Missing benchmark binaries:\n  " + "\n  ".join(missing))
167
    if not subjects:
168
        raise SystemExit("No enabled benchmark subjects")
169
    return subjects
170
171
172
def subject_version(subject: Subject) -> str:
173
    try:
174
        return subprocess.check_output(
175
            [str(subject.bin), "--version"], text=True, stderr=subprocess.STDOUT, timeout=5
176
        ).strip()
177
    except Exception as exc:  # noqa: BLE001 - metadata only
178
        return f"unknown ({exc})"
179
180
181
def subject_role(subject: Subject) -> str:
182
    if subject.name == "oak_installed":
183
        return "oak_release_latest_installed"
184
    if subject.name == "oak_local":
185
        return "oak_current_changeset"
186
    if subject.name == "oak_main":
187
        return "oak_main_clean_build"
188
    if subject.name == "oak_installed_aa":
189
        return "oak_baseline_aa_control"
190
    return subject.kind
191
192
193
def binary_sha256(path: Path) -> str | None:
194
    try:
195
        digest = hashlib.sha256()
196
        with path.open("rb") as fh:
197
            for chunk in iter(lambda: fh.read(1024 * 1024), b""):
198
                digest.update(chunk)
199
        return digest.hexdigest()
200
    except OSError:
201
        return None
202
203
204
def subject_details(subjects: list[Subject]) -> dict[str, dict[str, Any]]:
205
    details = {}
206
    for subject in subjects:
207
        item = {
208
            "kind": subject.kind,
209
            "label": subject.label,
210
            "bin": str(subject.bin),
211
            "binary_sha256": binary_sha256(subject.bin),
212
            "version": subject_version(subject),
213
            "role": subject_role(subject),
214
        }
215
        if subject.kind == "git":
216
            item.update(requested_bin=str(subject.requested_bin) if subject.requested_bin else None,
217
                        git_shim_measured=subject.git_shim,
218
                        git_shim_resolved=subject.requested_bin is not None and not subject.git_shim,
219
                        git_config_profile=git_config_profile(subject.git_mode))
220
        details[subject.name] = item
221
    return details
222
223
224
def git_config_profile(mode: str | None) -> str:
225
    """Row label that separates stock Git from tuned configurations."""
226
    return "stock" if not mode else ("scalar_recommended" if mode == "scalar" else f"tuned_{mode}")
227
228
229
def subject_versions(subjects: list[Subject]) -> dict[str, str]:
230
    return {subject.name: subject_version(subject) for subject in subjects}
231
232
233
def command_text(command: list[str], cwd: Path | None = None) -> str | None:
234
    try:
235
        return subprocess.check_output(
236
            command, cwd=cwd, text=True, stderr=subprocess.STDOUT, timeout=10
237
        ).strip()
238
    except Exception:
239
        return None
240
241
242
def source_metadata(oak_repo: Path) -> dict[str, Any]:
243
    """Oak-source provenance for tying rows back to a changeset, without using Git."""
244
    oak_bin = shutil.which("oak")
245
    if not oak_bin or not oak_repo.exists():
246
        return {"oak_repo": str(oak_repo), "oak_hash": None, "oak_status": None}
247
    return {
248
        "oak_repo": str(oak_repo),
249
        "oak_hash": command_text([oak_bin, "hash"], cwd=oak_repo),
250
        "oak_status": command_text([oak_bin, "status"], cwd=oak_repo),
251
    }