| 1 | """Subject identity: which VCS binaries are under measurement.
|
| 2 |
|
| 3 | Subjects have two identity layers (CONTEXT.md): the logical name used for
|
| 4 | trend grouping, and binary+version+source-hash provenance recorded per run.
|
| 5 | """
|
| 6 |
|
| 7 | from __future__ import annotations
|
| 8 |
|
| 9 | import argparse
|
| 10 | import hashlib
|
| 11 | import os
|
| 12 | import platform
|
| 13 | import shutil
|
| 14 | import subprocess
|
| 15 | from dataclasses import dataclass
|
| 16 | from pathlib import Path
|
| 17 | from typing import Any
|
| 18 |
|
| 19 | from oakbench.configio import load_subject_config
|
| 20 |
|
| 21 | ROOT = Path(__file__).resolve().parents[2]
|
| 22 | DEFAULT_OAK_REPO = Path(os.environ.get("OAK_REPO", ROOT.parent / "oak"))
|
| 23 |
|
| 24 |
|
| 25 | @dataclass(frozen=True)
|
| 26 | class Subject:
|
| 27 | name: str
|
| 28 | kind: str
|
| 29 | label: str
|
| 30 | bin: Path
|
| 31 | git_mode: str | None = None
|
| 32 | # Provenance of shim resolution: the configured path when it was the
|
| 33 | # macOS xcrun trampoline and was resolved to the real binary (or kept,
|
| 34 | # explicitly allowed, in which case git_shim is True).
|
| 35 | requested_bin: Path | None = None
|
| 36 | git_shim: bool = False
|
| 37 |
|
| 38 |
|
| 39 | # macOS ships /usr/bin/git as an xcrun trampoline: every call re-resolves the
|
| 40 | # developer directory before exec'ing the real binary (~3 ms per call on the
|
| 41 | # benchmark workstation), and hashing it identifies the stub, not the Git
|
| 42 | # that ran. Only the canonical stub location is treated as a shim.
|
| 43 | XCRUN_SHIM_PATHS = frozenset({"/usr/bin/git"})
|
| 44 | ALLOW_GIT_SHIM_ENV = "OAK_BENCH_ALLOW_GIT_SHIM"
|
| 45 |
|
| 46 |
|
| 47 | def is_xcrun_git_shim(path: Path | str, system: str | None = None) -> bool:
|
| 48 | if (system or platform.system()) != "Darwin":
|
| 49 | return False
|
| 50 | try:
|
| 51 | resolved = str(Path(path).resolve())
|
| 52 | except OSError:
|
| 53 | resolved = str(path)
|
| 54 | return resolved in XCRUN_SHIM_PATHS or str(path) in XCRUN_SHIM_PATHS
|
| 55 |
|
| 56 |
|
| 57 | def resolve_real_git(runner=subprocess.run) -> Path | None:
|
| 58 | """The real Git behind the xcrun shim, or None when it cannot be found.
|
| 59 |
|
| 60 | Order: ``xcrun --find git`` (honours DEVELOPER_DIR / xcode-select), then
|
| 61 | the selected developer directory, then Command Line Tools.
|
| 62 | """
|
| 63 | candidates: list[str] = []
|
| 64 | for command in (["/usr/bin/xcrun", "--find", "git"], ["/usr/bin/xcode-select", "-p"]):
|
| 65 | try:
|
| 66 | result = runner(command, capture_output=True, text=True, timeout=10, check=False)
|
| 67 | except (OSError, subprocess.TimeoutExpired):
|
| 68 | continue
|
| 69 | text = (result.stdout or "").strip()
|
| 70 | if result.returncode == 0 and text:
|
| 71 | candidates.append(text if command[-1] == "git" else str(Path(text) / "usr" / "bin" / "git"))
|
| 72 | candidates.append("/Library/Developer/CommandLineTools/usr/bin/git")
|
| 73 | for candidate in candidates:
|
| 74 | path = Path(candidate)
|
| 75 | if path.is_file() and os.access(path, os.X_OK) and not is_xcrun_git_shim(path, "Darwin"):
|
| 76 | return path.resolve()
|
| 77 | return None
|
| 78 |
|
| 79 |
|
| 80 | def which_measured_git() -> str | None:
|
| 81 | """``shutil.which("git")`` with the xcrun shim resolved to the real Git.
|
| 82 |
|
| 83 | For auxiliary resolution sites; timed subjects go through load_subjects,
|
| 84 | which refuses an unresolvable shim instead of silently measuring it.
|
| 85 | """
|
| 86 | found = shutil.which("git")
|
| 87 | if found and is_xcrun_git_shim(found) and not allow_git_shim():
|
| 88 | real = resolve_real_git()
|
| 89 | return str(real) if real else found
|
| 90 | return found
|
| 91 |
|
| 92 |
|
| 93 | def allow_git_shim(args: argparse.Namespace | None = None) -> bool:
|
| 94 | return bool(getattr(args, "allow_git_shim", False)) or os.environ.get(ALLOW_GIT_SHIM_ENV) == "1"
|
| 95 |
|
| 96 |
|
| 97 | def resolve_git_subject_bin(path: Path, *, allow_shim: bool, system: str | None = None,
|
| 98 | resolver=resolve_real_git) -> tuple[Path, Path | None, bool]:
|
| 99 | """Return (bin to execute, requested bin if resolved/kept, shim_measured).
|
| 100 |
|
| 101 | Refuses (SystemExit) to measure the shim unless explicitly allowed.
|
| 102 | """
|
| 103 | if not is_xcrun_git_shim(path, system):
|
| 104 | return path, None, False
|
| 105 | if allow_shim:
|
| 106 | return path, path, True
|
| 107 | real = resolver()
|
| 108 | if real is None:
|
| 109 | raise SystemExit(
|
| 110 | f"{path} is the macOS xcrun git shim and the real Git could not be resolved "
|
| 111 | "(xcrun --find git failed). Pass --git-bin <real git>, or set "
|
| 112 | f"{ALLOW_GIT_SHIM_ENV}=1 / --allow-git-shim to measure the shim deliberately.")
|
| 113 | return real, path, False
|
| 114 |
|
| 115 |
|
| 116 | def resolve_subject_bin(raw: str | Path, override: Path | None = None) -> Path:
|
| 117 | if override is not None:
|
| 118 | path = override.expanduser()
|
| 119 | return path if path.is_absolute() else (ROOT / path).resolve()
|
| 120 |
|
| 121 | raw_text = str(raw)
|
| 122 | path = Path(raw_text).expanduser()
|
| 123 | if path.is_absolute():
|
| 124 | return path
|
| 125 |
|
| 126 | if "/" in raw_text or "\\" in raw_text:
|
| 127 | return (ROOT / path).resolve()
|
| 128 |
|
| 129 | found = shutil.which(raw_text)
|
| 130 | return Path(found) if found else path
|
| 131 |
|
| 132 |
|
| 133 | def load_subjects(args: argparse.Namespace) -> list[Subject]:
|
| 134 | """Load subjects from config/subjects.toml with the standard CLI overrides.
|
| 135 |
|
| 136 | Expects args to carry: config, subjects, git_bin, oak_installed_bin,
|
| 137 | oak_local_bin — the shared flag set every lane defines.
|
| 138 | """
|
| 139 | data = load_subject_config(args.config)
|
| 140 | subjects: list[Subject] = []
|
| 141 | wanted = {item.strip() for item in args.subjects.split(",") if item.strip()} if args.subjects else None
|
| 142 | for name, raw in data["subjects"].items():
|
| 143 | if not raw.get("enabled", False) and (wanted is None or name not in wanted):
|
| 144 | continue
|
| 145 | override = None
|
| 146 | if name in ("git", "git_aa"):
|
| 147 | override = args.git_bin
|
| 148 | elif name == "oak_installed":
|
| 149 | override = args.oak_installed_bin
|
| 150 | elif name == "oak_local":
|
| 151 | override = args.oak_local_bin
|
| 152 | elif name == "oak_installed_aa":
|
| 153 | override = getattr(args, "oak_aa_bin", None)
|
| 154 | path = resolve_subject_bin(raw["bin"], override)
|
| 155 | requested, shim = None, False
|
| 156 | if raw["kind"] == "git":
|
| 157 | path, requested, shim = resolve_git_subject_bin(path, allow_shim=allow_git_shim(args))
|
| 158 | subjects.append(Subject(name=name, kind=raw["kind"], label=raw.get("label", name), bin=path,
|
| 159 | requested_bin=requested, git_shim=shim))
|
| 160 |
|
| 161 | if wanted is not None:
|
| 162 | subjects = [subject for subject in subjects if subject.name in wanted]
|
| 163 |
|
| 164 | missing = [str(subject.bin) for subject in subjects if not subject.bin.exists()]
|
| 165 | if missing:
|
| 166 | raise SystemExit("Missing benchmark binaries:\n " + "\n ".join(missing))
|
| 167 | if not subjects:
|
| 168 | raise SystemExit("No enabled benchmark subjects")
|
| 169 | return subjects
|
| 170 |
|
| 171 |
|
| 172 | def subject_version(subject: Subject) -> str:
|
| 173 | try:
|
| 174 | return subprocess.check_output(
|
| 175 | [str(subject.bin), "--version"], text=True, stderr=subprocess.STDOUT, timeout=5
|
| 176 | ).strip()
|
| 177 | except Exception as exc: # noqa: BLE001 - metadata only
|
| 178 | return f"unknown ({exc})"
|
| 179 |
|
| 180 |
|
| 181 | def subject_role(subject: Subject) -> str:
|
| 182 | if subject.name == "oak_installed":
|
| 183 | return "oak_release_latest_installed"
|
| 184 | if subject.name == "oak_local":
|
| 185 | return "oak_current_changeset"
|
| 186 | if subject.name == "oak_main":
|
| 187 | return "oak_main_clean_build"
|
| 188 | if subject.name == "oak_installed_aa":
|
| 189 | return "oak_baseline_aa_control"
|
| 190 | return subject.kind
|
| 191 |
|
| 192 |
|
| 193 | def binary_sha256(path: Path) -> str | None:
|
| 194 | try:
|
| 195 | digest = hashlib.sha256()
|
| 196 | with path.open("rb") as fh:
|
| 197 | for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
| 198 | digest.update(chunk)
|
| 199 | return digest.hexdigest()
|
| 200 | except OSError:
|
| 201 | return None
|
| 202 |
|
| 203 |
|
| 204 | def subject_details(subjects: list[Subject]) -> dict[str, dict[str, Any]]:
|
| 205 | details = {}
|
| 206 | for subject in subjects:
|
| 207 | item = {
|
| 208 | "kind": subject.kind,
|
| 209 | "label": subject.label,
|
| 210 | "bin": str(subject.bin),
|
| 211 | "binary_sha256": binary_sha256(subject.bin),
|
| 212 | "version": subject_version(subject),
|
| 213 | "role": subject_role(subject),
|
| 214 | }
|
| 215 | if subject.kind == "git":
|
| 216 | item.update(requested_bin=str(subject.requested_bin) if subject.requested_bin else None,
|
| 217 | git_shim_measured=subject.git_shim,
|
| 218 | git_shim_resolved=subject.requested_bin is not None and not subject.git_shim,
|
| 219 | git_config_profile=git_config_profile(subject.git_mode))
|
| 220 | details[subject.name] = item
|
| 221 | return details
|
| 222 |
|
| 223 |
|
| 224 | def git_config_profile(mode: str | None) -> str:
|
| 225 | """Row label that separates stock Git from tuned configurations."""
|
| 226 | return "stock" if not mode else ("scalar_recommended" if mode == "scalar" else f"tuned_{mode}")
|
| 227 |
|
| 228 |
|
| 229 | def subject_versions(subjects: list[Subject]) -> dict[str, str]:
|
| 230 | return {subject.name: subject_version(subject) for subject in subjects}
|
| 231 |
|
| 232 |
|
| 233 | def command_text(command: list[str], cwd: Path | None = None) -> str | None:
|
| 234 | try:
|
| 235 | return subprocess.check_output(
|
| 236 | command, cwd=cwd, text=True, stderr=subprocess.STDOUT, timeout=10
|
| 237 | ).strip()
|
| 238 | except Exception:
|
| 239 | return None
|
| 240 |
|
| 241 |
|
| 242 | def source_metadata(oak_repo: Path) -> dict[str, Any]:
|
| 243 | """Oak-source provenance for tying rows back to a changeset, without using Git."""
|
| 244 | oak_bin = shutil.which("oak")
|
| 245 | if not oak_bin or not oak_repo.exists():
|
| 246 | return {"oak_repo": str(oak_repo), "oak_hash": None, "oak_status": None}
|
| 247 | return {
|
| 248 | "oak_repo": str(oak_repo),
|
| 249 | "oak_hash": command_text([oak_bin, "hash"], cwd=oak_repo),
|
| 250 | "oak_status": command_text([oak_bin, "status"], cwd=oak_repo),
|
| 251 | }
|