| 1 | """Subject identity: which VCS binaries are under measurement.
|
| 2 |
|
| 3 | Subjects have two identity layers (CONTEXT.md): the logical name used for
|
| 4 | trend grouping, and binary+version+source-hash provenance recorded per run.
|
| 5 | """
|
| 6 |
|
| 7 | from __future__ import annotations
|
| 8 |
|
| 9 | import argparse
|
| 10 | import hashlib
|
| 11 | import os
|
| 12 | import shutil
|
| 13 | import subprocess
|
| 14 | from dataclasses import dataclass
|
| 15 | from pathlib import Path
|
| 16 | from typing import Any
|
| 17 |
|
| 18 | from oakbench.configio import load_subject_config
|
| 19 |
|
| 20 | ROOT = Path(__file__).resolve().parents[2]
|
| 21 | DEFAULT_OAK_REPO = Path(os.environ.get("OAK_REPO", ROOT.parent / "oak"))
|
| 22 |
|
| 23 |
|
| 24 | @dataclass(frozen=True)
|
| 25 | class Subject:
|
| 26 | name: str
|
| 27 | kind: str
|
| 28 | label: str
|
| 29 | bin: Path
|
| 30 | git_mode: str | None = None
|
| 31 |
|
| 32 |
|
| 33 | def resolve_subject_bin(raw: str | Path, override: Path | None = None) -> Path:
|
| 34 | if override is not None:
|
| 35 | path = override.expanduser()
|
| 36 | return path if path.is_absolute() else (ROOT / path).resolve()
|
| 37 |
|
| 38 | raw_text = str(raw)
|
| 39 | path = Path(raw_text).expanduser()
|
| 40 | if path.is_absolute():
|
| 41 | return path
|
| 42 |
|
| 43 | if "/" in raw_text or "\\" in raw_text:
|
| 44 | return (ROOT / path).resolve()
|
| 45 |
|
| 46 | found = shutil.which(raw_text)
|
| 47 | return Path(found) if found else path
|
| 48 |
|
| 49 |
|
| 50 | def load_subjects(args: argparse.Namespace) -> list[Subject]:
|
| 51 | """Load subjects from config/subjects.toml with the standard CLI overrides.
|
| 52 |
|
| 53 | Expects args to carry: config, subjects, git_bin, oak_installed_bin,
|
| 54 | oak_local_bin — the shared flag set every lane defines.
|
| 55 | """
|
| 56 | data = load_subject_config(args.config)
|
| 57 | subjects: list[Subject] = []
|
| 58 | wanted = {item.strip() for item in args.subjects.split(",") if item.strip()} if args.subjects else None
|
| 59 | for name, raw in data["subjects"].items():
|
| 60 | if not raw.get("enabled", False) and (wanted is None or name not in wanted):
|
| 61 | continue
|
| 62 | override = None
|
| 63 | if name == "git":
|
| 64 | override = args.git_bin
|
| 65 | elif name == "oak_installed":
|
| 66 | override = args.oak_installed_bin
|
| 67 | elif name == "oak_local":
|
| 68 | override = args.oak_local_bin
|
| 69 | path = resolve_subject_bin(raw["bin"], override)
|
| 70 | subjects.append(Subject(name=name, kind=raw["kind"], label=raw.get("label", name), bin=path))
|
| 71 |
|
| 72 | if wanted is not None:
|
| 73 | subjects = [subject for subject in subjects if subject.name in wanted]
|
| 74 |
|
| 75 | missing = [str(subject.bin) for subject in subjects if not subject.bin.exists()]
|
| 76 | if missing:
|
| 77 | raise SystemExit("Missing benchmark binaries:\n " + "\n ".join(missing))
|
| 78 | if not subjects:
|
| 79 | raise SystemExit("No enabled benchmark subjects")
|
| 80 | return subjects
|
| 81 |
|
| 82 |
|
| 83 | def subject_version(subject: Subject) -> str:
|
| 84 | try:
|
| 85 | return subprocess.check_output(
|
| 86 | [str(subject.bin), "--version"], text=True, stderr=subprocess.STDOUT, timeout=5
|
| 87 | ).strip()
|
| 88 | except Exception as exc: # noqa: BLE001 - metadata only
|
| 89 | return f"unknown ({exc})"
|
| 90 |
|
| 91 |
|
| 92 | def subject_role(subject: Subject) -> str:
|
| 93 | if subject.name == "oak_installed":
|
| 94 | return "oak_release_latest_installed"
|
| 95 | if subject.name == "oak_local":
|
| 96 | return "oak_current_changeset"
|
| 97 | if subject.name == "oak_main":
|
| 98 | return "oak_main_clean_build"
|
| 99 | return subject.kind
|
| 100 |
|
| 101 |
|
| 102 | def binary_sha256(path: Path) -> str | None:
|
| 103 | try:
|
| 104 | digest = hashlib.sha256()
|
| 105 | with path.open("rb") as fh:
|
| 106 | for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
| 107 | digest.update(chunk)
|
| 108 | return digest.hexdigest()
|
| 109 | except OSError:
|
| 110 | return None
|
| 111 |
|
| 112 |
|
| 113 | def subject_details(subjects: list[Subject]) -> dict[str, dict[str, Any]]:
|
| 114 | return {
|
| 115 | subject.name: {
|
| 116 | "kind": subject.kind,
|
| 117 | "label": subject.label,
|
| 118 | "bin": str(subject.bin),
|
| 119 | "binary_sha256": binary_sha256(subject.bin),
|
| 120 | "version": subject_version(subject),
|
| 121 | "role": subject_role(subject),
|
| 122 | }
|
| 123 | for subject in subjects
|
| 124 | }
|
| 125 |
|
| 126 |
|
| 127 | def subject_versions(subjects: list[Subject]) -> dict[str, str]:
|
| 128 | return {subject.name: subject_version(subject) for subject in subjects}
|
| 129 |
|
| 130 |
|
| 131 | def command_text(command: list[str], cwd: Path | None = None) -> str | None:
|
| 132 | try:
|
| 133 | return subprocess.check_output(
|
| 134 | command, cwd=cwd, text=True, stderr=subprocess.STDOUT, timeout=10
|
| 135 | ).strip()
|
| 136 | except Exception:
|
| 137 | return None
|
| 138 |
|
| 139 |
|
| 140 | def source_metadata(oak_repo: Path) -> dict[str, Any]:
|
| 141 | """Oak-source provenance for tying rows back to a changeset, without using Git."""
|
| 142 | oak_bin = shutil.which("oak")
|
| 143 | if not oak_bin or not oak_repo.exists():
|
| 144 | return {"oak_repo": str(oak_repo), "oak_hash": None, "oak_status": None}
|
| 145 | return {
|
| 146 | "oak_repo": str(oak_repo),
|
| 147 | "oak_hash": command_text([oak_bin, "hash"], cwd=oak_repo),
|
| 148 | "oak_status": command_text([oak_bin, "status"], cwd=oak_repo),
|
| 149 | }
|