Viewing files at commit f86d1109f5dfCommit detailsView latest

scripts/oakbench/subjects.py 149 lines · 4.7 KB · python
1
"""Subject identity: which VCS binaries are under measurement.
2
3
Subjects have two identity layers (CONTEXT.md): the logical name used for
4
trend grouping, and binary+version+source-hash provenance recorded per run.
5
"""
6
7
from __future__ import annotations
8
9
import argparse
10
import hashlib
11
import os
12
import shutil
13
import subprocess
14
from dataclasses import dataclass
15
from pathlib import Path
16
from typing import Any
17
18
from oakbench.configio import load_subject_config
19
20
ROOT = Path(__file__).resolve().parents[2]
21
DEFAULT_OAK_REPO = Path(os.environ.get("OAK_REPO", ROOT.parent / "oak"))
22
23
24
@dataclass(frozen=True)
25
class Subject:
26
    name: str
27
    kind: str
28
    label: str
29
    bin: Path
30
    git_mode: str | None = None
31
32
33
def resolve_subject_bin(raw: str | Path, override: Path | None = None) -> Path:
34
    if override is not None:
35
        path = override.expanduser()
36
        return path if path.is_absolute() else (ROOT / path).resolve()
37
38
    raw_text = str(raw)
39
    path = Path(raw_text).expanduser()
40
    if path.is_absolute():
41
        return path
42
43
    if "/" in raw_text or "\\" in raw_text:
44
        return (ROOT / path).resolve()
45
46
    found = shutil.which(raw_text)
47
    return Path(found) if found else path
48
49
50
def load_subjects(args: argparse.Namespace) -> list[Subject]:
51
    """Load subjects from config/subjects.toml with the standard CLI overrides.
52
53
    Expects args to carry: config, subjects, git_bin, oak_installed_bin,
54
    oak_local_bin — the shared flag set every lane defines.
55
    """
56
    data = load_subject_config(args.config)
57
    subjects: list[Subject] = []
58
    wanted = {item.strip() for item in args.subjects.split(",") if item.strip()} if args.subjects else None
59
    for name, raw in data["subjects"].items():
60
        if not raw.get("enabled", False) and (wanted is None or name not in wanted):
61
            continue
62
        override = None
63
        if name == "git":
64
            override = args.git_bin
65
        elif name == "oak_installed":
66
            override = args.oak_installed_bin
67
        elif name == "oak_local":
68
            override = args.oak_local_bin
69
        path = resolve_subject_bin(raw["bin"], override)
70
        subjects.append(Subject(name=name, kind=raw["kind"], label=raw.get("label", name), bin=path))
71
72
    if wanted is not None:
73
        subjects = [subject for subject in subjects if subject.name in wanted]
74
75
    missing = [str(subject.bin) for subject in subjects if not subject.bin.exists()]
76
    if missing:
77
        raise SystemExit("Missing benchmark binaries:\n  " + "\n  ".join(missing))
78
    if not subjects:
79
        raise SystemExit("No enabled benchmark subjects")
80
    return subjects
81
82
83
def subject_version(subject: Subject) -> str:
84
    try:
85
        return subprocess.check_output(
86
            [str(subject.bin), "--version"], text=True, stderr=subprocess.STDOUT, timeout=5
87
        ).strip()
88
    except Exception as exc:  # noqa: BLE001 - metadata only
89
        return f"unknown ({exc})"
90
91
92
def subject_role(subject: Subject) -> str:
93
    if subject.name == "oak_installed":
94
        return "oak_release_latest_installed"
95
    if subject.name == "oak_local":
96
        return "oak_current_changeset"
97
    if subject.name == "oak_main":
98
        return "oak_main_clean_build"
99
    return subject.kind
100
101
102
def binary_sha256(path: Path) -> str | None:
103
    try:
104
        digest = hashlib.sha256()
105
        with path.open("rb") as fh:
106
            for chunk in iter(lambda: fh.read(1024 * 1024), b""):
107
                digest.update(chunk)
108
        return digest.hexdigest()
109
    except OSError:
110
        return None
111
112
113
def subject_details(subjects: list[Subject]) -> dict[str, dict[str, Any]]:
114
    return {
115
        subject.name: {
116
            "kind": subject.kind,
117
            "label": subject.label,
118
            "bin": str(subject.bin),
119
            "binary_sha256": binary_sha256(subject.bin),
120
            "version": subject_version(subject),
121
            "role": subject_role(subject),
122
        }
123
        for subject in subjects
124
    }
125
126
127
def subject_versions(subjects: list[Subject]) -> dict[str, str]:
128
    return {subject.name: subject_version(subject) for subject in subjects}
129
130
131
def command_text(command: list[str], cwd: Path | None = None) -> str | None:
132
    try:
133
        return subprocess.check_output(
134
            command, cwd=cwd, text=True, stderr=subprocess.STDOUT, timeout=10
135
        ).strip()
136
    except Exception:
137
        return None
138
139
140
def source_metadata(oak_repo: Path) -> dict[str, Any]:
141
    """Oak-source provenance for tying rows back to a changeset, without using Git."""
142
    oak_bin = shutil.which("oak")
143
    if not oak_bin or not oak_repo.exists():
144
        return {"oak_repo": str(oak_repo), "oak_hash": None, "oak_status": None}
145
    return {
146
        "oak_repo": str(oak_repo),
147
        "oak_hash": command_text([oak_bin, "hash"], cwd=oak_repo),
148
        "oak_status": command_text([oak_bin, "status"], cwd=oak_repo),
149
    }