/oak
Log in
benchmarks
scripts/fixture_ensure.py 89 lines · 3.0 KB · blame Source
88e86b3faccd Rebuild benchmarks as a clean single-root reposi 2 months ago
1
#!/usr/bin/env python3
2
"""Inspect and verify registered benchmark fixtures.
3
4
Generation is intentionally not implemented for planned fixtures yet; this
5
script establishes the registry contract and verification path without making
6
large unpublished fixture bytes part of this changeset.
7
"""
8
9
from __future__ import annotations
10
11
import argparse
12
import json
13
import sys
14
from pathlib import Path
15
16
ROOT = Path(__file__).resolve().parents[1]
17
sys.path.insert(0, str(ROOT / "scripts"))
18
19
from oakbench.fixture_registry import load_fixture_registry, verify_fixture  # noqa: E402
20
21
22
def parse_args() -> argparse.Namespace:
23
    parser = argparse.ArgumentParser(description=__doc__)
24
    parser.add_argument("--registry", type=Path, default=ROOT / "config" / "fixtures.json")
25
    parser.add_argument("--fixture-id")
26
    parser.add_argument("--path", type=Path, help="Existing fixture directory to verify")
27
    parser.add_argument("--json", action="store_true")
28
    return parser.parse_args()
29
30
31
def main() -> int:
32
    args = parse_args()
33
    registry = load_fixture_registry(args.registry)
34
    if args.fixture_id:
35
        if args.fixture_id not in registry:
36
            raise SystemExit(f"unknown fixture id {args.fixture_id!r}")
37
        specs = [registry[args.fixture_id]]
38
    else:
39
        specs = list(registry.values())
40
41
    rows = []
42
    for spec in specs:
43
        row = {
44
            "fixture_id": spec.fixture_id,
45
            "status": spec.status,
46
            "generator": spec.generator,
47
            "version": spec.version,
48
            "seed": spec.seed,
49
            "manifest": spec.manifest,
50
            "hosting": list(spec.hosting),
51
        }
52
        if args.path and len(specs) == 1:
53
            status, actual = verify_fixture(spec, args.path)
54
            if status == "verified":
55
                row.update({"verified": True, "actual_manifest": actual})
56
            elif status.startswith("unverifiable:"):
57
                # Null means unmeasured: no pinned manifest, nothing verified.
58
                row.update(
59
                    {
60
                        "verified": None,
61
                        "reason": status.split(":", 1)[1],
62
                        "actual_manifest": actual,
63
                    }
64
                )
65
            else:
66
                row.update(
67
                    {
68
                        "verified": False,
69
                        "reason": status.split(":", 1)[1],
70
                        "actual_manifest": actual,
71
                    }
72
                )
73
                rows.append(row)
74
                if args.json:
75
                    print(json.dumps(rows, sort_keys=True))
76
                raise SystemExit(1)
77
        rows.append(row)
78
79
    if args.json:
80
        print(json.dumps(rows, sort_keys=True))
81
    else:
82
        for row in rows:
83
            suffix = f" manifest={row['manifest']}" if row["manifest"] else " manifest=unrecorded"
84
            print(f"{row['fixture_id']} {row['status']} {row['version']} seed={row['seed']}{suffix}")
85
    return 0
86
87
88
if __name__ == "__main__":
89
    raise SystemExit(main())