Log in
scripts/repro_bundle.py 251 lines · 8.7 KB · python Blame
1
#!/usr/bin/env python3
2
"""Build a repro bundle for a published drop.
3
4
A repro bundle pins everything a third party needs to reproduce a run:
5
suite revision, scenario/config hashes, fixture seeds + generator versions,
6
subject binary SHA256s, runner profile, noise floor, and raw-JSONL hashes
7
(plus optional R2 storage keys). The bundle carries hashes of the raw rows,
8
not the rows themselves.
9
10
Usage:
11
12
  python3 scripts/repro_bundle.py results/rows.jsonl ... \
13
      --out bundle.json --suite-revision <oak-hash> \
14
      [--noise-floor-pct 4.2] [--r2-key raw/rows.jsonl ...]
15
16
Null means unmeasured (ADR-0002): an absent noise floor is recorded as null,
17
never 0.
18
"""
19
20
from __future__ import annotations
21
22
import argparse
23
import json
24
import sys
25
from datetime import datetime, timezone
26
from pathlib import Path
27
from typing import Any
28
29
ROOT = Path(__file__).resolve().parents[1]
30
sys.path.insert(0, str(ROOT / "scripts"))
31
32
from oakbench.baseline_book import sha256_file  # noqa: E402
33
from oakbench.fixture_registry import load_fixture_registry  # noqa: E402
34
from oakbench.runner import machine_profile, runner_class, runner_id  # noqa: E402
35
36
SCHEMA_VERSION = 1
37
CONTENT_INTEGRITY_SOURCE_STRENGTH = {
38
    "git_head_payload_sha256": 3,
39
    "git_lfs_worktree_payload_sha256": 1,
40
    "worktree_payload_sha256": 1,
41
    "mixed_payload_sources": 1,
42
}
43
44
45
def hash_directory(directory: Path, pattern: str, root: Path) -> dict[str, str]:
46
    """SHA256 every file matching ``pattern`` under ``directory``."""
47
    hashes: dict[str, str] = {}
48
    if directory.is_dir():
49
        for path in sorted(directory.glob(pattern)):
50
            if path.is_file():
51
                hashes[path.relative_to(root).as_posix()] = sha256_file(path)
52
    return hashes
53
54
55
def count_jsonl_rows(path: Path) -> int:
56
    count = 0
57
    with path.open() as fh:
58
        for line in fh:
59
            if line.strip():
60
                count += 1
61
    return count
62
63
64
def iter_jsonl_rows(paths: list[Path]) -> list[dict[str, Any]]:
65
    rows: list[dict[str, Any]] = []
66
    for path in paths:
67
        with path.open() as fh:
68
            for raw_line in fh:
69
                line = raw_line.strip()
70
                if line:
71
                    rows.append(json.loads(line))
72
    return rows
73
74
75
def subject_binaries_from_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
76
    """Map subject -> binary SHA256 collected from rows' binary_sha256 fields.
77
78
    A subject that recorded more than one distinct binary hash keeps the full
79
    sorted list β€” silently picking one would hide a mixed-binary run.
80
    """
81
    found: dict[str, set[str]] = {}
82
    for row in rows:
83
        subject = row.get("subject")
84
        sha = row.get("binary_sha256")
85
        if isinstance(subject, str) and subject and isinstance(sha, str) and sha:
86
            found.setdefault(subject, set()).add(sha)
87
    result: dict[str, Any] = {}
88
    for subject in sorted(found):
89
        ordered = sorted(found[subject])
90
        result[subject] = ordered[0] if len(ordered) == 1 else ordered
91
    return result
92
93
94
def content_integrity_summary_from_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
95
    """Summarize content-attestation strength by comparable row group.
96
97
    The boolean content_integrity_check_passed says only whether the row's
98
    chosen oracle passed. Publication/replay decisions also need the source
99
    strength: HEAD-payload attestation is stronger than worktree/LFS-worktree
100
    attestation, and mixed groups must be treated as their weakest source.
101
    """
102
    grouped: dict[str, dict[str, Any]] = {}
103
    for row in rows:
104
        source = row.get("content_integrity_source")
105
        sources = row.get("content_integrity_sources")
106
        if not isinstance(source, str) and not isinstance(sources, list):
107
            continue
108
        profile = row.get("profile")
109
        key = "/".join(
110
            [
111
                "-" if profile is None else str(profile),
112
                str(row.get("scenario") or ""),
113
                str(row.get("operation") or ""),
114
                str(row.get("subject") or ""),
115
            ]
116
        )
117
        entry = grouped.setdefault(
118
            key,
119
            {
120
                "sources": set(),
121
                "rows": 0,
122
                "passed": 0,
123
                "failed": 0,
124
                "weakest_source_strength": None,
125
            },
126
        )
127
        row_sources = [value for value in ([source] + sources if isinstance(sources, list) else [source]) if isinstance(value, str)]
128
        for value in row_sources:
129
            entry["sources"].add(value)
130
            strength = CONTENT_INTEGRITY_SOURCE_STRENGTH.get(value, 0)
131
            current = entry["weakest_source_strength"]
132
            entry["weakest_source_strength"] = strength if current is None else min(current, strength)
133
        entry["rows"] += 1
134
        if row.get("content_integrity_check_passed") is True:
135
            entry["passed"] += 1
136
        elif row.get("content_integrity_check_passed") is False:
137
            entry["failed"] += 1
138
139
    return {
140
        key: {
141
            "sources": sorted(value["sources"]),
142
            "rows": value["rows"],
143
            "passed": value["passed"],
144
            "failed": value["failed"],
145
            "weakest_source_strength": value["weakest_source_strength"],
146
        }
147
        for key, value in sorted(grouped.items())
148
    }
149
150
151
def build_bundle(
152
    row_paths: list[Path],
153
    *,
154
    suite_revision: str,
155
    noise_floor_pct: float | None = None,
156
    r2_keys: list[str] | None = None,
157
    root: Path = ROOT,
158
) -> dict[str, Any]:
159
    profile = machine_profile()
160
    registry = load_fixture_registry(root / "config" / "fixtures.json")
161
    rows = iter_jsonl_rows(row_paths)
162
    return {
163
        "schema_version": SCHEMA_VERSION,
164
        "created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
165
        "suite_revision": suite_revision,
166
        "runner_profile": profile,
167
        "runner_id": runner_id(profile),
168
        "runner_class": runner_class(profile),
169
        "noise_floor_pct": noise_floor_pct,
170
        "config_hashes": {
171
            **hash_directory(root / "config", "**/*.json", root),
172
            **hash_directory(root / "config", "**/*.toml", root),
173
        },
174
        "scenario_hashes": hash_directory(root / "scenarios", "*.yaml", root),
175
        "fixtures": [
176
            {
177
                "id": spec.fixture_id,
178
                "seed": spec.seed,
179
                "generator": spec.generator,
180
                "version": spec.version,
181
            }
182
            for _, spec in sorted(registry.items())
183
        ],
184
        "subject_binaries": subject_binaries_from_rows(rows),
185
        "content_integrity": {
186
            "source_strength": dict(sorted(CONTENT_INTEGRITY_SOURCE_STRENGTH.items())),
187
            "groups": content_integrity_summary_from_rows(rows),
188
        },
189
        "raw_inputs": [
190
            {
191
                "path": str(path),
192
                "sha256": sha256_file(path),
193
                "bytes": path.stat().st_size,
194
                "rows": count_jsonl_rows(path),
195
            }
196
            for path in row_paths
197
        ],
198
        "r2_keys": list(r2_keys or []),
199
    }
200
201
202
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
203
    parser = argparse.ArgumentParser(description="Build a repro bundle for a published drop.")
204
    parser.add_argument("rows", nargs="+", type=Path, help="raw row JSONL paths covered by the drop")
205
    parser.add_argument("--out", type=Path, required=True, help="output bundle JSON path")
206
    parser.add_argument(
207
        "--suite-revision",
208
        required=True,
209
        help="caller-supplied suite revision (e.g. the oak hash of the harness checkout)",
210
    )
211
    parser.add_argument(
212
        "--noise-floor-pct",
213
        type=float,
214
        default=None,
215
        help="measured same-binary noise floor percentage; omitted means unmeasured (null)",
216
    )
217
    parser.add_argument(
218
        "--r2-key",
219
        action="append",
220
        default=None,
221
        dest="r2_keys",
222
        help="R2 storage key for an uploaded raw artifact (repeatable)",
223
    )
224
    return parser.parse_args(argv)
225
226
227
def main(argv: list[str] | None = None) -> int:
228
    args = parse_args(argv)
229
    missing = [str(path) for path in args.rows if not path.is_file()]
230
    if missing:
231
        print(f"error: row file(s) not found: {', '.join(missing)}", file=sys.stderr)
232
        return 2
233
    bundle = build_bundle(
234
        args.rows,
235
        suite_revision=args.suite_revision,
236
        noise_floor_pct=args.noise_floor_pct,
237
        r2_keys=args.r2_keys,
238
    )
239
    args.out.parent.mkdir(parents=True, exist_ok=True)
240
    args.out.write_text(json.dumps(bundle, indent=2, sort_keys=True) + "\n")
241
    print(
242
        f"wrote {args.out}: {len(bundle['raw_inputs'])} raw input(s), "
243
        f"{len(bundle['config_hashes'])} config hash(es), "
244
        f"{len(bundle['scenario_hashes'])} scenario hash(es), "
245
        f"{len(bundle['fixtures'])} fixture(s)"
246
    )
247
    return 0
248
249
250
if __name__ == "__main__":
251
    sys.exit(main())