Viewing files at commit 19b295c14414Commit detailsView latest

Browse files
scripts/repro_bundle.py 252 lines · 8.8 KB · python
1
#!/usr/bin/env python3
2
"""Build a repro bundle for a published drop.
3
4
A repro bundle pins everything a third party needs to reproduce a run:
5
suite revision, scenario/config hashes, fixture seeds + generator versions,
6
subject binary SHA256s, runner profile, noise floor, and raw-JSONL hashes
7
(plus optional R2 storage keys). The bundle carries hashes of the raw rows,
8
not the rows themselves.
9
10
Usage:
11
12
  python3 scripts/repro_bundle.py results/rows.jsonl ... \
13
      --out bundle.json --suite-revision <oak-hash> \
14
      [--noise-floor-pct 4.2] [--r2-key raw/rows.jsonl ...]
15
16
Null means unmeasured (ADR-0002): an absent noise floor is recorded as null,
17
never 0.
18
"""
19
20
from __future__ import annotations
21
22
import argparse
23
import json
24
import sys
25
from datetime import datetime, timezone
26
from pathlib import Path
27
from typing import Any
28
29
ROOT = Path(__file__).resolve().parents[1]
30
sys.path.insert(0, str(ROOT / "scripts"))
31
32
from oakbench.baseline_book import sha256_file  # noqa: E402
33
from oakbench.fixture_registry import load_fixture_registry  # noqa: E402
34
from oakbench.runner import machine_profile, runner_class, runner_id  # noqa: E402
35
36
SCHEMA_VERSION = 1
37
CONTENT_INTEGRITY_SOURCE_STRENGTH = {
38
    "git_head_payload_sha256": 3,
39
    "oak_independent_reader_payload_sha256": 3,
40
    "git_lfs_worktree_payload_sha256": 1,
41
    "worktree_payload_sha256": 1,
42
    "mixed_payload_sources": 1,
43
}
44
45
46
def hash_directory(directory: Path, pattern: str, root: Path) -> dict[str, str]:
47
    """SHA256 every file matching ``pattern`` under ``directory``."""
48
    hashes: dict[str, str] = {}
49
    if directory.is_dir():
50
        for path in sorted(directory.glob(pattern)):
51
            if path.is_file():
52
                hashes[path.relative_to(root).as_posix()] = sha256_file(path)
53
    return hashes
54
55
56
def count_jsonl_rows(path: Path) -> int:
57
    count = 0
58
    with path.open() as fh:
59
        for line in fh:
60
            if line.strip():
61
                count += 1
62
    return count
63
64
65
def iter_jsonl_rows(paths: list[Path]) -> list[dict[str, Any]]:
66
    rows: list[dict[str, Any]] = []
67
    for path in paths:
68
        with path.open() as fh:
69
            for raw_line in fh:
70
                line = raw_line.strip()
71
                if line:
72
                    rows.append(json.loads(line))
73
    return rows
74
75
76
def subject_binaries_from_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
77
    """Map subject -> binary SHA256 collected from rows' binary_sha256 fields.
78
79
    A subject that recorded more than one distinct binary hash keeps the full
80
    sorted list — silently picking one would hide a mixed-binary run.
81
    """
82
    found: dict[str, set[str]] = {}
83
    for row in rows:
84
        subject = row.get("subject")
85
        sha = row.get("binary_sha256")
86
        if isinstance(subject, str) and subject and isinstance(sha, str) and sha:
87
            found.setdefault(subject, set()).add(sha)
88
    result: dict[str, Any] = {}
89
    for subject in sorted(found):
90
        ordered = sorted(found[subject])
91
        result[subject] = ordered[0] if len(ordered) == 1 else ordered
92
    return result
93
94
95
def content_integrity_summary_from_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
96
    """Summarize content-attestation strength by comparable row group.
97
98
    The boolean content_integrity_check_passed says only whether the row's
99
    chosen oracle passed. Publication/replay decisions also need the source
100
    strength: HEAD-payload attestation is stronger than worktree/LFS-worktree
101
    attestation, and mixed groups must be treated as their weakest source.
102
    """
103
    grouped: dict[str, dict[str, Any]] = {}
104
    for row in rows:
105
        source = row.get("content_integrity_source")
106
        sources = row.get("content_integrity_sources")
107
        if not isinstance(source, str) and not isinstance(sources, list):
108
            continue
109
        profile = row.get("profile")
110
        key = "/".join(
111
            [
112
                "-" if profile is None else str(profile),
113
                str(row.get("scenario") or ""),
114
                str(row.get("operation") or ""),
115
                str(row.get("subject") or ""),
116
            ]
117
        )
118
        entry = grouped.setdefault(
119
            key,
120
            {
121
                "sources": set(),
122
                "rows": 0,
123
                "passed": 0,
124
                "failed": 0,
125
                "weakest_source_strength": None,
126
            },
127
        )
128
        row_sources = [value for value in ([source] + sources if isinstance(sources, list) else [source]) if isinstance(value, str)]
129
        for value in row_sources:
130
            entry["sources"].add(value)
131
            strength = CONTENT_INTEGRITY_SOURCE_STRENGTH.get(value, 0)
132
            current = entry["weakest_source_strength"]
133
            entry["weakest_source_strength"] = strength if current is None else min(current, strength)
134
        entry["rows"] += 1
135
        if row.get("content_integrity_check_passed") is True:
136
            entry["passed"] += 1
137
        elif row.get("content_integrity_check_passed") is False:
138
            entry["failed"] += 1
139
140
    return {
141
        key: {
142
            "sources": sorted(value["sources"]),
143
            "rows": value["rows"],
144
            "passed": value["passed"],
145
            "failed": value["failed"],
146
            "weakest_source_strength": value["weakest_source_strength"],
147
        }
148
        for key, value in sorted(grouped.items())
149
    }
150
151
152
def build_bundle(
153
    row_paths: list[Path],
154
    *,
155
    suite_revision: str,
156
    noise_floor_pct: float | None = None,
157
    r2_keys: list[str] | None = None,
158
    root: Path = ROOT,
159
) -> dict[str, Any]:
160
    profile = machine_profile()
161
    registry = load_fixture_registry(root / "config" / "fixtures.json")
162
    rows = iter_jsonl_rows(row_paths)
163
    return {
164
        "schema_version": SCHEMA_VERSION,
165
        "created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
166
        "suite_revision": suite_revision,
167
        "runner_profile": profile,
168
        "runner_id": runner_id(profile),
169
        "runner_class": runner_class(profile),
170
        "noise_floor_pct": noise_floor_pct,
171
        "config_hashes": {
172
            **hash_directory(root / "config", "**/*.json", root),
173
            **hash_directory(root / "config", "**/*.toml", root),
174
        },
175
        "scenario_hashes": hash_directory(root / "scenarios", "*.yaml", root),
176
        "fixtures": [
177
            {
178
                "id": spec.fixture_id,
179
                "seed": spec.seed,
180
                "generator": spec.generator,
181
                "version": spec.version,
182
            }
183
            for _, spec in sorted(registry.items())
184
        ],
185
        "subject_binaries": subject_binaries_from_rows(rows),
186
        "content_integrity": {
187
            "source_strength": dict(sorted(CONTENT_INTEGRITY_SOURCE_STRENGTH.items())),
188
            "groups": content_integrity_summary_from_rows(rows),
189
        },
190
        "raw_inputs": [
191
            {
192
                "path": str(path),
193
                "sha256": sha256_file(path),
194
                "bytes": path.stat().st_size,
195
                "rows": count_jsonl_rows(path),
196
            }
197
            for path in row_paths
198
        ],
199
        "r2_keys": list(r2_keys or []),
200
    }
201
202
203
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
204
    parser = argparse.ArgumentParser(description="Build a repro bundle for a published drop.")
205
    parser.add_argument("rows", nargs="+", type=Path, help="raw row JSONL paths covered by the drop")
206
    parser.add_argument("--out", type=Path, required=True, help="output bundle JSON path")
207
    parser.add_argument(
208
        "--suite-revision",
209
        required=True,
210
        help="caller-supplied suite revision (e.g. the oak hash of the harness checkout)",
211
    )
212
    parser.add_argument(
213
        "--noise-floor-pct",
214
        type=float,
215
        default=None,
216
        help="measured same-binary noise floor percentage; omitted means unmeasured (null)",
217
    )
218
    parser.add_argument(
219
        "--r2-key",
220
        action="append",
221
        default=None,
222
        dest="r2_keys",
223
        help="R2 storage key for an uploaded raw artifact (repeatable)",
224
    )
225
    return parser.parse_args(argv)
226
227
228
def main(argv: list[str] | None = None) -> int:
229
    args = parse_args(argv)
230
    missing = [str(path) for path in args.rows if not path.is_file()]
231
    if missing:
232
        print(f"error: row file(s) not found: {', '.join(missing)}", file=sys.stderr)
233
        return 2
234
    bundle = build_bundle(
235
        args.rows,
236
        suite_revision=args.suite_revision,
237
        noise_floor_pct=args.noise_floor_pct,
238
        r2_keys=args.r2_keys,
239
    )
240
    args.out.parent.mkdir(parents=True, exist_ok=True)
241
    args.out.write_text(json.dumps(bundle, indent=2, sort_keys=True) + "\n")
242
    print(
243
        f"wrote {args.out}: {len(bundle['raw_inputs'])} raw input(s), "
244
        f"{len(bundle['config_hashes'])} config hash(es), "
245
        f"{len(bundle['scenario_hashes'])} scenario hash(es), "
246
        f"{len(bundle['fixtures'])} fixture(s)"
247
    )
248
    return 0
249
250
251
if __name__ == "__main__":
252
    sys.exit(main())