88e86b3faccd
Rebuild benchmarks as a clean single-root reposi
2 months ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Generate the branch-triage-shape local fixture with BRANCH_MANIFEST.json.
|
| 3 |
|
| 4 | Creates a single git repository under ``<out>/repo`` with ``main`` plus N
|
| 5 | labeled branches whose expected triage labels are recorded in
|
| 6 | ``<out>/BRANCH_MANIFEST.json``. Shapes cycle through empty, superseded_exact,
|
| 7 | clean_contributes, conflict_additive, contributes_and_reverts_target, and
|
| 8 | missing_data.
|
| 9 |
|
| 10 | Determinism follows make_conflict_corpus.py: fixed identity, stepped commit
|
| 11 | timestamps, seed-derived content, isolated git config. The same seed and
|
| 12 | branch count always yield identical repos and manifests.
|
| 13 |
|
| 14 | Self-test after generation asserts manifest validity, branch presence, and
|
| 15 | deterministic regeneration.
|
| 16 |
|
| 17 | Exit codes: 0 ok, 1 usage/generation/self-test error, 3 git unavailable.
|
| 18 | """
|
| 19 |
|
| 20 | from __future__ import annotations
|
| 21 |
|
| 22 | import argparse
|
| 23 | import hashlib
|
| 24 | import json
|
| 25 | import os
|
| 26 | import shutil
|
| 27 | import subprocess
|
| 28 | import sys
|
| 29 | import tempfile
|
| 30 | from pathlib import Path
|
| 31 |
|
| 32 | ROOT = Path(__file__).resolve().parents[1]
|
| 33 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 34 |
|
| 35 | from oakbench.branch_triage_shape import (
|
| 36 | GENERATOR_VERSION,
|
| 37 | MANIFEST_FILENAME,
|
| 38 | branch_name,
|
| 39 | shape_name,
|
| 40 | validate_manifest,
|
| 41 | write_manifest,
|
| 42 | )
|
| 43 |
|
| 44 | IDENTITY_NAME = "Oak Bench"
|
| 45 | IDENTITY_EMAIL = "[email protected]"
|
| 46 | EPOCH = 1_710_000_000
|
| 47 |
|
| 48 |
|
| 49 | def git_env(tick: int = 0) -> dict[str, str]:
|
| 50 | stamp = f"{EPOCH + 60 * tick} +0000"
|
| 51 | env = dict(os.environ)
|
| 52 | env.update(
|
| 53 | GIT_CONFIG_GLOBAL=os.devnull,
|
| 54 | GIT_CONFIG_NOSYSTEM="1",
|
| 55 | GIT_AUTHOR_NAME=IDENTITY_NAME,
|
| 56 | GIT_AUTHOR_EMAIL=IDENTITY_EMAIL,
|
| 57 | GIT_COMMITTER_NAME=IDENTITY_NAME,
|
| 58 | GIT_COMMITTER_EMAIL=IDENTITY_EMAIL,
|
| 59 | GIT_AUTHOR_DATE=stamp,
|
| 60 | GIT_COMMITTER_DATE=stamp,
|
| 61 | )
|
| 62 | return env
|
| 63 |
|
| 64 |
|
| 65 | def run_git(args: list[str], cwd: Path, tick: int = 0, check: bool = True) -> subprocess.CompletedProcess:
|
| 66 | return subprocess.run(
|
| 67 | ["git", *args],
|
| 68 | cwd=cwd,
|
| 69 | env=git_env(tick),
|
| 70 | check=check,
|
| 71 | capture_output=True,
|
| 72 | text=True,
|
| 73 | )
|
| 74 |
|
| 75 |
|
| 76 | class Repo:
|
| 77 | def __init__(self, path: Path) -> None:
|
| 78 | self.path = path
|
| 79 | self.tick = 0
|
| 80 | path.mkdir(parents=True, exist_ok=True)
|
| 81 | run_git(["init", "-q"], path)
|
| 82 | run_git(["symbolic-ref", "HEAD", "refs/heads/main"], path)
|
| 83 |
|
| 84 | def write(self, rel: str, data: str | bytes) -> None:
|
| 85 | target = self.path / rel
|
| 86 | target.parent.mkdir(parents=True, exist_ok=True)
|
| 87 | if isinstance(data, str):
|
| 88 | target.write_text(data)
|
| 89 | else:
|
| 90 | target.write_bytes(data)
|
| 91 |
|
| 92 | def commit(self, message: str) -> str:
|
| 93 | run_git(["add", "-A"], self.path, self.tick)
|
| 94 | run_git(["commit", "-q", "-m", message], self.path, self.tick)
|
| 95 | self.tick += 1
|
| 96 | return self.rev_parse()
|
| 97 |
|
| 98 | def rev_parse(self, rev: str = "HEAD") -> str:
|
| 99 | return run_git(["rev-parse", rev], self.path, self.tick).stdout.strip()
|
| 100 |
|
| 101 | def branch(self, name: str, start: str = "HEAD") -> None:
|
| 102 | run_git(["branch", name, start], self.path, self.tick)
|
| 103 |
|
| 104 | def checkout(self, branch: str, create: bool = False, start: str | None = None) -> None:
|
| 105 | args = ["checkout", "-q"]
|
| 106 | if create:
|
| 107 | args.extend(["-b", branch])
|
| 108 | if start is not None:
|
| 109 | args.append(start)
|
| 110 | else:
|
| 111 | args.append(branch)
|
| 112 | run_git(args, self.path, self.tick)
|
| 113 |
|
| 114 | def cherry_pick(self, commit: str) -> None:
|
| 115 | run_git(["cherry-pick", commit], self.path, self.tick)
|
| 116 | self.tick += 1
|
| 117 |
|
| 118 |
|
| 119 | def superseded_path(index: int) -> str:
|
| 120 | return f"states/superseded-{index}.txt"
|
| 121 |
|
| 122 |
|
| 123 | def conflict_path(index: int) -> str:
|
| 124 | return f"conflict/shared-{index}.txt"
|
| 125 |
|
| 126 |
|
| 127 | def target_path(index: int) -> str:
|
| 128 | return f"target/risk-{index}.txt"
|
| 129 |
|
| 130 |
|
| 131 | def superseded_placeholder(seed: str, index: int) -> str:
|
| 132 | return f"superseded placeholder {seed} {index}\n"
|
| 133 |
|
| 134 |
|
| 135 | def superseded_final(seed: str, index: int) -> str:
|
| 136 | return f"superseded exact final {seed} {index}\n"
|
| 137 |
|
| 138 |
|
| 139 | def conflict_base(seed: str, index: int) -> str:
|
| 140 | return f"shared base {seed} {index}\n"
|
| 141 |
|
| 142 |
|
| 143 | def conflict_main(seed: str, index: int) -> str:
|
| 144 | return f"main additive {seed} {index}\n"
|
| 145 |
|
| 146 |
|
| 147 | def conflict_branch(seed: str, index: int) -> str:
|
| 148 | return f"branch additive {seed} {index}\n"
|
| 149 |
|
| 150 |
|
| 151 | def target_before_fix(seed: str, index: int) -> str:
|
| 152 | return f"target before fix {seed} {index}\n"
|
| 153 |
|
| 154 |
|
| 155 | def target_after_fix(seed: str, index: int) -> str:
|
| 156 | return f"target fixed {seed} {index}\n"
|
| 157 |
|
| 158 |
|
| 159 | def missing_payload(seed: str, index: int) -> bytes:
|
| 160 | return hashlib.sha256(f"{seed}:missing:{index}".encode()).digest()
|
| 161 |
|
| 162 |
|
| 163 | def support_key(shape: str, index: int) -> str:
|
| 164 | return f"{shape}:{index}"
|
| 165 |
|
| 166 |
|
| 167 | def seed_main(repo: Repo, seed: str, branch_count: int) -> tuple[str, list[tuple[str, str]]]:
|
| 168 | repo.write("README.md", f"# branch triage fixture\n\nseed={seed}\n")
|
| 169 | repo.write(".gitignore", ".fixture-ready\n")
|
| 170 | for index in range(branch_count):
|
| 171 | shape = shape_name(index)
|
| 172 | if shape == "superseded_exact":
|
| 173 | repo.write(superseded_path(index), superseded_placeholder(seed, index))
|
| 174 | elif shape == "conflict_additive":
|
| 175 | repo.write(conflict_path(index), conflict_base(seed, index))
|
| 176 | elif shape == "contributes_and_reverts_target":
|
| 177 | repo.write(target_path(index), target_before_fix(seed, index))
|
| 178 | base_head = repo.commit("init main")
|
| 179 |
|
| 180 | support_commits: list[tuple[str, str]] = []
|
| 181 | for index in range(branch_count):
|
| 182 | shape = shape_name(index)
|
| 183 | key = support_key(shape, index)
|
| 184 | if shape == "superseded_exact":
|
| 185 | repo.write(superseded_path(index), superseded_final(seed, index))
|
| 186 | support_commits.append((key, repo.commit(f"main superseded content {index}")))
|
| 187 | elif shape == "conflict_additive":
|
| 188 | repo.write(conflict_path(index), conflict_main(seed, index))
|
| 189 | support_commits.append((key, repo.commit(f"main conflict side {index}")))
|
| 190 | elif shape == "contributes_and_reverts_target":
|
| 191 | repo.write(target_path(index), target_after_fix(seed, index))
|
| 192 | support_commits.append((key, repo.commit(f"main target fix {index}")))
|
| 193 | return base_head, support_commits
|
| 194 |
|
| 195 |
|
| 196 | def cherry_pick_support(repo: Repo, *, own_key: str, support_commits: list[tuple[str, str]]) -> None:
|
| 197 | for key, commit in support_commits:
|
| 198 | if key == own_key:
|
| 199 | continue
|
| 200 | repo.cherry_pick(commit)
|
| 201 |
|
| 202 |
|
| 203 | def build_empty(repo: Repo, index: int) -> None:
|
| 204 | repo.branch(branch_name(index), "main")
|
| 205 |
|
| 206 |
|
| 207 | def build_superseded_exact(
|
| 208 | repo: Repo,
|
| 209 | seed: str,
|
| 210 | index: int,
|
| 211 | *,
|
| 212 | base_head: str,
|
| 213 | support_commits: list[tuple[str, str]],
|
| 214 | ) -> None:
|
| 215 | repo.checkout(branch_name(index), create=True, start=base_head)
|
| 216 | repo.write(superseded_path(index), superseded_final(seed, index))
|
| 217 | repo.commit(f"branch superseded content {index}")
|
| 218 | cherry_pick_support(repo, own_key=support_key("superseded_exact", index), support_commits=support_commits)
|
| 219 | repo.checkout("main")
|
| 220 |
|
| 221 |
|
| 222 | def build_clean_contributes(repo: Repo, seed: str, index: int) -> None:
|
| 223 | repo.checkout(branch_name(index), create=True, start="main")
|
| 224 | repo.write(f"contrib/clean-{index}.txt", f"clean contribution {seed} {index}\n")
|
| 225 | repo.commit(f"clean contribution {index}")
|
| 226 | repo.checkout("main")
|
| 227 |
|
| 228 |
|
| 229 | def build_conflict_additive(
|
| 230 | repo: Repo,
|
| 231 | seed: str,
|
| 232 | index: int,
|
| 233 | *,
|
| 234 | base_head: str,
|
| 235 | support_commits: list[tuple[str, str]],
|
| 236 | ) -> None:
|
| 237 | repo.checkout(branch_name(index), create=True, start=base_head)
|
| 238 | repo.write(conflict_path(index), conflict_branch(seed, index))
|
| 239 | repo.commit(f"branch conflict side {index}")
|
| 240 | cherry_pick_support(repo, own_key=support_key("conflict_additive", index), support_commits=support_commits)
|
| 241 | repo.checkout("main")
|
| 242 |
|
| 243 |
|
| 244 | def build_contributes_and_reverts_target(repo: Repo, seed: str, index: int) -> None:
|
| 245 | repo.checkout(branch_name(index), create=True, start="main")
|
| 246 | repo.write(f"contrib/reverts-target-{index}.txt", f"contributes while reverting target {seed} {index}\n")
|
| 247 | repo.write(target_path(index), target_before_fix(seed, index))
|
| 248 | repo.commit(f"regressive contribution {index}")
|
| 249 | repo.checkout("main")
|
| 250 |
|
| 251 |
|
| 252 | def build_missing_data(repo: Repo, seed: str, index: int) -> None:
|
| 253 | repo.checkout(branch_name(index), create=True, start="main")
|
| 254 | repo.write(f"missing/manual-{index}.txt", f"manual review required {seed} {index}\n")
|
| 255 | repo.write(f"missing/payload-{index}.bin", missing_payload(seed, index))
|
| 256 | repo.commit(f"missing data branch {index}")
|
| 257 | repo.checkout("main")
|
| 258 |
|
| 259 |
|
| 260 | def build_shape(
|
| 261 | repo: Repo,
|
| 262 | seed: str,
|
| 263 | index: int,
|
| 264 | *,
|
| 265 | base_head: str,
|
| 266 | support_commits: list[tuple[str, str]],
|
| 267 | ) -> None:
|
| 268 | shape = shape_name(index)
|
| 269 | if shape == "empty":
|
| 270 | build_empty(repo, index)
|
| 271 | elif shape == "superseded_exact":
|
| 272 | build_superseded_exact(repo, seed, index, base_head=base_head, support_commits=support_commits)
|
| 273 | elif shape == "clean_contributes":
|
| 274 | build_clean_contributes(repo, seed, index)
|
| 275 | elif shape == "conflict_additive":
|
| 276 | build_conflict_additive(repo, seed, index, base_head=base_head, support_commits=support_commits)
|
| 277 | elif shape == "contributes_and_reverts_target":
|
| 278 | build_contributes_and_reverts_target(repo, seed, index)
|
| 279 | elif shape == "missing_data":
|
| 280 | build_missing_data(repo, seed, index)
|
| 281 | else:
|
| 282 | raise ValueError(f"unknown shape {shape!r}")
|
| 283 |
|
| 284 |
|
| 285 | def generate_fixture(out: Path, seed: str, branch_count: int) -> dict[str, object]:
|
| 286 | if shutil.which("git") is None:
|
| 287 | raise RuntimeError("git unavailable")
|
| 288 | repo_root = out / "repo"
|
| 289 | if repo_root.exists():
|
| 290 | shutil.rmtree(repo_root)
|
| 291 | repo = Repo(repo_root)
|
| 292 | base_head, support_commits = seed_main(repo, seed, branch_count)
|
| 293 | for index in range(branch_count):
|
| 294 | build_shape(repo, seed, index, base_head=base_head, support_commits=support_commits)
|
| 295 | manifest = write_manifest(out / MANIFEST_FILENAME, seed=seed, branch_count=branch_count)
|
| 296 | errors = validate_manifest(manifest)
|
| 297 | if errors:
|
| 298 | raise RuntimeError("manifest validation failed: " + "; ".join(errors))
|
| 299 | branches = run_git(["branch", "--format=%(refname:short)"], repo_root, repo.tick, check=False)
|
| 300 | present = {line.strip() for line in branches.stdout.splitlines() if line.strip()}
|
| 301 | for entry in manifest["branches"]:
|
| 302 | name = entry["name"]
|
| 303 | if name not in present:
|
| 304 | raise RuntimeError(f"missing generated branch {name!r}")
|
| 305 | return manifest
|
| 306 |
|
| 307 |
|
| 308 | def self_test(out: Path, seed: str, branch_count: int) -> None:
|
| 309 | manifest = json.loads((out / MANIFEST_FILENAME).read_text())
|
| 310 | if validate_manifest(manifest):
|
| 311 | raise RuntimeError("self-test manifest invalid")
|
| 312 | if branch_count > 6:
|
| 313 | return
|
| 314 | regen = Path(tempfile.mkdtemp(prefix="branch-triage-selftest-"))
|
| 315 | try:
|
| 316 | generate_fixture(regen, seed, branch_count)
|
| 317 | first = (out / MANIFEST_FILENAME).read_text()
|
| 318 | second = (regen / MANIFEST_FILENAME).read_text()
|
| 319 | if first != second:
|
| 320 | raise RuntimeError("manifest not deterministic across regeneration")
|
| 321 | finally:
|
| 322 | shutil.rmtree(regen, ignore_errors=True)
|
| 323 |
|
| 324 |
|
| 325 | def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
| 326 | parser = argparse.ArgumentParser(description="Generate branch triage shape fixture")
|
| 327 | parser.add_argument("--out", type=Path, required=True)
|
| 328 | parser.add_argument("--seed", default="oakbench-branch-triage-v2")
|
| 329 | parser.add_argument("--branch-count", type=int, default=6)
|
| 330 | parser.add_argument("--quick", action="store_true", help="Alias for --branch-count 6")
|
| 331 | parser.add_argument("--medium", action="store_true", help="Alias for --branch-count 60")
|
| 332 | return parser.parse_args(argv)
|
| 333 |
|
| 334 |
|
| 335 | def main(argv: list[str] | None = None) -> int:
|
| 336 | args = parse_args(argv)
|
| 337 | if args.quick:
|
| 338 | args.branch_count = 6
|
| 339 | if args.medium:
|
| 340 | args.branch_count = 60
|
| 341 | if args.branch_count < 1:
|
| 342 | print("branch-count must be at least 1", file=sys.stderr)
|
| 343 | return 1
|
| 344 | if shutil.which("git") is None:
|
| 345 | print("git unavailable", file=sys.stderr)
|
| 346 | return 3
|
| 347 | try:
|
| 348 | generate_fixture(args.out, args.seed, args.branch_count)
|
| 349 | self_test(args.out, args.seed, args.branch_count)
|
| 350 | except RuntimeError as exc:
|
| 351 | print(str(exc), file=sys.stderr)
|
| 352 | return 1
|
| 353 | print(json.dumps({"ok": True, "generator": GENERATOR_VERSION, "out": str(args.out)}, sort_keys=True))
|
| 354 | return 0
|
| 355 |
|
| 356 |
|
| 357 | if __name__ == "__main__":
|
| 358 | raise SystemExit(main())
|