Viewing files at commit 085949193d78Commit detailsView latest

Browse files
scripts/oakbench/merge_history_fixture.py 227 lines · 9.0 KB · python
1
"""Deterministic Git history with two-parent merges, for ancestry-walk lanes.
2
3
``make_monorepo_fixture.py`` produces linear history. Lanes that measure how a
4
client walks ancestry (e.g. first parent refresh in a shallow worker) need the
5
merge shape real repositories have: a mainline with short side branches
6
merged back with ``--no-ff`` two-parent commits.
7
8
Identity (ADR-0005): the same ``(commits, seed, FIXTURE_SCHEMA)`` always yields
9
byte-identical history — fixed author/committer, timestamps stepping from a
10
fixed epoch, a seeded RNG for topology/content, one ``git fast-import`` stream.
11
Commit SHAs are therefore reproducible across runs and machines, and the
12
resulting head is recorded in the manifest so lanes can check it. Changing
13
the generator's output is a ``FIXTURE_SCHEMA`` bump.
14
15
The model tracks the full file map for every commit, so the expected tree at
16
``main`` (``expected_files``) is known independently of Git and of the VCS
17
under test. ``tree_content_hash`` hashes that map canonically.
18
"""
19
20
from __future__ import annotations
21
22
import hashlib
23
import random
24
import subprocess
25
from dataclasses import dataclass, field
26
from pathlib import Path
27
28
FIXTURE_SCHEMA = "merge-history-v1"
29
EPOCH = 1_767_225_600  # 2026-01-01T00:00:00Z
30
IDENTITY = b"Oak Bench Fixture <[email protected]>"
31
MAINLINE_FILES = 48
32
SIDE_BRANCH_PROBABILITY = 0.035
33
SIDE_BRANCH_MIN_COMMITS = 1
34
SIDE_BRANCH_MAX_COMMITS = 3
35
36
37
@dataclass
38
class MergeHistoryManifest:
39
    schema: str
40
    commits: int
41
    seed: int
42
    merge_commits: int
43
    root_commits: int
44
    expected_files: dict[str, str] = field(repr=False)  # path -> sha256 hex of bytes
45
    head: str | None = None  # git SHA of main, filled after import
46
47
    @property
48
    def fixture_id(self) -> str:
49
        return f"{self.schema}-c{self.commits}-s{self.seed}"
50
51
    @property
52
    def merge_fraction(self) -> float:
53
        return self.merge_commits / self.commits if self.commits else 0.0
54
55
    @property
56
    def tree_content_hash(self) -> str:
57
        return tree_content_hash(self.expected_files)
58
59
    def as_dict(self) -> dict[str, object]:
60
        return {
61
            "fixture_generator": "oakbench.merge_history_fixture",
62
            "fixture_version": self.schema,
63
            "fixture_id": self.fixture_id,
64
            "fixture_commits": self.commits,
65
            "fixture_seed": str(self.seed),
66
            "fixture_merge_commits": self.merge_commits,
67
            "fixture_merge_fraction": round(self.merge_fraction, 4),
68
            "fixture_root_commits": self.root_commits,
69
            "fixture_file_count": len(self.expected_files),
70
            "fixture_tree_content_hash": self.tree_content_hash,
71
            "fixture_git_head": self.head,
72
        }
73
74
75
def tree_content_hash(files: dict[str, str]) -> str:
76
    """Canonical hash of a {posix path: sha256(bytes)} map (paths sorted)."""
77
    digest = hashlib.sha256()
78
    for path in sorted(files):
79
        digest.update(path.encode("utf-8") + b"\0" + files[path].encode("ascii") + b"\n")
80
    return digest.hexdigest()
81
82
83
def worktree_files(root: Path, ignored_top_level: tuple[str, ...] = (".oak", ".git")) -> dict[str, str]:
84
    """Regular files under ``root`` as {posix path: sha256}; symlinks/dirs excluded."""
85
    files: dict[str, str] = {}
86
    for path in sorted(root.rglob("*")):
87
        relative = path.relative_to(root)
88
        if relative.parts and relative.parts[0] in ignored_top_level:
89
            continue
90
        if path.is_file() and not path.is_symlink():
91
            files[relative.as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest()
92
    return files
93
94
95
def _content(path: str, revision: int, rng: random.Random) -> bytes:
96
    token = rng.getrandbits(64)
97
    lines = [f"{path} revision {revision} token {token:016x}"]
98
    lines.extend(f"line {index} {token ^ index:016x}" for index in range(8))
99
    return ("\n".join(lines) + "\n").encode("utf-8")
100
101
102
def _blob(data: bytes) -> bytes:
103
    return b"data " + str(len(data)).encode() + b"\n" + data + b"\n"
104
105
106
def build_stream(commits: int, seed: int) -> tuple[bytes, MergeHistoryManifest]:
107
    """Return the fast-import stream and the independent model manifest."""
108
    if commits < 1:
109
        raise ValueError("commits must be >= 1")
110
    rng = random.Random(f"{FIXTURE_SCHEMA}:{seed}")
111
    out: list[bytes] = []
112
    main_files: dict[str, bytes] = {}
113
    revision = 0
114
    mark = 0
115
    produced = 0
116
    merges = 0
117
    main_mark: int | None = None
118
    side_serial = 0
119
120
    def emit_commit(ref: bytes, parents: list[int], changes: dict[str, bytes], message: str) -> int:
121
        nonlocal mark, produced
122
        mark += 1
123
        produced += 1
124
        when = str(EPOCH + produced * 60).encode()
125
        body = message.encode("utf-8")
126
        out.append(b"commit " + ref + b"\n")
127
        out.append(b"mark :" + str(mark).encode() + b"\n")
128
        out.append(b"author " + IDENTITY + b" " + when + b" +0000\n")
129
        out.append(b"committer " + IDENTITY + b" " + when + b" +0000\n")
130
        out.append(b"data " + str(len(body)).encode() + b"\n" + body + b"\n")
131
        if parents:
132
            out.append(b"from :" + str(parents[0]).encode() + b"\n")
133
        for parent in parents[1:]:
134
            out.append(b"merge :" + str(parent).encode() + b"\n")
135
        for path in sorted(changes):
136
            out.append(b"M 100644 inline " + path.encode("utf-8") + b"\n")
137
            out.append(_blob(changes[path]))
138
        out.append(b"\n")
139
        return mark
140
141
    def mainline_change() -> dict[str, str]:
142
        nonlocal revision
143
        revision += 1
144
        path = f"src/d{rng.randrange(6):02d}/f{rng.randrange(MAINLINE_FILES):03d}.txt"
145
        return {path: _content(path, revision, rng)}
146
147
    # Root commit: the full mainline tree.
148
    initial = {}
149
    for index in range(MAINLINE_FILES):
150
        path = f"src/d{index % 6:02d}/f{index:03d}.txt"
151
        initial[path] = _content(path, 0, rng)
152
    main_files.update(initial)
153
    main_mark = emit_commit(b"refs/heads/main", [], initial, "fixture root")
154
155
    while produced < commits:
156
        remaining = commits - produced
157
        # A side branch needs >= 1 side commit, one mainline commit (so the
158
        # merge is a true two-parent merge, not a fast-forward), and the merge.
159
        if remaining >= 3 and rng.random() < SIDE_BRANCH_PROBABILITY * 2:
160
            side_len = rng.randint(SIDE_BRANCH_MIN_COMMITS, min(SIDE_BRANCH_MAX_COMMITS, remaining - 2))
161
            side_serial += 1
162
            side_ref = f"refs/heads/side-{side_serial:04d}".encode()
163
            side_changes_total: dict[str, bytes] = {}
164
            side_mark = main_mark
165
            for step in range(side_len):
166
                revision += 1
167
                path = f"side/s{side_serial:04d}/n{step % 2}.txt"
168
                change = {path: _content(path, revision, rng)}
169
                side_changes_total.update(change)
170
                side_mark = emit_commit(side_ref, [side_mark], change, f"side {side_serial} step {step}")
171
            main_change = mainline_change()
172
            main_files.update(main_change)
173
            main_mark = emit_commit(b"refs/heads/main", [main_mark], main_change, f"main {produced}")
174
            # Side paths never collide with mainline paths: the merge tree is
175
            # main's tree plus the side branch's files.
176
            main_files.update(side_changes_total)
177
            main_mark = emit_commit(
178
                b"refs/heads/main", [main_mark, side_mark], side_changes_total,
179
                f"Merge side-{side_serial:04d}",
180
            )
181
            merges += 1
182
            out.append(b"reset " + side_ref + b"\n" + b"from 0000000000000000000000000000000000000000\n\n")
183
            continue
184
        change = mainline_change()
185
        main_files.update(change)
186
        main_mark = emit_commit(b"refs/heads/main", [main_mark], change, f"main {produced}")
187
188
    manifest = MergeHistoryManifest(
189
        schema=FIXTURE_SCHEMA,
190
        commits=produced,
191
        seed=seed,
192
        merge_commits=merges,
193
        root_commits=1,
194
        expected_files={path: hashlib.sha256(data).hexdigest() for path, data in main_files.items()},
195
    )
196
    return b"".join(out) + b"done\n", manifest
197
198
199
def generate(dest: Path, commits: int, seed: int, *, git_bin: str = "git", env: dict[str, str] | None = None) -> MergeHistoryManifest:
200
    """Create a non-bare Git repo at ``dest`` with ``main`` checked out."""
201
    stream, manifest = build_stream(commits, seed)
202
    dest.mkdir(parents=True, exist_ok=False)
203
204
    def git(*args: str, data: bytes | None = None) -> bytes:
205
        result = subprocess.run(
206
            [git_bin, *args], cwd=dest, input=data, env=env,
207
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False,
208
        )
209
        if result.returncode != 0:
210
            raise RuntimeError(f"git {args[0]} failed: {result.stderr.decode(errors='replace')[-2000:]}")
211
        return result.stdout
212
213
    git("init", "-q", "-b", "main")
214
    git("fast-import", "--quiet", "--done", data=stream)
215
    git("checkout", "-q", "-f", "main")
216
    manifest.head = git("rev-parse", "refs/heads/main").decode().strip()
217
    return manifest
218
219
220
__all__ = [
221
    "FIXTURE_SCHEMA",
222
    "MergeHistoryManifest",
223
    "build_stream",
224
    "generate",
225
    "tree_content_hash",
226
    "worktree_files",
227
]