Viewing files at commit d3edd8c5e361Commit detailsView latest

Browse files
scripts/seed_agent_loop_fixture.py 190 lines · 8.7 KB · python
1
#!/usr/bin/env python3
2
"""Seed the identical Oak/Git fixture history for ``agent_loop_hosted_v1``.
3
4
Setup only; nothing here is timed. Produces a local Git repository whose
5
history is deterministic given (source tree, seed):
6
7
* commit 1 imports a source tree (oak/oak's tree, with CI config directories
8
  ``.github/`` and ``.oak/workflows/`` removed so neither host runs CI);
9
* commits 2..N edit synthetic files under ``bench-history/`` (1-3 files per
10
  commit, fixed PRNG), and every ``--merge-every`` steps a two-commit side
11
  branch is merged with ``--no-ff`` (the merge commit also appends to
12
  ``bench-history/MERGES.txt`` so every commit changes the tree).
13
14
``--publish-oak-clone`` then replays the same commits into an Oak clone of
15
the (already created) scratch repo as branch commits landed by hosted squash
16
merges -- see ``publish_oak_cycles`` for why direct pushes of ``main`` are
17
impossible today. The GitHub fixture is a plain ``git push`` of ``main``.
18
19
Measured seeding constraints (2026-09-27, oak 0.105.0 vs oak.space):
20
* a single push carrying >128 blobs is refused client-side (server does not
21
  advertise ``/api/capabilities`` staged push), so the source tree is imported
22
  in 100-file commits and the very first push can hold only one of them;
23
* a fresh ``oak clone file://<git>`` import cannot extend a remote ``main``
24
  ("push admission phase could not prove older remote commit edge(s)").
25
"""
26
27
from __future__ import annotations
28
29
import argparse
30
import os
31
import random
32
import shutil
33
import subprocess
34
import sys
35
from pathlib import Path
36
37
EXCLUDE_DIRS = {".oak", ".git", ".github"}
38
IMPORT_SLICE = 100
39
40
41
def run(cmd, cwd, env=None, quiet=True):
42
    r = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True)
43
    if r.returncode != 0:
44
        raise SystemExit(f"command failed ({r.returncode}): {cmd}\n{r.stdout[-2000:]}\n{r.stderr[-2000:]}")
45
    return r.stdout
46
47
48
def copy_tree(src: Path, dst: Path) -> int:
49
    n = 0
50
    for root, dirs, files in os.walk(src):
51
        rel = Path(root).relative_to(src)
52
        if rel == Path("."):
53
            dirs[:] = [d for d in dirs if d not in EXCLUDE_DIRS]
54
        for f in files:
55
            s = Path(root) / f
56
            d = dst / rel / f
57
            d.parent.mkdir(parents=True, exist_ok=True)
58
            shutil.copy2(s, d)
59
            n += 1
60
    return n
61
62
63
def git_env(i: int) -> dict:
64
    ts = f"{1_780_000_000 + i * 600} +0000"
65
    return dict(
66
        os.environ,
67
        GIT_AUTHOR_NAME="bench-seed", GIT_AUTHOR_EMAIL="[email protected]",
68
        GIT_COMMITTER_NAME="bench-seed", GIT_COMMITTER_EMAIL="[email protected]",
69
        GIT_AUTHOR_DATE=ts, GIT_COMMITTER_DATE=ts,
70
    )
71
72
73
def edit(repo: Path, rng: random.Random, step: int, files: int) -> None:
74
    for _ in range(rng.randint(1, 3)):
75
        p = repo / "bench-history" / f"f{rng.randrange(files):03d}.txt"
76
        p.parent.mkdir(parents=True, exist_ok=True)
77
        with p.open("a") as fh:
78
            for j in range(rng.randint(1, 6)):
79
                fh.write(f"step {step} line {j} value {rng.getrandbits(48):012x}\n")
80
81
82
def seed_git(src: Path, repo: Path, commits: int, seed: int, merge_every: int, files: int) -> None:
83
    rng = random.Random(seed)
84
    repo.mkdir(parents=True)
85
    run(["git", "init", "-q", "-b", "main"], repo)
86
    copy_tree(src, repo)
87
    # Import in slices of <= IMPORT_SLICE files: the hosted Oak server currently
88
    # refuses (client-side, before mutation) any single push carrying more than
89
    # MAPPING_PROOF_MAX_BLOBS=128 blobs, so the Oak mirror must be publishable
90
    # in increments. Git gets the identical commits.
91
    paths = sorted(run(["git", "ls-files", "--others", "--exclude-standard"], repo).split("\n"))
92
    paths = [x for x in paths if x]
93
    count = 0
94
    for i in range(0, len(paths), IMPORT_SLICE):
95
        run(["git", "add", "--"] + paths[i:i + IMPORT_SLICE], repo)
96
        count += 1
97
        run(["git", "commit", "-q", "-m", f"import source tree part {count}"], repo, git_env(count))
98
    run(["git", "add", "-A"], repo)
99
    if run(["git", "status", "--porcelain"], repo).strip():
100
        count += 1
101
        run(["git", "commit", "-q", "-m", "import source tree remainder"], repo, git_env(count))
102
    step = 0
103
    while count < commits:
104
        step += 1
105
        if merge_every and step % merge_every == 0 and commits - count >= 3:
106
            br = f"side-{step}"
107
            run(["git", "switch", "-q", "-c", br], repo)
108
            for k in range(2):
109
                edit(repo, rng, step * 10 + k, files)
110
                run(["git", "add", "-A"], repo)
111
                count += 1
112
                run(["git", "commit", "-q", "-m", f"side {step}.{k}"], repo, git_env(count))
113
            run(["git", "switch", "-q", "main"], repo)
114
            run(["git", "merge", "-q", "--no-ff", "--no-commit", br], repo, git_env(count))
115
            with (repo / "bench-history" / "MERGES.txt").open("a") as fh:
116
                fh.write(f"merged {br}\n")
117
            run(["git", "add", "-A"], repo)
118
            count += 1
119
            run(["git", "commit", "-q", "-m", f"Merge {br}"], repo, git_env(count))
120
            run(["git", "branch", "-q", "-D", br], repo)
121
        else:
122
            edit(repo, rng, step, files)
123
            run(["git", "add", "-A"], repo)
124
            count += 1
125
            run(["git", "commit", "-q", "-m", f"step {step}"], repo, git_env(count))
126
127
128
def publish_oak_cycles(git_repo: Path, oak_dir: Path, oak_bin: str, start_after: int, per_cycle: int = 10) -> int:
129
    """Replay Git commits onto an Oak clone as branch commits + hosted squash merges.
130
131
    Hosted Oak only lets ``main`` advance by server squash-merge (a first push
132
    can create ``main``, but the server refuses any push over 128 blobs without
133
    a staged protocol it does not yet advertise). So each cycle commits up to
134
    ``per_cycle`` Git commits (in ``--topo-order``) on the current branch and
135
    ends at a first-parent commit with ``oak desc`` + ``oak merge``. The
136
    squash commit's ``merge_parent`` is the branch head, so every replayed
137
    commit stays reachable from main; main's first-parent chain is shorter
138
    than Git's (one squash per cycle). ``start_after`` skips commits that are
139
    already on the Oak remote. Stops (never retries) on any merge failure.
140
    """
141
    env = dict(os.environ, OAK_NO_UPDATE_CHECK="1")
142
    shas = run(["git", "rev-list", "--reverse", "--topo-order", "main"], git_repo).split()
143
    first_parent = set(run(["git", "rev-list", "--first-parent", "main"], git_repo).split())
144
    idx = str(oak_dir.parent / (oak_dir.name + ".replay-index"))
145
    genv = dict(os.environ, GIT_INDEX_FILE=idx, GIT_WORK_TREE=str(oak_dir), GIT_DIR=str(git_repo / ".git"))
146
    pending = 0
147
    cycles = 0
148
    for n, sha in enumerate(shas, 1):
149
        if n <= start_after:
150
            continue
151
        run(["git", "read-tree", sha], git_repo, genv)
152
        run(["git", "checkout-index", "-a", "-f"], git_repo, genv)
153
        run([oak_bin, "commit", "--no-verify", "--quiet"], oak_dir, env)
154
        pending += 1
155
        if (pending >= per_cycle and sha in first_parent) or n == len(shas):
156
            cycles += 1
157
            run([oak_bin, "desc", f"seed cycle {cycles}: replay through git commit {n} ({sha[:12]})"], oak_dir, env)
158
            out = run([oak_bin, "merge", "--json"], oak_dir, env)
159
            print(f"cycle {cycles} through commit {n}: {out.strip()[:140]}", file=sys.stderr)
160
            pending = 0
161
    if os.path.exists(idx):
162
        os.unlink(idx)
163
    return cycles
164
165
166
def main(argv=None) -> int:
167
    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
168
    ap.add_argument("--source", required=True, type=Path)
169
    ap.add_argument("--git-dir", required=True, type=Path)
170
    ap.add_argument("--oak-bin", default="oak")
171
    ap.add_argument("--commits", type=int, default=1000)
172
    ap.add_argument("--seed", type=int, default=20260927)
173
    ap.add_argument("--merge-every", type=int, default=25)
174
    ap.add_argument("--files", type=int, default=50)
175
    ap.add_argument("--publish-oak-clone", type=Path, help="Oak clone of the target repo to replay into via squash-merge cycles")
176
    ap.add_argument("--start-after", type=int, default=0, help="Git commits (topo order) already on the Oak remote")
177
    ap.add_argument("--per-cycle", type=int, default=10)
178
    a = ap.parse_args(argv)
179
    if a.publish_oak_clone:
180
        print("cycles:", publish_oak_cycles(a.git_dir, a.publish_oak_clone, a.oak_bin, a.start_after, a.per_cycle))
181
        return 0
182
    seed_git(a.source, a.git_dir, a.commits, a.seed, a.merge_every, a.files)
183
    print("git commits:", run(["git", "rev-list", "--count", "main"], a.git_dir).strip(),
184
          "merges:", run(["git", "rev-list", "--count", "--merges", "main"], a.git_dir).strip(),
185
          "tree:", run(["git", "rev-parse", "main^{tree}"], a.git_dir).strip())
186
    return 0
187
188
189
if __name__ == "__main__":
190
    raise SystemExit(main())