#!/usr/bin/env python3

"""Generate deterministic monorepo-shaped git fixtures via fast-import.

Produces synthetic repositories whose scale knobs (tree width, history depth,

ref count) are independent, for the xl/monorepo fixture family declared in

config/fixtures.json. The same seed always yields byte-identical history:

fixed author/committer identity, timestamps stepping from a fixed epoch,

seeded RNG for paths/content/touch-sets, deterministic fast-import marks —

so commit SHAs are reproducible run-to-run and machine-to-machine.

The whole history is written as a single fast-import stream piped to one

`git fast-import` subprocess (no per-file subprocess calls), which keeps

generation tractable at 500k-file / 100k-commit scale. The working tree is

left unpopulated; check out refs/heads/main if a materialized tree is needed.

Layout: file i lives at dir{i//1000}/file{i}.txt. The initial commit carries

the full tree; each subsequent commit rewrites a seeded subset (~10 files).

A seeded fraction of files (--binary-share) holds incompressible bytes so

pack/transfer measurements are not dominated by pathological compression.

refs/heads/main is the primary branch; refs/heads/branch-{i} and

refs/tags/tag-{i} point at seeded-deterministic commits.

After import, HISTORY_MANIFEST.json (schema_version 1) records the knobs and

the resulting HEAD SHA for fixture-identity checks.

Exit codes: 0 ok, 1 usage/generation error, 3 git unavailable (skip-row

honesty: the caller can record a skip instead of a bogus result).

Usage:

  python3 scripts/make_monorepo_fixture.py --out /tmp/xl-tree \\

      --seed oakbench-xl-tree-v1 --files 500000 --commits 1000 --refs 100

"""

from __future__ import annotations

import argparse

import hashlib

import json

import os

import random

import shutil

import subprocess

import sys

from pathlib import Path

GENERATOR_VERSION = "monorepo-v1"

IDENTITY = b"Oak Bench <[email protected]>"

EPOCH = 1_700_000_000  # first commit timestamp; +60s per commit

TOUCHES_PER_COMMIT = 10

PRIMARY_BRANCH = b"refs/heads/main"

MANIFEST_NAME = "HISTORY_MANIFEST.json"

QUICK_DEFAULTS = {"files": 50, "commits": 20, "refs": 5}

FULL_DEFAULTS = {"files": 10_000, "commits": 1_000, "refs": 50}

WORDS = (

    "handler request response buffer stream parse encode decode commit tree "

    "branch merge index cache token batch worker queue retry timeout config "

    "schema field record column filter reduce visit walk node edge graph "

    "mount hydrate snapshot manifest oracle subject lane scenario operation"

).split()

def make_rng(seed: str, *scope: object) -> random.Random:

    digest = hashlib.sha256(":".join([seed, *map(str, scope)]).encode()).digest()

    return random.Random(int.from_bytes(digest[:8], "big"))

def file_path(i: int) -> str:

    return f"dir{i // 1000}/file{i}.txt"

def text_blob(seed: str, i: int, version: int, quick: bool) -> bytes:

    rng = make_rng(seed, "text", i, version)

    # Log-normal-ish sizes centered near ~3.5 KB so 500k files land near 2 GiB.

    lines = 2 if quick else max(8, int(rng.lognormvariate(4.0, 0.6)))

    out = [f"# {GENERATOR_VERSION} {file_path(i)} v{version}"]

    for n in range(lines):

        w = [rng.choice(WORDS) for _ in range(rng.randint(4, 9))]

        out.append(f"{w[0]}_{rng.randint(0, 999)} = {w[1]}({', '.join(w[2:])})  # {n}")

    return ("\n".join(out) + "\n").encode()

def binary_blob(seed: str, i: int, version: int, quick: bool) -> bytes:

    rng = make_rng(seed, "binary", i, version)

    size = 64 if quick else rng.randint(4_096, 32_768)

    return rng.randbytes(size)

def binary_file_set(seed: str, files: int, binary_share: float) -> frozenset[int]:

    count = int(round(files * binary_share))

    if count <= 0:

        return frozenset()

    return frozenset(make_rng(seed, "binary-set").sample(range(files), count))

class StreamWriter:

    """Emits a deterministic fast-import stream to a binary file object."""

    def __init__(self, fh, seed: str, quick: bool, binary_files: frozenset[int]):

        self.fh = fh

        self.seed = seed

        self.quick = quick

        self.binary_files = binary_files

        self.mark = 0

        self.commit_marks: list[int] = []

    def blob_for(self, i: int, version: int) -> int:

        if i in self.binary_files:

            data = binary_blob(self.seed, i, version, self.quick)

        else:

            data = text_blob(self.seed, i, version, self.quick)

        self.mark += 1

        self.fh.write(b"blob\nmark :%d\ndata %d\n" % (self.mark, len(data)))

        self.fh.write(data)

        self.fh.write(b"\n")

        return self.mark

    def commit(self, index: int, message: str, changes: list[tuple[str, int]]) -> None:

        self.mark += 1

        ident = IDENTITY + b" %d +0000\n" % (EPOCH + 60 * index)

        msg = message.encode()

        fh = self.fh

        fh.write(b"commit " + PRIMARY_BRANCH + b"\nmark :%d\n" % self.mark)

        fh.write(b"author " + ident)

        fh.write(b"committer " + ident)

        fh.write(b"data %d\n" % len(msg))

        fh.write(msg)

        fh.write(b"\n")

        for path, blob_mark in changes:

            fh.write(b"M 100644 :%d %s\n" % (blob_mark, path.encode()))

        fh.write(b"\n")

        self.commit_marks.append(self.mark)

    def reset(self, ref: str, commit_mark: int) -> None:

        self.fh.write(b"reset %s\nfrom :%d\n\n" % (ref.encode(), commit_mark))

def write_stream(fh, *, seed: str, files: int, commits: int, refs: int,

                 tags: int, binary_share: float, quick: bool) -> None:

    writer = StreamWriter(fh, seed, quick, binary_file_set(seed, files, binary_share))

    fh.write(b"feature done\n")

    changes = [(file_path(i), writer.blob_for(i, 0)) for i in range(files)]

    writer.commit(0, f"initial tree: {files} files", changes)

    touches = min(TOUCHES_PER_COMMIT, files)

    for c in range(1, commits):

        subset = sorted(make_rng(seed, "touch", c).sample(range(files), touches))

        changes = [(file_path(i), writer.blob_for(i, c)) for i in subset]

        writer.commit(c, f"commit {c}: touch {touches} files", changes)

    marks = writer.commit_marks

    for i in range(refs):

        target = make_rng(seed, "branch", i).randrange(commits)

        writer.reset(f"refs/heads/branch-{i}", marks[target])

    for i in range(tags):

        target = make_rng(seed, "tag", i).randrange(commits)

        writer.reset(f"refs/tags/tag-{i}", marks[target])

    fh.write(b"done\n")

def git_env() -> dict[str, str]:

    # Isolate from user/system git config so generation is config-independent.

    env = dict(os.environ)

    env["GIT_CONFIG_GLOBAL"] = os.devnull

    env["GIT_CONFIG_NOSYSTEM"] = "1"

    return env

def run_git(args: list[str], cwd: Path) -> str:

    proc = subprocess.run(["git", *args], cwd=cwd, env=git_env(), check=True,

                          capture_output=True, text=True)

    return proc.stdout.strip()

def generate(out: Path, *, seed: str, files: int, commits: int, refs: int,

             tags: int, binary_share: float, quick: bool) -> dict[str, object]:

    run_git(["init", "-q"], out)

    importer = subprocess.Popen(

        ["git", "fast-import", "--quiet", "--done"],

        cwd=out, env=git_env(), stdin=subprocess.PIPE, stderr=subprocess.PIPE,

    )

    assert importer.stdin is not None

    try:

        write_stream(importer.stdin, seed=seed, files=files, commits=commits,

                     refs=refs, tags=tags, binary_share=binary_share, quick=quick)

    except BrokenPipeError:

        pass

    finally:

        try:

            importer.stdin.close()

        except BrokenPipeError:

            pass

    stderr = importer.stderr.read()

    importer.stderr.close()

    importer.wait()

    if importer.returncode != 0:

        sys.stderr.write(stderr.decode(errors="replace"))

        raise RuntimeError(f"git fast-import failed with exit code {importer.returncode}")

    run_git(["symbolic-ref", "HEAD", PRIMARY_BRANCH.decode()], out)

    head_sha = run_git(["rev-parse", "HEAD"], out)

    manifest = {

        "schema_version": 1,

        "seed": seed,

        "files": files,

        "commits": commits,

        "refs": refs,

        "tags": tags,

        "head_sha": head_sha,

        "generator_version": GENERATOR_VERSION,

    }

    (out / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2) + "\n")

    return manifest

def parse_args() -> argparse.Namespace:

    ap = argparse.ArgumentParser(

        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)

    ap.add_argument("--out", type=Path, required=True, help="target directory (git init + fast-import)")

    ap.add_argument("--files", type=int, default=None)

    ap.add_argument("--commits", type=int, default=None)

    ap.add_argument("--refs", type=int, default=None)

    ap.add_argument("--tags", type=int, default=0)

    ap.add_argument("--seed", required=True)

    ap.add_argument("--binary-share", type=float, default=0.02,

                    help="fraction of files with seeded binary content (default 0.02)")

    ap.add_argument("--quick", action="store_true", help="tiny sizes for tests")

    return ap.parse_args()

def main() -> int:

    args = parse_args()

    if shutil.which("git") is None:

        print("make_monorepo_fixture: git not found on PATH; cannot generate "

              "fixture — record this run as skipped, not failed", file=sys.stderr)

        return 3

    defaults = QUICK_DEFAULTS if args.quick else FULL_DEFAULTS

    files = args.files if args.files is not None else defaults["files"]

    commits = args.commits if args.commits is not None else defaults["commits"]

    refs = args.refs if args.refs is not None else defaults["refs"]

    if files < 1 or commits < 1:

        print("--files and --commits must be >= 1", file=sys.stderr)

        return 1

    if refs < 0 or args.tags < 0:

        print("--refs and --tags must be >= 0", file=sys.stderr)

        return 1

    if not 0.0 <= args.binary_share <= 1.0:

        print("--binary-share must be within [0, 1]", file=sys.stderr)

        return 1

    out = args.out

    out.mkdir(parents=True, exist_ok=True)

    if any(out.iterdir()):

        print(f"refusing to write into non-empty {out}", file=sys.stderr)

        return 1

    manifest = generate(out, seed=args.seed, files=files, commits=commits,

                        refs=refs, tags=args.tags,

                        binary_share=args.binary_share, quick=args.quick)

    print(f"{GENERATOR_VERSION} seed={args.seed} files={files} commits={commits} "

          f"refs={refs} tags={args.tags} -> {out}")

    print(f"HEAD {manifest['head_sha']}")

    return 0

if __name__ == "__main__":

    raise SystemExit(main())