Viewing files at commit d3edd8c5e361Commit detailsView latest

Browse files
scripts/agent_loop_readonly_probe.py 105 lines · 5.1 KB · python
1
#!/usr/bin/env python3
2
"""Read-only Oak probes against a real hosted repo (default oak/oak).
3
4
Companion to agent_loop_hosted_v1 for read-side steps on realistic history
5
that the harness must not write to. One counting proxy lives for the whole
6
sequence so a proxied clone's stored remote stays valid for later steps.
7
Performs no remote writes: clone, log --remote, branch review --remote,
8
branch diff --remote, fetch, and pull on a never-pushed local branch.
9
"""
10
11
from __future__ import annotations
12
13
import argparse
14
import json
15
import os
16
import shutil
17
import subprocess
18
import sys
19
import time
20
from pathlib import Path
21
22
sys.path.insert(0, str(Path(__file__).resolve().parent))
23
24
from agent_loop_hosted import oak_token, serial_rounds  # noqa: E402
25
from oakbench.httpcount import HttpCountingProxy  # noqa: E402
26
from oakbench.runlock import measurement_lock  # noqa: E402
27
28
29
def main(argv=None) -> int:
30
    ap = argparse.ArgumentParser()
31
    ap.add_argument("--oak-bin", required=True)
32
    ap.add_argument("--repo", default="oak/oak")
33
    ap.add_argument("--review-branch", required=True)
34
    ap.add_argument("--workdir", required=True, type=Path)
35
    ap.add_argument("--results", required=True, type=Path)
36
    ap.add_argument("--mode", choices=("proxied", "direct"), default="proxied")
37
    ap.add_argument("--steps", default="clone_shallow,log_remote,review_remote,diff_remote,fetch,pull_after_shallow,pull_noop,clone_full")
38
    ap.add_argument("--trial", type=int, default=0)
39
    a = ap.parse_args(argv)
40
    tok = oak_token()
41
    a.workdir.mkdir(parents=True, exist_ok=True)
42
    a.results.mkdir(parents=True, exist_ok=True)
43
    rows = []
44
    with measurement_lock("agent_loop_readonly_probe"):
45
        proxy = HttpCountingProxy("oak.space").start() if a.mode == "proxied" else None
46
        env = dict(os.environ, OAK_NO_UPDATE_CHECK="1", NO_COLOR="1")
47
        env.pop("OAK_REMOTE", None)
48
        rarg = []
49
        if proxy:
50
            env.update(OAK_API_KEY=tok, OAK_REMOTE=proxy.url, HTTPS_PROXY=proxy.url, https_proxy=proxy.url,
51
                       NO_PROXY="127.0.0.1,localhost", no_proxy="127.0.0.1,localhost")
52
            rarg = ["-r", proxy.url]
53
        ws = a.workdir / f"ro-{a.mode}-{a.trial}"
54
        full = a.workdir / f"ro-full-{a.mode}-{a.trial}"
55
        for d in (ws, full):
56
            if d.exists():
57
                shutil.rmtree(d)
58
        b = a.oak_bin
59
        cmds = {
60
            "clone_shallow": ([b, "clone", "--shallow", a.repo, str(ws)] + rarg, a.workdir),
61
            "log_remote": ([b, "log", "--remote", "--branch", "main", "-n", "20", "--json"], ws),
62
            "review_remote": ([b, "branch", "review", a.review_branch, "--remote", "--json"], ws),
63
            "diff_remote": ([b, "branch", "diff", a.review_branch, "--remote", "--json"], ws),
64
            "fetch": ([b, "fetch"], ws),
65
            "pull_after_shallow": ([b, "pull"], ws),
66
            "pull_noop": ([b, "pull"], ws),
67
            "clone_full": ([b, "clone", a.repo, str(full)] + rarg, a.workdir),
68
        }
69
        for step in a.steps.split(","):
70
            cmd, cwd = cmds[step]
71
            if proxy:
72
                proxy.reset()
73
            t = time.monotonic()
74
            r = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True)
75
            wall = time.monotonic() - t
76
            row = {"lane": "agent_loop_hosted", "scenario": "agent_loop_hosted_v1.readonly", "subject": "oak", "repo": a.repo,
77
                   "remote_repo": a.repo, "remote_server": "oak.space", "remote_transport": "network", "mode": a.mode,
78
                   "trial": a.trial, "step": step, "operation": f"readonly.{step}", "wall_s": wall, "elapsed_ms": wall * 1000.0,
79
                   "rc": r.returncode, "returncode": r.returncode, "output_bytes": len(r.stdout) + len(r.stderr),
80
                   "measurement_source": "httpcount_proxy+git_trace_curl" if proxy else "wall_clock_only"}
81
            if proxy:
82
                time.sleep(0.05)
83
                s = proxy.snapshot()
84
                row.update(http_requests=s.requests, http_request_bytes=s.request_bytes, http_response_bytes=s.response_bytes,
85
                           serial_rounds=serial_rounds(s.records), tunnel_bytes=s.tunnel_bytes, tunnel_connections=len(s.tunnels),
86
                           routes=s.by_route())
87
            if r.returncode != 0:
88
                row["tail"] = (r.stdout[-300:] + r.stderr[-500:]).decode(errors="replace").replace(tok, "<redacted>")
89
            rows.append(row)
90
            print(f"{a.repo} {a.mode} {step:20} rc={r.returncode} {wall:8.3f}s req={row.get('http_requests')} rounds={row.get('serial_rounds')} "
91
                  f"down={row.get('http_response_bytes')} tun={row.get('tunnel_bytes')}", file=sys.stderr, flush=True)
92
            if r.returncode != 0:
93
                print("   ", row["tail"][-400:], file=sys.stderr)
94
        if proxy:
95
            proxy.stop()
96
    out = a.results / f"agent_loop_readonly-{a.repo.replace('/', '_')}-{a.mode}-{a.trial}-{time.strftime('%H%M%S', time.gmtime())}.jsonl"
97
    with out.open("w") as fh:
98
        for r in rows:
99
            fh.write(json.dumps(r) + "\n")
100
    print(out, file=sys.stderr)
101
    return 0
102
103
104
if __name__ == "__main__":
105
    raise SystemExit(main())