| 1 | package main
|
| 2 |
|
| 3 | import (
|
| 4 | "encoding/json"
|
| 5 | "fmt"
|
| 6 | "os"
|
| 7 | "os/exec"
|
| 8 | "strings"
|
| 9 | )
|
| 10 |
|
| 11 | // runCmd runs a command in dir with env, returning combined output as error
|
| 12 | // text on failure. Shared by seeding and any other CLI shell-outs.
|
| 13 | func runCmd(bin, dir string, env []string, args ...string) error {
|
| 14 | cmd := exec.Command(bin, args...)
|
| 15 | cmd.Dir = dir
|
| 16 | cmd.Env = env
|
| 17 | out, err := cmd.CombinedOutput()
|
| 18 | if err != nil {
|
| 19 | return fmt.Errorf("%s %v: %v: %s", bin, args, err, strings.TrimSpace(string(out)))
|
| 20 | }
|
| 21 | return nil
|
| 22 | }
|
| 23 |
|
| 24 | // printLevel writes a one-line human summary of a level to stderr (stdout is
|
| 25 | // reserved for the markdown report).
|
| 26 | func printLevel(r levelResult) {
|
| 27 | line := fmt.Sprintf(" c=%-4d ok=%-6d %8.1f push/s %s push/hr p50=%.0fms p95=%.0fms p99=%.0fms cas=%d err=%d",
|
| 28 | r.Concurrency, r.OK, r.OpsPerSec, humanInt(r.OpsPerSec*3600), r.P50ms, r.P95ms, r.P99ms, r.CASFailures, r.OtherErrors)
|
| 29 | if r.Clones > 0 {
|
| 30 | clonesPerHr := float64(r.CloneOK) / r.WindowSec * 3600
|
| 31 | line += fmt.Sprintf(" | clone_ok=%d %s clone/hr p50=%.0fms", r.CloneOK, humanInt(clonesPerHr), r.CloneP50ms)
|
| 32 | }
|
| 33 | fmt.Fprintln(os.Stderr, line)
|
| 34 | }
|
| 35 |
|
| 36 | // resultsDoc is the top-level JSON: run metadata + ForgeMark's per-level
|
| 37 | // levelResult array (unchanged schema, so charts/consumers built for ForgeMark
|
| 38 | // work against oakmark output too).
|
| 39 | type resultsDoc struct {
|
| 40 | Tool string `json:"tool"`
|
| 41 | RunID string `json:"run_id"`
|
| 42 | Remote string `json:"remote"`
|
| 43 | Transport string `json:"transport"`
|
| 44 | Levels []levelResult `json:"levels"`
|
| 45 | }
|
| 46 |
|
| 47 | func emitJSON(results []levelResult, out, runID, remote, transport string) {
|
| 48 | doc := resultsDoc{Tool: "oakmark", RunID: runID, Remote: remote, Transport: transport, Levels: results}
|
| 49 | b, err := json.MarshalIndent(doc, "", " ")
|
| 50 | if err != nil {
|
| 51 | fmt.Fprintf(os.Stderr, "oakmark: marshal results: %v\n", err)
|
| 52 | return
|
| 53 | }
|
| 54 | if out != "" {
|
| 55 | if err := os.WriteFile(out, b, 0o644); err != nil {
|
| 56 | fmt.Fprintf(os.Stderr, "oakmark: write %s: %v\n", out, err)
|
| 57 | } else {
|
| 58 | fmt.Fprintf(os.Stderr, "oakmark: results → %s\n", out)
|
| 59 | }
|
| 60 | }
|
| 61 | }
|
| 62 |
|
| 63 | // renderReport builds the markdown throughput report — the headline artifact.
|
| 64 | // It leads with per-repo pushes/hour (and clones/hour in session mode) so the
|
| 65 | // numbers sit directly next to Entire's and Cursor's published per-repo claims.
|
| 66 | func renderReport(results []levelResult, runID, remote, transport, strategy string) string {
|
| 67 | var b strings.Builder
|
| 68 | fmt.Fprintf(&b, "# oakmark — oak per-repo throughput\n\n")
|
| 69 | fmt.Fprintf(&b, "- **remote:** `%s`\n- **transport:** `%s` ", remote, transport)
|
| 70 | if transport == "protocol" {
|
| 71 | b.WriteString("_(in-process oak-engine; ForgeMark-comparable ceiling)_\n")
|
| 72 | } else {
|
| 73 | b.WriteString("_(real `oak` CLI; honest real-agent floor)_\n")
|
| 74 | }
|
| 75 | fmt.Fprintf(&b, "- **strategy:** `%s`\n- **run:** `%s`\n\n", strategy, runID)
|
| 76 |
|
| 77 | best := bestPush(results)
|
| 78 | fmt.Fprintf(&b, "**Headline — peak sustained pushes/hour per repo: `%s`** (at c=%d, %.1f push/s).\n\n",
|
| 79 | humanInt(best.OpsPerSec*3600), best.Concurrency, best.OpsPerSec)
|
| 80 | if strategy == "session" {
|
| 81 | bc := bestClone(results)
|
| 82 | if bc.CloneOK > 0 {
|
| 83 | fmt.Fprintf(&b, "**Peak clones/hour per repo: `%s`** (at c=%d).\n\n",
|
| 84 | humanInt(float64(bc.CloneOK)/bc.WindowSec*3600), bc.Concurrency)
|
| 85 | }
|
| 86 | }
|
| 87 |
|
| 88 | b.WriteString("## Push throughput vs concurrency\n\n")
|
| 89 | b.WriteString("| Concurrency | OK pushes | Pushes/s | **Pushes/hour** | p50 ms | p95 ms | p99 ms | p99.9 ms | CAS | Errors |\n")
|
| 90 | b.WriteString("| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n")
|
| 91 | for _, r := range results {
|
| 92 | fmt.Fprintf(&b, "| %d | %d | %.1f | **%s** | %.0f | %.0f | %.0f | %.0f | %d | %d |\n",
|
| 93 | r.Concurrency, r.OK, r.OpsPerSec, humanInt(r.OpsPerSec*3600),
|
| 94 | r.P50ms, r.P95ms, r.P99ms, r.P999ms, r.CASFailures, r.OtherErrors)
|
| 95 | }
|
| 96 | b.WriteString("\n")
|
| 97 |
|
| 98 | if strategy == "session" && anyClones(results) {
|
| 99 | b.WriteString("## Clone (read) throughput vs concurrency\n\n")
|
| 100 | b.WriteString("| Concurrency | OK clones | **Clones/hour** | clone p50 ms | clone p95 ms | clone p99 ms |\n")
|
| 101 | b.WriteString("| ---: | ---: | ---: | ---: | ---: | ---: |\n")
|
| 102 | for _, r := range results {
|
| 103 | fmt.Fprintf(&b, "| %d | %d | **%s** | %.0f | %.0f | %.0f |\n",
|
| 104 | r.Concurrency, r.CloneOK, humanInt(float64(r.CloneOK)/r.WindowSec*3600),
|
| 105 | r.CloneP50ms, r.CloneP95ms, r.CloneP99ms)
|
| 106 | }
|
| 107 | b.WriteString("\n")
|
| 108 | }
|
| 109 |
|
| 110 | b.WriteString("## Reference — published per-repo claims\n\n")
|
| 111 | b.WriteString("| Source | Pushes/hour | Clones/hour |\n| --- | ---: | ---: |\n")
|
| 112 | b.WriteString("| Entire (blog) | 21,000,000 | 570,000 |\n")
|
| 113 | b.WriteString("| Cursor Origin (blog) | 81,000 | 296,000 |\n")
|
| 114 | fmt.Fprintf(&b, "| **oak (%s, this run)** | **%s** | %s |\n\n",
|
| 115 | transport, humanInt(best.OpsPerSec*3600), cloneHeadline(results, strategy))
|
| 116 |
|
| 117 | b.WriteString("> Methodology note: oak's push is not a git `receive-pack` — it is a content-addressed, chunked, flat-branch protocol. ")
|
| 118 | b.WriteString("These numbers are measured at the **workload** level (an agent publishing a changeset of N files), the same level the referenced per-repo claims live at. ")
|
| 119 | b.WriteString("Warm-up is excluded; latency percentiles are nearest-rank over successful pushes; pushes/hour = successful pushes/s × 3600. ")
|
| 120 | if transport == "cli" {
|
| 121 | b.WriteString("The `cli` transport is the real-agent floor and can become **generator-bound** before the server does at high concurrency — check generator CPU; the `protocol` transport is the ceiling.\n")
|
| 122 | } else {
|
| 123 | b.WriteString("The `protocol` transport drives oak's wire in-process (no CLI/subprocess overhead), analogous to ForgeMark driving go-git in-process.\n")
|
| 124 | }
|
| 125 | return b.String()
|
| 126 | }
|
| 127 |
|
| 128 | func bestPush(rs []levelResult) levelResult {
|
| 129 | var best levelResult
|
| 130 | for _, r := range rs {
|
| 131 | if r.OpsPerSec > best.OpsPerSec {
|
| 132 | best = r
|
| 133 | }
|
| 134 | }
|
| 135 | return best
|
| 136 | }
|
| 137 |
|
| 138 | func bestClone(rs []levelResult) levelResult {
|
| 139 | var best levelResult
|
| 140 | for _, r := range rs {
|
| 141 | cph := float64(r.CloneOK) / r.WindowSec
|
| 142 | bph := float64(best.CloneOK) / max1(best.WindowSec)
|
| 143 | if cph > bph {
|
| 144 | best = r
|
| 145 | }
|
| 146 | }
|
| 147 | return best
|
| 148 | }
|
| 149 |
|
| 150 | func anyClones(rs []levelResult) bool {
|
| 151 | for _, r := range rs {
|
| 152 | if r.Clones > 0 {
|
| 153 | return true
|
| 154 | }
|
| 155 | }
|
| 156 | return false
|
| 157 | }
|
| 158 |
|
| 159 | func cloneHeadline(rs []levelResult, strategy string) string {
|
| 160 | if strategy != "session" {
|
| 161 | return "—"
|
| 162 | }
|
| 163 | bc := bestClone(rs)
|
| 164 | if bc.CloneOK == 0 {
|
| 165 | return "—"
|
| 166 | }
|
| 167 | return humanInt(float64(bc.CloneOK) / bc.WindowSec * 3600)
|
| 168 | }
|
| 169 |
|
| 170 | func max1(f float64) float64 {
|
| 171 | if f <= 0 {
|
| 172 | return 1
|
| 173 | }
|
| 174 | return f
|
| 175 | }
|
| 176 |
|
| 177 | // humanInt formats a rate with thousands separators and no decimals, e.g.
|
| 178 | // 1234567 → "1,234,567".
|
| 179 | func humanInt(f float64) string {
|
| 180 | n := int64(f + 0.5)
|
| 181 | s := fmt.Sprintf("%d", n)
|
| 182 | neg := strings.HasPrefix(s, "-")
|
| 183 | if neg {
|
| 184 | s = s[1:]
|
| 185 | }
|
| 186 | var parts []string
|
| 187 | for len(s) > 3 {
|
| 188 | parts = append([]string{s[len(s)-3:]}, parts...)
|
| 189 | s = s[:len(s)-3]
|
| 190 | }
|
| 191 | parts = append([]string{s}, parts...)
|
| 192 | out := strings.Join(parts, ",")
|
| 193 | if neg {
|
| 194 | out = "-" + out
|
| 195 | }
|
| 196 | return out
|
| 197 | }
|