| 1 | package main
|
| 2 |
|
| 3 | import (
|
| 4 | "math"
|
| 5 | "sort"
|
| 6 | "time"
|
| 7 | )
|
| 8 |
|
| 9 | // outcome classifies a single push attempt.
|
| 10 | type outcome int
|
| 11 |
|
| 12 | const (
|
| 13 | outcomeOK outcome = iota // push landed
|
| 14 | outcomeCAS // rejected by ref CAS / non-fast-forward (contention)
|
| 15 | outcomeErr // any other failure (transport, auth, server error)
|
| 16 | )
|
| 17 |
|
| 18 | // opKind distinguishes which operation a sample measured. Zero value is opPush
|
| 19 | // so existing push-only call sites need no change.
|
| 20 | type opKind int
|
| 21 |
|
| 22 | const (
|
| 23 | opPush opKind = iota // a receive-pack push (the write path)
|
| 24 | opClone // a clone (the read path; session strategy)
|
| 25 | )
|
| 26 |
|
| 27 | // sample is one recorded operation. offset is measured from the start of the
|
| 28 | // concurrency level so warm-up samples can be dropped post-hoc.
|
| 29 | type sample struct {
|
| 30 | offset time.Duration
|
| 31 | dur time.Duration
|
| 32 | res outcome
|
| 33 | op opKind
|
| 34 | msg string // raw error text, retained only when res == outcomeErr
|
| 35 | }
|
| 36 |
|
| 37 | // errMsg returns the message to retain on a sample. Only genuine errors carry
|
| 38 | // text; OK and CAS outcomes stay message-free.
|
| 39 | func errMsg(res outcome, err error) string {
|
| 40 | if res != outcomeErr || err == nil {
|
| 41 | return ""
|
| 42 | }
|
| 43 | return err.Error()
|
| 44 | }
|
| 45 |
|
| 46 | // levelResult is the published summary for one concurrency level. JSON tags
|
| 47 | // are the columns charts consume.
|
| 48 | type levelResult struct {
|
| 49 | Concurrency int `json:"concurrency"`
|
| 50 | Strategy string `json:"strategy"`
|
| 51 | Repos int `json:"repos"`
|
| 52 | Nodes int `json:"nodes"`
|
| 53 | WindowSec float64 `json:"window_sec"`
|
| 54 | Pushes int `json:"pushes"` // attempts in the measured window
|
| 55 | OK int `json:"ok"` // successful pushes
|
| 56 | CASFailures int `json:"cas_failures"`
|
| 57 | OtherErrors int `json:"other_errors"`
|
| 58 | OpsPerSec float64 `json:"ops_per_sec"` // OK / window
|
| 59 | P50ms float64 `json:"p50_ms"`
|
| 60 | P95ms float64 `json:"p95_ms"`
|
| 61 | P99ms float64 `json:"p99_ms"`
|
| 62 | P999ms float64 `json:"p999_ms"`
|
| 63 | Maxms float64 `json:"max_ms"`
|
| 64 | CommitFiles string `json:"commit_files"` // e.g. "1-10 x 2048B"
|
| 65 |
|
| 66 | // Session-strategy read side (op==clone). Omitted in push/repo modes.
|
| 67 | Clones int `json:"clones,omitempty"`
|
| 68 | CloneOK int `json:"clone_ok,omitempty"`
|
| 69 | CloneErrors int `json:"clone_errors,omitempty"`
|
| 70 | CloneP50ms float64 `json:"clone_p50_ms,omitempty"`
|
| 71 | CloneP95ms float64 `json:"clone_p95_ms,omitempty"`
|
| 72 | CloneP99ms float64 `json:"clone_p99_ms,omitempty"`
|
| 73 |
|
| 74 | // Distinct error messages (normalized) with occurrence counts, for the err
|
| 75 | // bucket only. Omitted when there were no errors.
|
| 76 | ErrorMessages []errGroup `json:"error_messages,omitempty"`
|
| 77 | CloneErrorMessages []errGroup `json:"clone_error_messages,omitempty"`
|
| 78 | }
|
| 79 |
|
| 80 | // errGroup is one distinct (normalized) error message and how often it occurred.
|
| 81 | type errGroup struct {
|
| 82 | Message string `json:"message"`
|
| 83 | Count int `json:"count"`
|
| 84 | }
|
| 85 |
|
| 86 | // maxDistinctErrors caps how many distinct messages a level reports; the rest
|
| 87 | // fold into an "(other errors)" bucket.
|
| 88 | const maxDistinctErrors = 20
|
| 89 |
|
| 90 | // errGroups folds raw error messages into a deterministic, bounded breakdown:
|
| 91 | // normalize each distinct raw once (collapsing this run's volatile URLs/refs),
|
| 92 | // tally by the resulting key, then keep the maxDistinctErrors most frequent
|
| 93 | // (count desc, message asc) and fold the remainder into "(other errors)".
|
| 94 | // Returns nil when there were no errors, so the JSON field is omitted.
|
| 95 | func errGroups(raw map[string]int) []errGroup {
|
| 96 | if len(raw) == 0 {
|
| 97 | return nil
|
| 98 | }
|
| 99 | // Normalize once per distinct raw message, not once per occurrence — an
|
| 100 | // outage repeating one error tens of thousands of times normalizes it once.
|
| 101 | counts := make(map[string]int, len(raw))
|
| 102 | for msg, n := range raw {
|
| 103 | counts[normalizeErr(msg)] += n
|
| 104 | }
|
| 105 | out := make([]errGroup, 0, len(counts))
|
| 106 | for msg, n := range counts {
|
| 107 | out = append(out, errGroup{Message: msg, Count: n})
|
| 108 | }
|
| 109 | byFreq := func(i, j int) bool {
|
| 110 | if out[i].Count != out[j].Count {
|
| 111 | return out[i].Count > out[j].Count
|
| 112 | }
|
| 113 | return out[i].Message < out[j].Message
|
| 114 | }
|
| 115 | sort.Slice(out, byFreq)
|
| 116 | if len(out) > maxDistinctErrors {
|
| 117 | other := 0
|
| 118 | for _, g := range out[maxDistinctErrors:] {
|
| 119 | other += g.Count
|
| 120 | }
|
| 121 | out = append(out[:maxDistinctErrors], errGroup{Message: "(other errors)", Count: other})
|
| 122 | sort.Slice(out, byFreq) // the overflow bucket may outrank the kept singletons
|
| 123 | }
|
| 124 | return out
|
| 125 | }
|
| 126 |
|
| 127 | // summarize folds raw samples into a levelResult, dropping anything inside the
|
| 128 | // warm-up window and computing exact percentiles over the OK samples.
|
| 129 | func summarize(samples []sample, concurrency int, strategy string, repos, nodes int,
|
| 130 | warmup, window time.Duration, commitFiles string) levelResult {
|
| 131 | r := levelResult{
|
| 132 | Concurrency: concurrency,
|
| 133 | Strategy: strategy,
|
| 134 | Repos: repos,
|
| 135 | Nodes: nodes,
|
| 136 | WindowSec: window.Seconds(),
|
| 137 | CommitFiles: commitFiles,
|
| 138 | }
|
| 139 |
|
| 140 | var okDurs, cloneDurs []float64
|
| 141 | pushErrs := map[string]int{}
|
| 142 | cloneErrs := map[string]int{}
|
| 143 | for _, s := range samples {
|
| 144 | if s.offset < warmup {
|
| 145 | continue // warm-up: excluded from every statistic
|
| 146 | }
|
| 147 | if s.op == opClone {
|
| 148 | r.Clones++
|
| 149 | if s.res == outcomeOK {
|
| 150 | r.CloneOK++
|
| 151 | cloneDurs = append(cloneDurs, float64(s.dur)/float64(time.Millisecond))
|
| 152 | } else {
|
| 153 | r.CloneErrors++
|
| 154 | cloneErrs[s.msg]++
|
| 155 | }
|
| 156 | continue
|
| 157 | }
|
| 158 | r.Pushes++
|
| 159 | switch s.res {
|
| 160 | case outcomeOK:
|
| 161 | r.OK++
|
| 162 | okDurs = append(okDurs, float64(s.dur)/float64(time.Millisecond))
|
| 163 | case outcomeCAS:
|
| 164 | r.CASFailures++
|
| 165 | case outcomeErr:
|
| 166 | r.OtherErrors++
|
| 167 | pushErrs[s.msg]++
|
| 168 | }
|
| 169 | }
|
| 170 | r.ErrorMessages = errGroups(pushErrs)
|
| 171 | r.CloneErrorMessages = errGroups(cloneErrs)
|
| 172 |
|
| 173 | if window > 0 {
|
| 174 | r.OpsPerSec = float64(r.OK) / window.Seconds()
|
| 175 | }
|
| 176 | if len(okDurs) > 0 {
|
| 177 | sort.Float64s(okDurs)
|
| 178 | r.P50ms = percentile(okDurs, 50)
|
| 179 | r.P95ms = percentile(okDurs, 95)
|
| 180 | r.P99ms = percentile(okDurs, 99)
|
| 181 | r.P999ms = percentile(okDurs, 99.9)
|
| 182 | r.Maxms = okDurs[len(okDurs)-1]
|
| 183 | }
|
| 184 | if len(cloneDurs) > 0 {
|
| 185 | sort.Float64s(cloneDurs)
|
| 186 | r.CloneP50ms = percentile(cloneDurs, 50)
|
| 187 | r.CloneP95ms = percentile(cloneDurs, 95)
|
| 188 | r.CloneP99ms = percentile(cloneDurs, 99)
|
| 189 | }
|
| 190 | return r
|
| 191 | }
|
| 192 |
|
| 193 | // percentile returns the nearest-rank percentile of an already-sorted slice.
|
| 194 | func percentile(sorted []float64, p float64) float64 {
|
| 195 | if len(sorted) == 0 {
|
| 196 | return 0
|
| 197 | }
|
| 198 | rank := int(math.Ceil(p/100*float64(len(sorted)))) - 1
|
| 199 | rank = max(rank, 0)
|
| 200 | rank = min(rank, len(sorted)-1)
|
| 201 | return sorted[rank]
|
| 202 | }
|