| 1 | // Shared reporting math + ingest transforms for the dashboard worker.
|
| 2 | //
|
| 3 | // Every formula here is a faithful port of the Python definitions in
|
| 4 | // scripts/oakbench/stats.py and scripts/oakbench/reporting.py. Parity is
|
| 5 | // enforced by tests/fixtures/reporting_golden.json, asserted from BOTH sides
|
| 6 | // (tests/test_reporting_golden.py and cloudflare/reporting_parity.test.mjs).
|
| 7 | // Null means unmeasured (ADR-0002): a null input NEVER becomes 0.
|
| 8 |
|
| 9 | // --- stats.py ports -------------------------------------------------------
|
| 10 |
|
| 11 | /** Port of oakbench.stats.percentile_nearest (nearest-rank, empty -> null). */
|
| 12 | export function percentileNearest(values, percentile) {
|
| 13 | const ordered = [...values].map(Number).sort((a, b) => a - b);
|
| 14 | if (ordered.length === 0) {
|
| 15 | return null;
|
| 16 | }
|
| 17 | const rank = Math.ceil((percentile / 100.0) * ordered.length);
|
| 18 | const index = Math.min(Math.max(rank, 1), ordered.length) - 1;
|
| 19 | return ordered[index];
|
| 20 | }
|
| 21 |
|
| 22 | /** Port of oakbench.reporting.median (statistics.median, empty -> null). */
|
| 23 | export function median(values) {
|
| 24 | const ordered = [...values].map(Number).sort((a, b) => a - b);
|
| 25 | const n = ordered.length;
|
| 26 | if (n === 0) {
|
| 27 | return null;
|
| 28 | }
|
| 29 | if (n % 2 === 1) {
|
| 30 | return ordered[(n - 1) / 2];
|
| 31 | }
|
| 32 | return (ordered[n / 2 - 1] + ordered[n / 2]) / 2;
|
| 33 | }
|
| 34 |
|
| 35 | export function mean(values) {
|
| 36 | if (values.length === 0) {
|
| 37 | return null;
|
| 38 | }
|
| 39 | let total = 0;
|
| 40 | for (const value of values) {
|
| 41 | total += Number(value);
|
| 42 | }
|
| 43 | return total / values.length;
|
| 44 | }
|
| 45 |
|
| 46 | // --- reporting.py ports ---------------------------------------------------
|
| 47 |
|
| 48 | /**
|
| 49 | * Port of oakbench.reporting.delta_pct.
|
| 50 | * POSITIVE means value is LOWER/BETTER than base. base == 0 -> 0.0 (guard).
|
| 51 | */
|
| 52 | export function deltaPct(base, value) {
|
| 53 | if (base === null || base === undefined || value === null || value === undefined) {
|
| 54 | return null;
|
| 55 | }
|
| 56 | if (base === 0) {
|
| 57 | return 0.0;
|
| 58 | }
|
| 59 | return ((base - value) / base) * 100.0;
|
| 60 | }
|
| 61 |
|
| 62 | /**
|
| 63 | * Port of oakbench.reporting.slower_pct.
|
| 64 | * POSITIVE means value is HIGHER/WORSE than baseline; non-positive or
|
| 65 | * unmeasured baseline -> null.
|
| 66 | */
|
| 67 | export function slowerPct(baseline, value) {
|
| 68 | if (baseline === null || baseline === undefined || value === null || value === undefined) {
|
| 69 | return null;
|
| 70 | }
|
| 71 | if (baseline <= 0) {
|
| 72 | return null;
|
| 73 | }
|
| 74 | return ((value - baseline) / baseline) * 100.0;
|
| 75 | }
|
| 76 |
|
| 77 | /** Port of oakbench.reporting.delta_ms (null when either side unmeasured). */
|
| 78 | export function deltaMs(baseline, value) {
|
| 79 | if (baseline === null || baseline === undefined || value === null || value === undefined) {
|
| 80 | return null;
|
| 81 | }
|
| 82 | return value - baseline;
|
| 83 | }
|
| 84 |
|
| 85 | // --- rows.py accessor ports -----------------------------------------------
|
| 86 |
|
| 87 | export function numberOrNone(value) {
|
| 88 | if (value === null || value === undefined) {
|
| 89 | return null;
|
| 90 | }
|
| 91 | const parsed = Number(value);
|
| 92 | return Number.isFinite(parsed) ? parsed : null;
|
| 93 | }
|
| 94 |
|
| 95 | /** Port of oakbench.rows.row_returncode (missing/null -> failure, never crash). */
|
| 96 | export function rowReturncode(row) {
|
| 97 | const value = row.returncode;
|
| 98 | if (value === null || value === undefined) {
|
| 99 | return 1;
|
| 100 | }
|
| 101 | const parsed = Number(value);
|
| 102 | return Number.isInteger(parsed) ? parsed : 1;
|
| 103 | }
|
| 104 |
|
| 105 | const SKIP_RETURNCODE = 77;
|
| 106 | const SUBOP_SUFFIXES = [".add", ".commit"];
|
| 107 |
|
| 108 | // Explicit separator for every composite map key. Plain concatenation is
|
| 109 | // collision-prone ("a" + "bc" === "ab" + "c"); ASCII unit separator cannot
|
| 110 | // appear in subject/scenario/operation strings produced by the harness.
|
| 111 | const KEY_SEP = "\u001f";
|
| 112 |
|
| 113 | /**
|
| 114 | * Deterministic ingest identity for a row.
|
| 115 | *
|
| 116 | * Platform-lane rows (scripts/oakbench/rows.py LANE_REQUIRED_KEYS["platform"])
|
| 117 | * legitimately carry NO subject field — their identity is platform/driver
|
| 118 | * (e.g. "github/cli"). Rows with neither subject nor platform have no subject
|
| 119 | * identity and return null; callers must skip them from subject-keyed tables
|
| 120 | * explicitly and count them in an ingest report (never silently).
|
| 121 | */
|
| 122 | export function effectiveSubject(row) {
|
| 123 | if (row.subject !== null && row.subject !== undefined) {
|
| 124 | return String(row.subject);
|
| 125 | }
|
| 126 | if (row.platform !== null && row.platform !== undefined) {
|
| 127 | return `${row.platform}/${row.driver}`;
|
| 128 | }
|
| 129 | return null;
|
| 130 | }
|
| 131 |
|
| 132 | /**
|
| 133 | * Subject kind matching effectiveSubject: explicit subject_kind when present;
|
| 134 | * for subject-less platform rows the kind is the platform name. Null when the
|
| 135 | * row carries neither (null means unmeasured, ADR-0002).
|
| 136 | */
|
| 137 | export function effectiveSubjectKind(row) {
|
| 138 | if (row.subject_kind !== null && row.subject_kind !== undefined) {
|
| 139 | return String(row.subject_kind);
|
| 140 | }
|
| 141 | if (
|
| 142 | (row.subject === null || row.subject === undefined) &&
|
| 143 | row.platform !== null &&
|
| 144 | row.platform !== undefined
|
| 145 | ) {
|
| 146 | return String(row.platform);
|
| 147 | }
|
| 148 | return null;
|
| 149 | }
|
| 150 |
|
| 151 | function isObject(value) {
|
| 152 | return value !== null && typeof value === "object" && !Array.isArray(value);
|
| 153 | }
|
| 154 |
|
| 155 | /** Port of oakbench.rows.token_total (best-available, provider first). */
|
| 156 | export function tokenTotal(row) {
|
| 157 | const tokenMetrics = isObject(row.token_metrics) ? row.token_metrics : null;
|
| 158 | if (tokenMetrics) {
|
| 159 | const total = numberOrNone(tokenMetrics.total_tokens_reported);
|
| 160 | if (total !== null) {
|
| 161 | return total;
|
| 162 | }
|
| 163 | const parts = [
|
| 164 | numberOrNone(tokenMetrics.input_tokens_reported),
|
| 165 | numberOrNone(tokenMetrics.output_tokens_reported),
|
| 166 | numberOrNone(tokenMetrics.reasoning_tokens_reported),
|
| 167 | ];
|
| 168 | if (parts.some((part) => part !== null)) {
|
| 169 | return parts.reduce((sum, part) => sum + (part || 0), 0);
|
| 170 | }
|
| 171 | }
|
| 172 |
|
| 173 | for (const key of ["estimated_tokens_total", "token_count", "total_tokens"]) {
|
| 174 | const value = numberOrNone(row[key]);
|
| 175 | if (value !== null) {
|
| 176 | return value;
|
| 177 | }
|
| 178 | }
|
| 179 |
|
| 180 | if (isObject(row.token_estimate)) {
|
| 181 | const value = numberOrNone(row.token_estimate.total);
|
| 182 | if (value !== null) {
|
| 183 | return value;
|
| 184 | }
|
| 185 | }
|
| 186 |
|
| 187 | if (isObject(row.tokens)) {
|
| 188 | for (const key of ["total", "total_tokens", "provider_total", "estimated_total"]) {
|
| 189 | const value = numberOrNone(row.tokens[key]);
|
| 190 | if (value !== null) {
|
| 191 | return value;
|
| 192 | }
|
| 193 | }
|
| 194 | if (isObject(row.tokens.provider_reported)) {
|
| 195 | const value = numberOrNone(row.tokens.provider_reported.total);
|
| 196 | if (value !== null) {
|
| 197 | return value;
|
| 198 | }
|
| 199 | }
|
| 200 | }
|
| 201 |
|
| 202 | if (tokenMetrics) {
|
| 203 | let chars = numberOrNone(tokenMetrics.transcript_chars);
|
| 204 | if (chars === null) {
|
| 205 | const inputChars = numberOrNone(tokenMetrics.normalized_input_chars);
|
| 206 | const outputChars = numberOrNone(tokenMetrics.normalized_output_chars);
|
| 207 | if (inputChars !== null || outputChars !== null) {
|
| 208 | chars = (inputChars || 0) + (outputChars || 0);
|
| 209 | }
|
| 210 | }
|
| 211 | if (chars !== null) {
|
| 212 | return Math.max(1, Math.ceil(chars / 4.0));
|
| 213 | }
|
| 214 | }
|
| 215 | return null;
|
| 216 | }
|
| 217 |
|
| 218 | /** Port of oakbench.rows.tool_calls. */
|
| 219 | export function toolCalls(row) {
|
| 220 | for (const key of ["tool_call_count", "total_tool_calls"]) {
|
| 221 | const value = numberOrNone(row[key]);
|
| 222 | if (value !== null) {
|
| 223 | return value;
|
| 224 | }
|
| 225 | }
|
| 226 | if (isObject(row.tool_calls)) {
|
| 227 | for (const key of ["total", "all"]) {
|
| 228 | const value = numberOrNone(row.tool_calls[key]);
|
| 229 | if (value !== null) {
|
| 230 | return value;
|
| 231 | }
|
| 232 | }
|
| 233 | }
|
| 234 | if (isObject(row.tool_call_metrics)) {
|
| 235 | const value = numberOrNone(row.tool_call_metrics.tool_calls_total);
|
| 236 | if (value !== null) {
|
| 237 | return value;
|
| 238 | }
|
| 239 | }
|
| 240 | return null;
|
| 241 | }
|
| 242 |
|
| 243 | /** Port of oakbench.rows.output_bytes. */
|
| 244 | export function outputBytes(row) {
|
| 245 | const raw = numberOrNone(row.raw_output_bytes);
|
| 246 | if (raw !== null) {
|
| 247 | return raw;
|
| 248 | }
|
| 249 | let stdout = numberOrNone(row.stdout_bytes);
|
| 250 | let stderr = numberOrNone(row.stderr_bytes);
|
| 251 | if (stdout !== null || stderr !== null) {
|
| 252 | return (stdout || 0) + (stderr || 0);
|
| 253 | }
|
| 254 | if (isObject(row.token_estimate)) {
|
| 255 | stdout = numberOrNone(row.token_estimate.stdout_bytes);
|
| 256 | stderr = numberOrNone(row.token_estimate.stderr_bytes);
|
| 257 | if (stdout !== null || stderr !== null) {
|
| 258 | return (stdout || 0) + (stderr || 0);
|
| 259 | }
|
| 260 | }
|
| 261 | return null;
|
| 262 | }
|
| 263 |
|
| 264 | /** Port of oakbench.rows.output_truncated. */
|
| 265 | export function outputTruncated(row) {
|
| 266 | if (row.output_truncated || row.stdout_truncated || row.stderr_truncated) {
|
| 267 | return true;
|
| 268 | }
|
| 269 | if (isObject(row.token_estimate)) {
|
| 270 | return Boolean(row.token_estimate.stdout_truncated || row.token_estimate.stderr_truncated);
|
| 271 | }
|
| 272 | return false;
|
| 273 | }
|
| 274 |
|
| 275 | // --- Ingest step 1: rows -> measurements -----------------------------------
|
| 276 |
|
| 277 | /**
|
| 278 | * Project JSONL rows into measurements-table records.
|
| 279 | *
|
| 280 | * Pure field projection: null/missing stays null (NEVER coerced to 0).
|
| 281 | * Sub-operation rows (operation ending .add/.commit) are marked
|
| 282 | * is_canonical = 0, matching oakbench.baseline_book.SUBOP_SUFFIXES.
|
| 283 | *
|
| 284 | * Subject identity uses effectiveSubject/effectiveSubjectKind so platform
|
| 285 | * rows (no subject field; identity platform/driver) group correctly. Rows
|
| 286 | * with no subject identity at all (no subject AND no platform) are skipped
|
| 287 | * from this subject-keyed projection explicitly: when a `report` object is
|
| 288 | * passed, its skipped_no_subject counter is incremented per skipped row so
|
| 289 | * the ingest response surfaces them (never silent).
|
| 290 | *
|
| 291 | * `subjectIds` optionally maps subjectKey(row) -> subject_id; when absent the
|
| 292 | * effective subject name is used so the transform stays pure and testable.
|
| 293 | */
|
| 294 | export function rowsToMeasurements(rows, runKey, now, subjectIds = null, report = null) {
|
| 295 | const measurements = [];
|
| 296 | for (const row of rows) {
|
| 297 | const subject = effectiveSubject(row);
|
| 298 | if (subject === null) {
|
| 299 | if (report) {
|
| 300 | report.skipped_no_subject = (report.skipped_no_subject ?? 0) + 1;
|
| 301 | }
|
| 302 | continue;
|
| 303 | }
|
| 304 | const operation = String(row.operation ?? "");
|
| 305 | const key = subjectKey(row);
|
| 306 | const kind = effectiveSubjectKind(row);
|
| 307 | measurements.push({
|
| 308 | run_key: runKey,
|
| 309 | subject_id: (subjectIds && subjectIds.get(key)) || subject,
|
| 310 | subject,
|
| 311 | subject_kind: kind,
|
| 312 | scenario: String(row.scenario ?? ""),
|
| 313 | operation,
|
| 314 | run_index: numberOrNone(row.run) ?? 0,
|
| 315 | is_canonical: SUBOP_SUFFIXES.some((suffix) => operation.endsWith(suffix)) ? 0 : 1,
|
| 316 | elapsed_ms: numberOrNone(row.elapsed_ms),
|
| 317 | estimated_tokens_total: numberOrNone(row.estimated_tokens_total),
|
| 318 | estimated_tokens_input: numberOrNone(row.estimated_tokens_input),
|
| 319 | estimated_tokens_output: numberOrNone(row.estimated_tokens_output),
|
| 320 | token_source: row.token_source === null || row.token_source === undefined ? null : String(row.token_source),
|
| 321 | raw_output_bytes: numberOrNone(row.raw_output_bytes),
|
| 322 | stdout_bytes: numberOrNone(row.stdout_bytes),
|
| 323 | stderr_bytes: numberOrNone(row.stderr_bytes),
|
| 324 | output_truncated: outputTruncated(row) ? 1 : 0,
|
| 325 | tool_call_count: numberOrNone(row.tool_call_count),
|
| 326 | vcs_tool_call_count: numberOrNone(row.vcs_tool_call_count),
|
| 327 | terminal_tool_call_count: numberOrNone(row.terminal_tool_call_count),
|
| 328 | returncode: rowReturncode(row),
|
| 329 | command_json: row.command === undefined || row.command === null ? null : JSON.stringify(row.command),
|
| 330 | stderr: typeof row.stderr === "string" && row.stderr ? row.stderr.slice(-4000) : null,
|
| 331 | row_json: JSON.stringify(row),
|
| 332 | created_at: now,
|
| 333 | // schema_v2 additive columns (null when the lane did not record them).
|
| 334 | runner_id: row.runner_id === null || row.runner_id === undefined ? null : String(row.runner_id),
|
| 335 | runner_class: row.runner_class === null || row.runner_class === undefined ? null : String(row.runner_class),
|
| 336 | load_tier: row.load_tier === null || row.load_tier === undefined ? null : String(row.load_tier),
|
| 337 | netshape_profile:
|
| 338 | row.netshape_profile === null || row.netshape_profile === undefined ? null : String(row.netshape_profile),
|
| 339 | // Best-available derived metrics (oakbench.rows accessors); used by
|
| 340 | // computeAggregates, not persisted as columns.
|
| 341 | tokens_best: tokenTotal(row),
|
| 342 | output_bytes_best: outputBytes(row),
|
| 343 | tool_calls_best: toolCalls(row),
|
| 344 | });
|
| 345 | }
|
| 346 | return measurements;
|
| 347 | }
|
| 348 |
|
| 349 | // --- Ingest step 2: subjects upsert ----------------------------------------
|
| 350 |
|
| 351 | /**
|
| 352 | * Identity key for a subject within a run: (effective subject, effective
|
| 353 | * subject_kind, binary_sha256). Platform rows (no subject field) key on
|
| 354 | * their platform/driver identity. Null when the row has no subject identity
|
| 355 | * at all (no subject AND no platform) so callers skip it explicitly.
|
| 356 | */
|
| 357 | export function subjectKey(row) {
|
| 358 | const subject = effectiveSubject(row);
|
| 359 | if (subject === null) {
|
| 360 | return null;
|
| 361 | }
|
| 362 | const kind = effectiveSubjectKind(row);
|
| 363 | return [
|
| 364 | subject,
|
| 365 | kind === null ? "" : kind,
|
| 366 | row.binary_sha256 === null || row.binary_sha256 === undefined ? "" : String(row.binary_sha256),
|
| 367 | ].join(KEY_SEP);
|
| 368 | }
|
| 369 |
|
| 370 | /**
|
| 371 | * Build run-scoped subject upsert records, keyed (effective subject,
|
| 372 | * effective subject_kind, binary_sha256), enriched from the first row's
|
| 373 | * subject_details map. Platform rows (no subject field) are identified by
|
| 374 | * platform/driver via effectiveSubject so they ingest like any other subject.
|
| 375 | * Returns { records, idsByKey, skipped }: idsByKey maps subjectKey ->
|
| 376 | * subject_id; skipped counts rows with no subject identity at all (no
|
| 377 | * subject AND no platform), which are excluded from subject-keyed tables.
|
| 378 | */
|
| 379 | export async function upsertSubjects(rows, runKey, subjectDetails = {}, subjectVersions = {}) {
|
| 380 | const records = [];
|
| 381 | const idsByKey = new Map();
|
| 382 | let skipped = 0;
|
| 383 | for (const row of rows) {
|
| 384 | const name = effectiveSubject(row);
|
| 385 | if (name === null) {
|
| 386 | skipped += 1;
|
| 387 | continue;
|
| 388 | }
|
| 389 | const key = subjectKey(row);
|
| 390 | if (idsByKey.has(key)) {
|
| 391 | continue;
|
| 392 | }
|
| 393 | const details = isObject(subjectDetails[name]) ? subjectDetails[name] : {};
|
| 394 | const binarySha = row.binary_sha256 ?? details.binary_sha256 ?? null;
|
| 395 | const subjectId = await stableId(`${runKey}${KEY_SEP}${key}`);
|
| 396 | idsByKey.set(key, subjectId);
|
| 397 | records.push({
|
| 398 | subject_id: subjectId,
|
| 399 | run_key: runKey,
|
| 400 | name,
|
| 401 | kind: String(effectiveSubjectKind(row) ?? details.kind ?? "unknown"),
|
| 402 | label: details.label === undefined ? null : details.label,
|
| 403 | version: subjectVersions[name] ?? details.version ?? null,
|
| 404 | binary_path: details.bin === undefined ? null : details.bin,
|
| 405 | metadata_json: JSON.stringify({ binary_sha256: binarySha, role: details.role ?? null }),
|
| 406 | });
|
| 407 | }
|
| 408 | return { records, idsByKey, skipped };
|
| 409 | }
|
| 410 |
|
| 411 | // --- Ingest step 3: aggregates ---------------------------------------------
|
| 412 |
|
| 413 | /**
|
| 414 | * Per (run_key, subject_id, scenario, operation) aggregates.
|
| 415 | *
|
| 416 | * Latency percentiles use nearest-rank (percentileNearest) over the
|
| 417 | * elapsed_ms of SUCCESSFUL (returncode 0) measurements only, exactly matching
|
| 418 | * oakbench.stats.percentile_nearest semantics including empty -> null.
|
| 419 | * Skip rows (returncode 77) are excluded from sample/failure accounting.
|
| 420 | */
|
| 421 | // Tail honesty thresholds, mirroring scripts/benchmark_stats.py
|
| 422 | // TAIL_MIN_SAMPLES = {"p95": 20, "p99": 100} (itself a mirror of
|
| 423 | // oakbench.reporting.TAIL_PERCENTILE_MIN_SAMPLES): below 1/(1-p) samples a
|
| 424 | // nearest-rank tail percentile is just the max relabelled, so claiming it
|
| 425 | // would overstate precision. Gated percentiles are null (unmeasured,
|
| 426 | // ADR-0002), NEVER the lone sample. p50/p90 stay ungated to match the Python
|
| 427 | // reporting behavior (benchmark_stats gates only p95/p99).
|
| 428 | export const TAIL_MIN_SAMPLES = { p95: 20, p99: 100 };
|
| 429 |
|
| 430 | export function computeAggregates(measurements, profile, now) {
|
| 431 | const groups = new Map();
|
| 432 | for (const m of measurements) {
|
| 433 | const key = [m.run_key, m.subject_id, m.scenario, m.operation].join(KEY_SEP);
|
| 434 | if (!groups.has(key)) {
|
| 435 | groups.set(key, []);
|
| 436 | }
|
| 437 | groups.get(key).push(m);
|
| 438 | }
|
| 439 |
|
| 440 | const aggregates = [];
|
| 441 | for (const group of groups.values()) {
|
| 442 | const first = group[0];
|
| 443 | const successes = group.filter((m) => m.returncode === 0);
|
| 444 | const failures = group.filter((m) => m.returncode !== 0 && m.returncode !== SKIP_RETURNCODE);
|
| 445 | const sampleCount = successes.length + failures.length;
|
| 446 | const elapsed = successes.map((m) => m.elapsed_ms).filter((v) => v !== null);
|
| 447 | const tokens = successes.map((m) => m.tokens_best).filter((v) => v !== null && v !== undefined);
|
| 448 | const bytes = successes.map((m) => m.output_bytes_best).filter((v) => v !== null && v !== undefined);
|
| 449 | const calls = successes.map((m) => m.tool_calls_best).filter((v) => v !== null && v !== undefined);
|
| 450 | aggregates.push({
|
| 451 | run_key: first.run_key,
|
| 452 | subject_id: first.subject_id,
|
| 453 | subject: first.subject,
|
| 454 | subject_kind: first.subject_kind,
|
| 455 | profile,
|
| 456 | scenario: first.scenario,
|
| 457 | operation: first.operation,
|
| 458 | is_canonical: first.is_canonical,
|
| 459 | sample_count: sampleCount,
|
| 460 | failure_count: failures.length,
|
| 461 | failure_rate: sampleCount > 0 ? failures.length / sampleCount : null,
|
| 462 | n: elapsed.length,
|
| 463 | p50_ms: median(elapsed),
|
| 464 | p90_ms: percentileNearest(elapsed, 90),
|
| 465 | p95_ms: elapsed.length >= TAIL_MIN_SAMPLES.p95 ? percentileNearest(elapsed, 95) : null,
|
| 466 | p99_ms: elapsed.length >= TAIL_MIN_SAMPLES.p99 ? percentileNearest(elapsed, 99) : null,
|
| 467 | mean_ms: mean(elapsed),
|
| 468 | min_ms: elapsed.length ? Math.min(...elapsed) : null,
|
| 469 | max_ms: elapsed.length ? Math.max(...elapsed) : null,
|
| 470 | avg_tokens: mean(tokens),
|
| 471 | avg_output_bytes: mean(bytes),
|
| 472 | truncated_count: group.filter((m) => m.output_truncated === 1).length,
|
| 473 | avg_tool_calls: mean(calls),
|
| 474 | updated_at: now,
|
| 475 | });
|
| 476 | }
|
| 477 | return aggregates;
|
| 478 | }
|
| 479 |
|
| 480 | // --- Ingest step 4: comparisons --------------------------------------------
|
| 481 |
|
| 482 | /**
|
| 483 | * Target-vs-baseline comparison rows: every non-git subject aggregate is
|
| 484 | * compared against EVERY git-kind subject aggregate for the same
|
| 485 | * run/scenario/operation (e.g. stock git AND git_fsmonitor each yield a
|
| 486 | * comparison record, distinguished by baseline_subject_id). Ingest holds no
|
| 487 | * notion of "the" baseline: selecting which git variant to report against is
|
| 488 | * a reporting/targets concern (hardest-fair-comparator doctrine). Output is
|
| 489 | * sorted on (run_key, scenario, operation, target_subject_id,
|
| 490 | * baseline_subject_id) so results are deterministic regardless of input
|
| 491 | * row order.
|
| 492 | *
|
| 493 | * p50_ratio is target/baseline (null when baseline is unmeasured or 0 — a
|
| 494 | * zero baseline makes the ratio undefined, mirroring scorecard UNDECIDABLE).
|
| 495 | * p50_delta_pct_better uses deltaPct (positive = better) including its
|
| 496 | * base == 0 -> 0.0 guard; *_slower_pct uses slowerPct (positive = worse).
|
| 497 | *
|
| 498 | * is_regression is 1 when EITHER p50 or p90 breaches (slower_pct >
|
| 499 | * thresholdPct AND delta_ms > minDeltaMs, evaluated per dimension);
|
| 500 | * regression_dimensions lists the breaching dimensions ("p50"/"p90"),
|
| 501 | * empty when none.
|
| 502 | */
|
| 503 | export function computeComparisons(aggregates, options = {}) {
|
| 504 | const thresholdPct = options.thresholdPct ?? 10.0;
|
| 505 | const minDeltaMs = options.minDeltaMs ?? 5.0;
|
| 506 |
|
| 507 | const baselinesByGroup = new Map();
|
| 508 | for (const agg of aggregates) {
|
| 509 | if (agg.subject_kind === "git") {
|
| 510 | const key = [agg.run_key, agg.scenario, agg.operation].join(KEY_SEP);
|
| 511 | if (!baselinesByGroup.has(key)) {
|
| 512 | baselinesByGroup.set(key, []);
|
| 513 | }
|
| 514 | baselinesByGroup.get(key).push(agg);
|
| 515 | }
|
| 516 | }
|
| 517 |
|
| 518 | const comparisons = [];
|
| 519 | for (const target of aggregates) {
|
| 520 | if (target.subject_kind === "git") {
|
| 521 | continue;
|
| 522 | }
|
| 523 | const bases = baselinesByGroup.get([target.run_key, target.scenario, target.operation].join(KEY_SEP)) || [];
|
| 524 | for (const base of bases) {
|
| 525 | if (base.subject_id === target.subject_id) {
|
| 526 | continue;
|
| 527 | }
|
| 528 | const p50SlowerPct = slowerPct(base.p50_ms, target.p50_ms);
|
| 529 | const p50DeltaMs = deltaMs(base.p50_ms, target.p50_ms);
|
| 530 | const p90SlowerPct = slowerPct(base.p90_ms, target.p90_ms);
|
| 531 | const p90DeltaMs = deltaMs(base.p90_ms, target.p90_ms);
|
| 532 | // A regression in EITHER p50 or p90 flags the comparison: agents pay
|
| 533 | // tail latency, so a p90-only blowup must not hide behind a flat p50.
|
| 534 | // Each dimension breaches when slower_pct > thresholdPct AND
|
| 535 | // delta_ms > minDeltaMs (nulls — unmeasured — never breach).
|
| 536 | const breaches = (slower, delta) =>
|
| 537 | slower !== null && delta !== null && slower > thresholdPct && delta > minDeltaMs;
|
| 538 | const regressionDimensions = [];
|
| 539 | if (breaches(p50SlowerPct, p50DeltaMs)) {
|
| 540 | regressionDimensions.push("p50");
|
| 541 | }
|
| 542 | if (breaches(p90SlowerPct, p90DeltaMs)) {
|
| 543 | regressionDimensions.push("p90");
|
| 544 | }
|
| 545 | const isRegression = regressionDimensions.length > 0 ? 1 : 0;
|
| 546 | comparisons.push({
|
| 547 | run_key: target.run_key,
|
| 548 | target_subject_id: target.subject_id,
|
| 549 | baseline_subject_id: base.subject_id,
|
| 550 | baseline_subject: base.subject,
|
| 551 | baseline_kind: "git",
|
| 552 | profile: target.profile,
|
| 553 | scenario: target.scenario,
|
| 554 | operation: target.operation,
|
| 555 | p50_delta_ms: p50DeltaMs,
|
| 556 | p50_slower_pct: p50SlowerPct,
|
| 557 | p50_ratio:
|
| 558 | base.p50_ms === null || base.p50_ms === 0 || target.p50_ms === null ? null : target.p50_ms / base.p50_ms,
|
| 559 | p50_delta_pct_better: deltaPct(base.p50_ms, target.p50_ms),
|
| 560 | p90_delta_ms: p90DeltaMs,
|
| 561 | p90_slower_pct: p90SlowerPct,
|
| 562 | p99_delta_ms: deltaMs(base.p99_ms, target.p99_ms),
|
| 563 | p99_slower_pct: slowerPct(base.p99_ms, target.p99_ms),
|
| 564 | avg_token_delta: deltaMs(base.avg_tokens, target.avg_tokens),
|
| 565 | token_higher_pct: slowerPct(base.avg_tokens, target.avg_tokens),
|
| 566 | avg_output_byte_delta: deltaMs(base.avg_output_bytes, target.avg_output_bytes),
|
| 567 | output_byte_higher_pct: slowerPct(base.avg_output_bytes, target.avg_output_bytes),
|
| 568 | avg_tool_call_delta: deltaMs(base.avg_tool_calls, target.avg_tool_calls),
|
| 569 | tool_call_higher_pct: slowerPct(base.avg_tool_calls, target.avg_tool_calls),
|
| 570 | failure_rate_delta_pp:
|
| 571 | target.failure_rate === null || target.failure_rate === undefined ||
|
| 572 | base.failure_rate === null || base.failure_rate === undefined
|
| 573 | ? null
|
| 574 | : (target.failure_rate - base.failure_rate) * 100.0,
|
| 575 | threshold_pct: thresholdPct,
|
| 576 | min_delta_ms: minDeltaMs,
|
| 577 | is_regression: isRegression,
|
| 578 | // Additive: which dimensions breached ("p50"/"p90"); empty when none.
|
| 579 | regression_dimensions: regressionDimensions,
|
| 580 | state: isRegression ? "regression" : "ok",
|
| 581 | severity_score: isRegression
|
| 582 | ? Math.max(
|
| 583 | regressionDimensions.includes("p50") ? p50SlowerPct : 0,
|
| 584 | regressionDimensions.includes("p90") ? p90SlowerPct : 0,
|
| 585 | )
|
| 586 | : 0,
|
| 587 | });
|
| 588 | }
|
| 589 | }
|
| 590 | comparisons.sort((a, b) => {
|
| 591 | const left = [a.run_key, a.scenario, a.operation, a.target_subject_id, a.baseline_subject_id].join(KEY_SEP);
|
| 592 | const right = [b.run_key, b.scenario, b.operation, b.target_subject_id, b.baseline_subject_id].join(KEY_SEP);
|
| 593 | return left < right ? -1 : left > right ? 1 : 0;
|
| 594 | });
|
| 595 | return comparisons;
|
| 596 | }
|
| 597 |
|
| 598 | // --- Scorecard + trends -----------------------------------------------------
|
| 599 |
|
| 600 | const VERDICT_PRIORITY = ["RED", "INVALID", "UNDECIDABLE", "AMBER", "UNMEASURED", "GREEN"];
|
| 601 |
|
| 602 | /**
|
| 603 | * Summarize scorecard cells (the verdict dicts produced by
|
| 604 | * oakbench.scorecard.evaluate_targets) into dashboard headline counts.
|
| 605 | */
|
| 606 | export function scorecardFromCells(cells) {
|
| 607 | const counts = {};
|
| 608 | for (const cell of cells || []) {
|
| 609 | const verdict = String(cell.verdict ?? "UNMEASURED");
|
| 610 | counts[verdict] = (counts[verdict] || 0) + 1;
|
| 611 | }
|
| 612 | let overall = "UNMEASURED";
|
| 613 | for (const verdict of VERDICT_PRIORITY) {
|
| 614 | if (counts[verdict]) {
|
| 615 | overall = verdict;
|
| 616 | break;
|
| 617 | }
|
| 618 | }
|
| 619 | return { total: (cells || []).length, counts, overall };
|
| 620 | }
|
| 621 |
|
| 622 | const TREND_METRICS = {
|
| 623 | p50: "p50_ms",
|
| 624 | median: "p50_ms",
|
| 625 | p50_ms: "p50_ms",
|
| 626 | p90: "p90_ms",
|
| 627 | p90_ms: "p90_ms",
|
| 628 | p95: "p95_ms",
|
| 629 | p95_ms: "p95_ms",
|
| 630 | p99: "p99_ms",
|
| 631 | p99_ms: "p99_ms",
|
| 632 | mean: "mean_ms",
|
| 633 | mean_ms: "mean_ms",
|
| 634 | avg_tokens: "avg_tokens",
|
| 635 | avg_output_bytes: "avg_output_bytes",
|
| 636 | avg_tool_calls: "avg_tool_calls",
|
| 637 | failure_rate: "failure_rate",
|
| 638 | };
|
| 639 |
|
| 640 | /**
|
| 641 | * Build a per-subject time series from aggregate rows joined with run
|
| 642 | * timestamps: rows need {timestamp_utc, run_key, subject, scenario,
|
| 643 | * operation, <metric columns>}. Unknown metric -> null (caller 400s).
|
| 644 | */
|
| 645 | export function trendSeries(rows, { scenario, operation, metric = "p50" } = {}) {
|
| 646 | const column = TREND_METRICS[metric];
|
| 647 | if (!column) {
|
| 648 | return null;
|
| 649 | }
|
| 650 | const yes = (row) =>
|
| 651 | (scenario === undefined || scenario === null || row.scenario === scenario) &&
|
| 652 | (operation === undefined || operation === null || row.operation === operation);
|
| 653 | const bySubject = new Map();
|
| 654 | const sorted = rows.filter(yes).sort((a, b) => String(a.timestamp_utc).localeCompare(String(b.timestamp_utc)));
|
| 655 | for (const row of sorted) {
|
| 656 | const subject = String(row.subject ?? row.name ?? "");
|
| 657 | if (!bySubject.has(subject)) {
|
| 658 | bySubject.set(subject, []);
|
| 659 | }
|
| 660 | bySubject.get(subject).push({
|
| 661 | t: row.timestamp_utc,
|
| 662 | run_key: row.run_key,
|
| 663 | value: row[column] === undefined ? null : row[column],
|
| 664 | });
|
| 665 | }
|
| 666 | return {
|
| 667 | scenario: scenario ?? null,
|
| 668 | operation: operation ?? null,
|
| 669 | metric: column,
|
| 670 | series: [...bySubject.entries()].map(([subject, points]) => ({ subject, points })),
|
| 671 | };
|
| 672 | }
|
| 673 |
|
| 674 | // --- Webhook signature ------------------------------------------------------
|
| 675 |
|
| 676 | /**
|
| 677 | * Verify a GitHub X-Hub-Signature-256 header ("sha256=<hex>") against the raw
|
| 678 | * request body using HMAC-SHA256 (crypto.subtle; works in Workers and node).
|
| 679 | */
|
| 680 | export async function verifySignature(secret, body, signatureHeader) {
|
| 681 | if (!secret || typeof signatureHeader !== "string" || !signatureHeader.startsWith("sha256=")) {
|
| 682 | return false;
|
| 683 | }
|
| 684 | const encoder = new TextEncoder();
|
| 685 | const key = await crypto.subtle.importKey(
|
| 686 | "raw",
|
| 687 | encoder.encode(secret),
|
| 688 | { name: "HMAC", hash: "SHA-256" },
|
| 689 | false,
|
| 690 | ["sign"],
|
| 691 | );
|
| 692 | const digest = await crypto.subtle.sign("HMAC", key, encoder.encode(body));
|
| 693 | const expected = [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
| 694 | const provided = signatureHeader.slice("sha256=".length).toLowerCase();
|
| 695 | if (provided.length !== expected.length) {
|
| 696 | return false;
|
| 697 | }
|
| 698 | let diff = 0;
|
| 699 | for (let index = 0; index < expected.length; index += 1) {
|
| 700 | diff |= expected.charCodeAt(index) ^ provided.charCodeAt(index);
|
| 701 | }
|
| 702 | return diff === 0;
|
| 703 | }
|
| 704 |
|
| 705 | // --- shared hashing ---------------------------------------------------------
|
| 706 |
|
| 707 | export async function sha256Hex(text) {
|
| 708 | const bytes = new TextEncoder().encode(text);
|
| 709 | const digest = await crypto.subtle.digest("SHA-256", bytes);
|
| 710 | return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
| 711 | }
|
| 712 |
|
| 713 | export async function stableId(text) {
|
| 714 | return (await sha256Hex(text)).slice(0, 24);
|
| 715 | }
|