Viewing files at commit 653060c4e6b1Commit detailsView latest

Browse files
cloudflare/reporting.mjs 715 lines · 26.3 KB · javascript
1
// Shared reporting math + ingest transforms for the dashboard worker.
2
//
3
// Every formula here is a faithful port of the Python definitions in
4
// scripts/oakbench/stats.py and scripts/oakbench/reporting.py. Parity is
5
// enforced by tests/fixtures/reporting_golden.json, asserted from BOTH sides
6
// (tests/test_reporting_golden.py and cloudflare/reporting_parity.test.mjs).
7
// Null means unmeasured (ADR-0002): a null input NEVER becomes 0.
8
9
// --- stats.py ports -------------------------------------------------------
10
11
/** Port of oakbench.stats.percentile_nearest (nearest-rank, empty -> null). */
12
export function percentileNearest(values, percentile) {
13
  const ordered = [...values].map(Number).sort((a, b) => a - b);
14
  if (ordered.length === 0) {
15
    return null;
16
  }
17
  const rank = Math.ceil((percentile / 100.0) * ordered.length);
18
  const index = Math.min(Math.max(rank, 1), ordered.length) - 1;
19
  return ordered[index];
20
}
21
22
/** Port of oakbench.reporting.median (statistics.median, empty -> null). */
23
export function median(values) {
24
  const ordered = [...values].map(Number).sort((a, b) => a - b);
25
  const n = ordered.length;
26
  if (n === 0) {
27
    return null;
28
  }
29
  if (n % 2 === 1) {
30
    return ordered[(n - 1) / 2];
31
  }
32
  return (ordered[n / 2 - 1] + ordered[n / 2]) / 2;
33
}
34
35
export function mean(values) {
36
  if (values.length === 0) {
37
    return null;
38
  }
39
  let total = 0;
40
  for (const value of values) {
41
    total += Number(value);
42
  }
43
  return total / values.length;
44
}
45
46
// --- reporting.py ports ---------------------------------------------------
47
48
/**
49
 * Port of oakbench.reporting.delta_pct.
50
 * POSITIVE means value is LOWER/BETTER than base. base == 0 -> 0.0 (guard).
51
 */
52
export function deltaPct(base, value) {
53
  if (base === null || base === undefined || value === null || value === undefined) {
54
    return null;
55
  }
56
  if (base === 0) {
57
    return 0.0;
58
  }
59
  return ((base - value) / base) * 100.0;
60
}
61
62
/**
63
 * Port of oakbench.reporting.slower_pct.
64
 * POSITIVE means value is HIGHER/WORSE than baseline; non-positive or
65
 * unmeasured baseline -> null.
66
 */
67
export function slowerPct(baseline, value) {
68
  if (baseline === null || baseline === undefined || value === null || value === undefined) {
69
    return null;
70
  }
71
  if (baseline <= 0) {
72
    return null;
73
  }
74
  return ((value - baseline) / baseline) * 100.0;
75
}
76
77
/** Port of oakbench.reporting.delta_ms (null when either side unmeasured). */
78
export function deltaMs(baseline, value) {
79
  if (baseline === null || baseline === undefined || value === null || value === undefined) {
80
    return null;
81
  }
82
  return value - baseline;
83
}
84
85
// --- rows.py accessor ports -----------------------------------------------
86
87
export function numberOrNone(value) {
88
  if (value === null || value === undefined) {
89
    return null;
90
  }
91
  const parsed = Number(value);
92
  return Number.isFinite(parsed) ? parsed : null;
93
}
94
95
/** Port of oakbench.rows.row_returncode (missing/null -> failure, never crash). */
96
export function rowReturncode(row) {
97
  const value = row.returncode;
98
  if (value === null || value === undefined) {
99
    return 1;
100
  }
101
  const parsed = Number(value);
102
  return Number.isInteger(parsed) ? parsed : 1;
103
}
104
105
const SKIP_RETURNCODE = 77;
106
const SUBOP_SUFFIXES = [".add", ".commit"];
107
108
// Explicit separator for every composite map key. Plain concatenation is
109
// collision-prone ("a" + "bc" === "ab" + "c"); ASCII unit separator cannot
110
// appear in subject/scenario/operation strings produced by the harness.
111
const KEY_SEP = "\u001f";
112
113
/**
114
 * Deterministic ingest identity for a row.
115
 *
116
 * Platform-lane rows (scripts/oakbench/rows.py LANE_REQUIRED_KEYS["platform"])
117
 * legitimately carry NO subject field — their identity is platform/driver
118
 * (e.g. "github/cli"). Rows with neither subject nor platform have no subject
119
 * identity and return null; callers must skip them from subject-keyed tables
120
 * explicitly and count them in an ingest report (never silently).
121
 */
122
export function effectiveSubject(row) {
123
  if (row.subject !== null && row.subject !== undefined) {
124
    return String(row.subject);
125
  }
126
  if (row.platform !== null && row.platform !== undefined) {
127
    return `${row.platform}/${row.driver}`;
128
  }
129
  return null;
130
}
131
132
/**
133
 * Subject kind matching effectiveSubject: explicit subject_kind when present;
134
 * for subject-less platform rows the kind is the platform name. Null when the
135
 * row carries neither (null means unmeasured, ADR-0002).
136
 */
137
export function effectiveSubjectKind(row) {
138
  if (row.subject_kind !== null && row.subject_kind !== undefined) {
139
    return String(row.subject_kind);
140
  }
141
  if (
142
    (row.subject === null || row.subject === undefined) &&
143
    row.platform !== null &&
144
    row.platform !== undefined
145
  ) {
146
    return String(row.platform);
147
  }
148
  return null;
149
}
150
151
function isObject(value) {
152
  return value !== null && typeof value === "object" && !Array.isArray(value);
153
}
154
155
/** Port of oakbench.rows.token_total (best-available, provider first). */
156
export function tokenTotal(row) {
157
  const tokenMetrics = isObject(row.token_metrics) ? row.token_metrics : null;
158
  if (tokenMetrics) {
159
    const total = numberOrNone(tokenMetrics.total_tokens_reported);
160
    if (total !== null) {
161
      return total;
162
    }
163
    const parts = [
164
      numberOrNone(tokenMetrics.input_tokens_reported),
165
      numberOrNone(tokenMetrics.output_tokens_reported),
166
      numberOrNone(tokenMetrics.reasoning_tokens_reported),
167
    ];
168
    if (parts.some((part) => part !== null)) {
169
      return parts.reduce((sum, part) => sum + (part || 0), 0);
170
    }
171
  }
172
173
  for (const key of ["estimated_tokens_total", "token_count", "total_tokens"]) {
174
    const value = numberOrNone(row[key]);
175
    if (value !== null) {
176
      return value;
177
    }
178
  }
179
180
  if (isObject(row.token_estimate)) {
181
    const value = numberOrNone(row.token_estimate.total);
182
    if (value !== null) {
183
      return value;
184
    }
185
  }
186
187
  if (isObject(row.tokens)) {
188
    for (const key of ["total", "total_tokens", "provider_total", "estimated_total"]) {
189
      const value = numberOrNone(row.tokens[key]);
190
      if (value !== null) {
191
        return value;
192
      }
193
    }
194
    if (isObject(row.tokens.provider_reported)) {
195
      const value = numberOrNone(row.tokens.provider_reported.total);
196
      if (value !== null) {
197
        return value;
198
      }
199
    }
200
  }
201
202
  if (tokenMetrics) {
203
    let chars = numberOrNone(tokenMetrics.transcript_chars);
204
    if (chars === null) {
205
      const inputChars = numberOrNone(tokenMetrics.normalized_input_chars);
206
      const outputChars = numberOrNone(tokenMetrics.normalized_output_chars);
207
      if (inputChars !== null || outputChars !== null) {
208
        chars = (inputChars || 0) + (outputChars || 0);
209
      }
210
    }
211
    if (chars !== null) {
212
      return Math.max(1, Math.ceil(chars / 4.0));
213
    }
214
  }
215
  return null;
216
}
217
218
/** Port of oakbench.rows.tool_calls. */
219
export function toolCalls(row) {
220
  for (const key of ["tool_call_count", "total_tool_calls"]) {
221
    const value = numberOrNone(row[key]);
222
    if (value !== null) {
223
      return value;
224
    }
225
  }
226
  if (isObject(row.tool_calls)) {
227
    for (const key of ["total", "all"]) {
228
      const value = numberOrNone(row.tool_calls[key]);
229
      if (value !== null) {
230
        return value;
231
      }
232
    }
233
  }
234
  if (isObject(row.tool_call_metrics)) {
235
    const value = numberOrNone(row.tool_call_metrics.tool_calls_total);
236
    if (value !== null) {
237
      return value;
238
    }
239
  }
240
  return null;
241
}
242
243
/** Port of oakbench.rows.output_bytes. */
244
export function outputBytes(row) {
245
  const raw = numberOrNone(row.raw_output_bytes);
246
  if (raw !== null) {
247
    return raw;
248
  }
249
  let stdout = numberOrNone(row.stdout_bytes);
250
  let stderr = numberOrNone(row.stderr_bytes);
251
  if (stdout !== null || stderr !== null) {
252
    return (stdout || 0) + (stderr || 0);
253
  }
254
  if (isObject(row.token_estimate)) {
255
    stdout = numberOrNone(row.token_estimate.stdout_bytes);
256
    stderr = numberOrNone(row.token_estimate.stderr_bytes);
257
    if (stdout !== null || stderr !== null) {
258
      return (stdout || 0) + (stderr || 0);
259
    }
260
  }
261
  return null;
262
}
263
264
/** Port of oakbench.rows.output_truncated. */
265
export function outputTruncated(row) {
266
  if (row.output_truncated || row.stdout_truncated || row.stderr_truncated) {
267
    return true;
268
  }
269
  if (isObject(row.token_estimate)) {
270
    return Boolean(row.token_estimate.stdout_truncated || row.token_estimate.stderr_truncated);
271
  }
272
  return false;
273
}
274
275
// --- Ingest step 1: rows -> measurements -----------------------------------
276
277
/**
278
 * Project JSONL rows into measurements-table records.
279
 *
280
 * Pure field projection: null/missing stays null (NEVER coerced to 0).
281
 * Sub-operation rows (operation ending .add/.commit) are marked
282
 * is_canonical = 0, matching oakbench.baseline_book.SUBOP_SUFFIXES.
283
 *
284
 * Subject identity uses effectiveSubject/effectiveSubjectKind so platform
285
 * rows (no subject field; identity platform/driver) group correctly. Rows
286
 * with no subject identity at all (no subject AND no platform) are skipped
287
 * from this subject-keyed projection explicitly: when a `report` object is
288
 * passed, its skipped_no_subject counter is incremented per skipped row so
289
 * the ingest response surfaces them (never silent).
290
 *
291
 * `subjectIds` optionally maps subjectKey(row) -> subject_id; when absent the
292
 * effective subject name is used so the transform stays pure and testable.
293
 */
294
export function rowsToMeasurements(rows, runKey, now, subjectIds = null, report = null) {
295
  const measurements = [];
296
  for (const row of rows) {
297
    const subject = effectiveSubject(row);
298
    if (subject === null) {
299
      if (report) {
300
        report.skipped_no_subject = (report.skipped_no_subject ?? 0) + 1;
301
      }
302
      continue;
303
    }
304
    const operation = String(row.operation ?? "");
305
    const key = subjectKey(row);
306
    const kind = effectiveSubjectKind(row);
307
    measurements.push({
308
      run_key: runKey,
309
      subject_id: (subjectIds && subjectIds.get(key)) || subject,
310
      subject,
311
      subject_kind: kind,
312
      scenario: String(row.scenario ?? ""),
313
      operation,
314
      run_index: numberOrNone(row.run) ?? 0,
315
      is_canonical: SUBOP_SUFFIXES.some((suffix) => operation.endsWith(suffix)) ? 0 : 1,
316
      elapsed_ms: numberOrNone(row.elapsed_ms),
317
      estimated_tokens_total: numberOrNone(row.estimated_tokens_total),
318
      estimated_tokens_input: numberOrNone(row.estimated_tokens_input),
319
      estimated_tokens_output: numberOrNone(row.estimated_tokens_output),
320
      token_source: row.token_source === null || row.token_source === undefined ? null : String(row.token_source),
321
      raw_output_bytes: numberOrNone(row.raw_output_bytes),
322
      stdout_bytes: numberOrNone(row.stdout_bytes),
323
      stderr_bytes: numberOrNone(row.stderr_bytes),
324
      output_truncated: outputTruncated(row) ? 1 : 0,
325
      tool_call_count: numberOrNone(row.tool_call_count),
326
      vcs_tool_call_count: numberOrNone(row.vcs_tool_call_count),
327
      terminal_tool_call_count: numberOrNone(row.terminal_tool_call_count),
328
      returncode: rowReturncode(row),
329
      command_json: row.command === undefined || row.command === null ? null : JSON.stringify(row.command),
330
      stderr: typeof row.stderr === "string" && row.stderr ? row.stderr.slice(-4000) : null,
331
      row_json: JSON.stringify(row),
332
      created_at: now,
333
      // schema_v2 additive columns (null when the lane did not record them).
334
      runner_id: row.runner_id === null || row.runner_id === undefined ? null : String(row.runner_id),
335
      runner_class: row.runner_class === null || row.runner_class === undefined ? null : String(row.runner_class),
336
      load_tier: row.load_tier === null || row.load_tier === undefined ? null : String(row.load_tier),
337
      netshape_profile:
338
        row.netshape_profile === null || row.netshape_profile === undefined ? null : String(row.netshape_profile),
339
      // Best-available derived metrics (oakbench.rows accessors); used by
340
      // computeAggregates, not persisted as columns.
341
      tokens_best: tokenTotal(row),
342
      output_bytes_best: outputBytes(row),
343
      tool_calls_best: toolCalls(row),
344
    });
345
  }
346
  return measurements;
347
}
348
349
// --- Ingest step 2: subjects upsert ----------------------------------------
350
351
/**
352
 * Identity key for a subject within a run: (effective subject, effective
353
 * subject_kind, binary_sha256). Platform rows (no subject field) key on
354
 * their platform/driver identity. Null when the row has no subject identity
355
 * at all (no subject AND no platform) so callers skip it explicitly.
356
 */
357
export function subjectKey(row) {
358
  const subject = effectiveSubject(row);
359
  if (subject === null) {
360
    return null;
361
  }
362
  const kind = effectiveSubjectKind(row);
363
  return [
364
    subject,
365
    kind === null ? "" : kind,
366
    row.binary_sha256 === null || row.binary_sha256 === undefined ? "" : String(row.binary_sha256),
367
  ].join(KEY_SEP);
368
}
369
370
/**
371
 * Build run-scoped subject upsert records, keyed (effective subject,
372
 * effective subject_kind, binary_sha256), enriched from the first row's
373
 * subject_details map. Platform rows (no subject field) are identified by
374
 * platform/driver via effectiveSubject so they ingest like any other subject.
375
 * Returns { records, idsByKey, skipped }: idsByKey maps subjectKey ->
376
 * subject_id; skipped counts rows with no subject identity at all (no
377
 * subject AND no platform), which are excluded from subject-keyed tables.
378
 */
379
export async function upsertSubjects(rows, runKey, subjectDetails = {}, subjectVersions = {}) {
380
  const records = [];
381
  const idsByKey = new Map();
382
  let skipped = 0;
383
  for (const row of rows) {
384
    const name = effectiveSubject(row);
385
    if (name === null) {
386
      skipped += 1;
387
      continue;
388
    }
389
    const key = subjectKey(row);
390
    if (idsByKey.has(key)) {
391
      continue;
392
    }
393
    const details = isObject(subjectDetails[name]) ? subjectDetails[name] : {};
394
    const binarySha = row.binary_sha256 ?? details.binary_sha256 ?? null;
395
    const subjectId = await stableId(`${runKey}${KEY_SEP}${key}`);
396
    idsByKey.set(key, subjectId);
397
    records.push({
398
      subject_id: subjectId,
399
      run_key: runKey,
400
      name,
401
      kind: String(effectiveSubjectKind(row) ?? details.kind ?? "unknown"),
402
      label: details.label === undefined ? null : details.label,
403
      version: subjectVersions[name] ?? details.version ?? null,
404
      binary_path: details.bin === undefined ? null : details.bin,
405
      metadata_json: JSON.stringify({ binary_sha256: binarySha, role: details.role ?? null }),
406
    });
407
  }
408
  return { records, idsByKey, skipped };
409
}
410
411
// --- Ingest step 3: aggregates ---------------------------------------------
412
413
/**
414
 * Per (run_key, subject_id, scenario, operation) aggregates.
415
 *
416
 * Latency percentiles use nearest-rank (percentileNearest) over the
417
 * elapsed_ms of SUCCESSFUL (returncode 0) measurements only, exactly matching
418
 * oakbench.stats.percentile_nearest semantics including empty -> null.
419
 * Skip rows (returncode 77) are excluded from sample/failure accounting.
420
 */
421
// Tail honesty thresholds, mirroring scripts/benchmark_stats.py
422
// TAIL_MIN_SAMPLES = {"p95": 20, "p99": 100} (itself a mirror of
423
// oakbench.reporting.TAIL_PERCENTILE_MIN_SAMPLES): below 1/(1-p) samples a
424
// nearest-rank tail percentile is just the max relabelled, so claiming it
425
// would overstate precision. Gated percentiles are null (unmeasured,
426
// ADR-0002), NEVER the lone sample. p50/p90 stay ungated to match the Python
427
// reporting behavior (benchmark_stats gates only p95/p99).
428
export const TAIL_MIN_SAMPLES = { p95: 20, p99: 100 };
429
430
export function computeAggregates(measurements, profile, now) {
431
  const groups = new Map();
432
  for (const m of measurements) {
433
    const key = [m.run_key, m.subject_id, m.scenario, m.operation].join(KEY_SEP);
434
    if (!groups.has(key)) {
435
      groups.set(key, []);
436
    }
437
    groups.get(key).push(m);
438
  }
439
440
  const aggregates = [];
441
  for (const group of groups.values()) {
442
    const first = group[0];
443
    const successes = group.filter((m) => m.returncode === 0);
444
    const failures = group.filter((m) => m.returncode !== 0 && m.returncode !== SKIP_RETURNCODE);
445
    const sampleCount = successes.length + failures.length;
446
    const elapsed = successes.map((m) => m.elapsed_ms).filter((v) => v !== null);
447
    const tokens = successes.map((m) => m.tokens_best).filter((v) => v !== null && v !== undefined);
448
    const bytes = successes.map((m) => m.output_bytes_best).filter((v) => v !== null && v !== undefined);
449
    const calls = successes.map((m) => m.tool_calls_best).filter((v) => v !== null && v !== undefined);
450
    aggregates.push({
451
      run_key: first.run_key,
452
      subject_id: first.subject_id,
453
      subject: first.subject,
454
      subject_kind: first.subject_kind,
455
      profile,
456
      scenario: first.scenario,
457
      operation: first.operation,
458
      is_canonical: first.is_canonical,
459
      sample_count: sampleCount,
460
      failure_count: failures.length,
461
      failure_rate: sampleCount > 0 ? failures.length / sampleCount : null,
462
      n: elapsed.length,
463
      p50_ms: median(elapsed),
464
      p90_ms: percentileNearest(elapsed, 90),
465
      p95_ms: elapsed.length >= TAIL_MIN_SAMPLES.p95 ? percentileNearest(elapsed, 95) : null,
466
      p99_ms: elapsed.length >= TAIL_MIN_SAMPLES.p99 ? percentileNearest(elapsed, 99) : null,
467
      mean_ms: mean(elapsed),
468
      min_ms: elapsed.length ? Math.min(...elapsed) : null,
469
      max_ms: elapsed.length ? Math.max(...elapsed) : null,
470
      avg_tokens: mean(tokens),
471
      avg_output_bytes: mean(bytes),
472
      truncated_count: group.filter((m) => m.output_truncated === 1).length,
473
      avg_tool_calls: mean(calls),
474
      updated_at: now,
475
    });
476
  }
477
  return aggregates;
478
}
479
480
// --- Ingest step 4: comparisons --------------------------------------------
481
482
/**
483
 * Target-vs-baseline comparison rows: every non-git subject aggregate is
484
 * compared against EVERY git-kind subject aggregate for the same
485
 * run/scenario/operation (e.g. stock git AND git_fsmonitor each yield a
486
 * comparison record, distinguished by baseline_subject_id). Ingest holds no
487
 * notion of "the" baseline: selecting which git variant to report against is
488
 * a reporting/targets concern (hardest-fair-comparator doctrine). Output is
489
 * sorted on (run_key, scenario, operation, target_subject_id,
490
 * baseline_subject_id) so results are deterministic regardless of input
491
 * row order.
492
 *
493
 * p50_ratio is target/baseline (null when baseline is unmeasured or 0 — a
494
 * zero baseline makes the ratio undefined, mirroring scorecard UNDECIDABLE).
495
 * p50_delta_pct_better uses deltaPct (positive = better) including its
496
 * base == 0 -> 0.0 guard; *_slower_pct uses slowerPct (positive = worse).
497
 *
498
 * is_regression is 1 when EITHER p50 or p90 breaches (slower_pct >
499
 * thresholdPct AND delta_ms > minDeltaMs, evaluated per dimension);
500
 * regression_dimensions lists the breaching dimensions ("p50"/"p90"),
501
 * empty when none.
502
 */
503
export function computeComparisons(aggregates, options = {}) {
504
  const thresholdPct = options.thresholdPct ?? 10.0;
505
  const minDeltaMs = options.minDeltaMs ?? 5.0;
506
507
  const baselinesByGroup = new Map();
508
  for (const agg of aggregates) {
509
    if (agg.subject_kind === "git") {
510
      const key = [agg.run_key, agg.scenario, agg.operation].join(KEY_SEP);
511
      if (!baselinesByGroup.has(key)) {
512
        baselinesByGroup.set(key, []);
513
      }
514
      baselinesByGroup.get(key).push(agg);
515
    }
516
  }
517
518
  const comparisons = [];
519
  for (const target of aggregates) {
520
    if (target.subject_kind === "git") {
521
      continue;
522
    }
523
    const bases = baselinesByGroup.get([target.run_key, target.scenario, target.operation].join(KEY_SEP)) || [];
524
    for (const base of bases) {
525
      if (base.subject_id === target.subject_id) {
526
        continue;
527
      }
528
      const p50SlowerPct = slowerPct(base.p50_ms, target.p50_ms);
529
      const p50DeltaMs = deltaMs(base.p50_ms, target.p50_ms);
530
      const p90SlowerPct = slowerPct(base.p90_ms, target.p90_ms);
531
      const p90DeltaMs = deltaMs(base.p90_ms, target.p90_ms);
532
      // A regression in EITHER p50 or p90 flags the comparison: agents pay
533
      // tail latency, so a p90-only blowup must not hide behind a flat p50.
534
      // Each dimension breaches when slower_pct > thresholdPct AND
535
      // delta_ms > minDeltaMs (nulls — unmeasured — never breach).
536
      const breaches = (slower, delta) =>
537
        slower !== null && delta !== null && slower > thresholdPct && delta > minDeltaMs;
538
      const regressionDimensions = [];
539
      if (breaches(p50SlowerPct, p50DeltaMs)) {
540
        regressionDimensions.push("p50");
541
      }
542
      if (breaches(p90SlowerPct, p90DeltaMs)) {
543
        regressionDimensions.push("p90");
544
      }
545
      const isRegression = regressionDimensions.length > 0 ? 1 : 0;
546
      comparisons.push({
547
        run_key: target.run_key,
548
        target_subject_id: target.subject_id,
549
        baseline_subject_id: base.subject_id,
550
        baseline_subject: base.subject,
551
        baseline_kind: "git",
552
        profile: target.profile,
553
        scenario: target.scenario,
554
        operation: target.operation,
555
        p50_delta_ms: p50DeltaMs,
556
        p50_slower_pct: p50SlowerPct,
557
        p50_ratio:
558
          base.p50_ms === null || base.p50_ms === 0 || target.p50_ms === null ? null : target.p50_ms / base.p50_ms,
559
        p50_delta_pct_better: deltaPct(base.p50_ms, target.p50_ms),
560
        p90_delta_ms: p90DeltaMs,
561
        p90_slower_pct: p90SlowerPct,
562
        p99_delta_ms: deltaMs(base.p99_ms, target.p99_ms),
563
        p99_slower_pct: slowerPct(base.p99_ms, target.p99_ms),
564
        avg_token_delta: deltaMs(base.avg_tokens, target.avg_tokens),
565
        token_higher_pct: slowerPct(base.avg_tokens, target.avg_tokens),
566
        avg_output_byte_delta: deltaMs(base.avg_output_bytes, target.avg_output_bytes),
567
        output_byte_higher_pct: slowerPct(base.avg_output_bytes, target.avg_output_bytes),
568
        avg_tool_call_delta: deltaMs(base.avg_tool_calls, target.avg_tool_calls),
569
        tool_call_higher_pct: slowerPct(base.avg_tool_calls, target.avg_tool_calls),
570
        failure_rate_delta_pp:
571
          target.failure_rate === null || target.failure_rate === undefined ||
572
          base.failure_rate === null || base.failure_rate === undefined
573
            ? null
574
            : (target.failure_rate - base.failure_rate) * 100.0,
575
        threshold_pct: thresholdPct,
576
        min_delta_ms: minDeltaMs,
577
        is_regression: isRegression,
578
        // Additive: which dimensions breached ("p50"/"p90"); empty when none.
579
        regression_dimensions: regressionDimensions,
580
        state: isRegression ? "regression" : "ok",
581
        severity_score: isRegression
582
          ? Math.max(
583
              regressionDimensions.includes("p50") ? p50SlowerPct : 0,
584
              regressionDimensions.includes("p90") ? p90SlowerPct : 0,
585
            )
586
          : 0,
587
      });
588
    }
589
  }
590
  comparisons.sort((a, b) => {
591
    const left = [a.run_key, a.scenario, a.operation, a.target_subject_id, a.baseline_subject_id].join(KEY_SEP);
592
    const right = [b.run_key, b.scenario, b.operation, b.target_subject_id, b.baseline_subject_id].join(KEY_SEP);
593
    return left < right ? -1 : left > right ? 1 : 0;
594
  });
595
  return comparisons;
596
}
597
598
// --- Scorecard + trends -----------------------------------------------------
599
600
const VERDICT_PRIORITY = ["RED", "INVALID", "UNDECIDABLE", "AMBER", "UNMEASURED", "GREEN"];
601
602
/**
603
 * Summarize scorecard cells (the verdict dicts produced by
604
 * oakbench.scorecard.evaluate_targets) into dashboard headline counts.
605
 */
606
export function scorecardFromCells(cells) {
607
  const counts = {};
608
  for (const cell of cells || []) {
609
    const verdict = String(cell.verdict ?? "UNMEASURED");
610
    counts[verdict] = (counts[verdict] || 0) + 1;
611
  }
612
  let overall = "UNMEASURED";
613
  for (const verdict of VERDICT_PRIORITY) {
614
    if (counts[verdict]) {
615
      overall = verdict;
616
      break;
617
    }
618
  }
619
  return { total: (cells || []).length, counts, overall };
620
}
621
622
const TREND_METRICS = {
623
  p50: "p50_ms",
624
  median: "p50_ms",
625
  p50_ms: "p50_ms",
626
  p90: "p90_ms",
627
  p90_ms: "p90_ms",
628
  p95: "p95_ms",
629
  p95_ms: "p95_ms",
630
  p99: "p99_ms",
631
  p99_ms: "p99_ms",
632
  mean: "mean_ms",
633
  mean_ms: "mean_ms",
634
  avg_tokens: "avg_tokens",
635
  avg_output_bytes: "avg_output_bytes",
636
  avg_tool_calls: "avg_tool_calls",
637
  failure_rate: "failure_rate",
638
};
639
640
/**
641
 * Build a per-subject time series from aggregate rows joined with run
642
 * timestamps: rows need {timestamp_utc, run_key, subject, scenario,
643
 * operation, <metric columns>}. Unknown metric -> null (caller 400s).
644
 */
645
export function trendSeries(rows, { scenario, operation, metric = "p50" } = {}) {
646
  const column = TREND_METRICS[metric];
647
  if (!column) {
648
    return null;
649
  }
650
  const yes = (row) =>
651
    (scenario === undefined || scenario === null || row.scenario === scenario) &&
652
    (operation === undefined || operation === null || row.operation === operation);
653
  const bySubject = new Map();
654
  const sorted = rows.filter(yes).sort((a, b) => String(a.timestamp_utc).localeCompare(String(b.timestamp_utc)));
655
  for (const row of sorted) {
656
    const subject = String(row.subject ?? row.name ?? "");
657
    if (!bySubject.has(subject)) {
658
      bySubject.set(subject, []);
659
    }
660
    bySubject.get(subject).push({
661
      t: row.timestamp_utc,
662
      run_key: row.run_key,
663
      value: row[column] === undefined ? null : row[column],
664
    });
665
  }
666
  return {
667
    scenario: scenario ?? null,
668
    operation: operation ?? null,
669
    metric: column,
670
    series: [...bySubject.entries()].map(([subject, points]) => ({ subject, points })),
671
  };
672
}
673
674
// --- Webhook signature ------------------------------------------------------
675
676
/**
677
 * Verify a GitHub X-Hub-Signature-256 header ("sha256=<hex>") against the raw
678
 * request body using HMAC-SHA256 (crypto.subtle; works in Workers and node).
679
 */
680
export async function verifySignature(secret, body, signatureHeader) {
681
  if (!secret || typeof signatureHeader !== "string" || !signatureHeader.startsWith("sha256=")) {
682
    return false;
683
  }
684
  const encoder = new TextEncoder();
685
  const key = await crypto.subtle.importKey(
686
    "raw",
687
    encoder.encode(secret),
688
    { name: "HMAC", hash: "SHA-256" },
689
    false,
690
    ["sign"],
691
  );
692
  const digest = await crypto.subtle.sign("HMAC", key, encoder.encode(body));
693
  const expected = [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
694
  const provided = signatureHeader.slice("sha256=".length).toLowerCase();
695
  if (provided.length !== expected.length) {
696
    return false;
697
  }
698
  let diff = 0;
699
  for (let index = 0; index < expected.length; index += 1) {
700
    diff |= expected.charCodeAt(index) ^ provided.charCodeAt(index);
701
  }
702
  return diff === 0;
703
}
704
705
// --- shared hashing ---------------------------------------------------------
706
707
export async function sha256Hex(text) {
708
  const bytes = new TextEncoder().encode(text);
709
  const digest = await crypto.subtle.digest("SHA-256", bytes);
710
  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
711
}
712
713
export async function stableId(text) {
714
  return (await sha256Hex(text)).slice(0, 24);
715
}