Viewing files at commit 5717cc9fc7edCommit detailsView latest

Browse files
cloudflare/reporting_parity.test.mjs 604 lines · 27.1 KB · javascript
1
// Python/JS reporting parity: assert the shared golden fixture
2
// (tests/fixtures/reporting_golden.json) against the worker's exported
3
// formulas. The same fixture is asserted from the Python side in
4
// tests/test_reporting_golden.py, so the two implementations cannot drift.
5
//
6
// Run: node --test cloudflare/reporting_parity.test.mjs
7
8
import assert from "node:assert/strict";
9
import { createHmac } from "node:crypto";
10
import { readFileSync } from "node:fs";
11
import { dirname, join } from "node:path";
12
import { test } from "node:test";
13
import { fileURLToPath } from "node:url";
14
15
import worker, {
16
  computeAggregates,
17
  computeComparisons,
18
  effectiveSubject,
19
  effectiveSubjectKind,
20
  ingestRows,
21
  MAX_UPLOAD_BYTES,
22
  rowsToMeasurements,
23
  scorecardFromCells,
24
  trendSeries,
25
  upsertSubjects,
26
  verifySignature,
27
} from "./worker.js";
28
import { TAIL_MIN_SAMPLES, deltaPct, percentileNearest, subjectKey } from "./reporting.mjs";
29
30
const HERE = dirname(fileURLToPath(import.meta.url));
31
const GOLDEN = JSON.parse(readFileSync(join(HERE, "..", "tests", "fixtures", "reporting_golden.json"), "utf8"));
32
const NOW = "2026-06-12T00:00:00Z";
33
34
test("percentile_nearest matches Python golden cases (empty, single, ties)", () => {
35
  for (const { values, p, expected } of GOLDEN.percentile_nearest) {
36
    assert.strictEqual(
37
      percentileNearest(values, p),
38
      expected,
39
      `percentileNearest(${JSON.stringify(values)}, ${p})`,
40
    );
41
  }
42
});
43
44
test("delta_pct matches Python golden cases including base == 0 guard", () => {
45
  for (const { base, value, expected } of GOLDEN.delta_pct) {
46
    assert.strictEqual(deltaPct(base, value), expected, `deltaPct(${base}, ${value})`);
47
  }
48
});
49
50
test("aggregate group matches Python golden (nulls preserved, skips/failures excluded)", () => {
51
  const { rows, expected } = GOLDEN.aggregate_group;
52
  const measurements = rowsToMeasurements(rows, "golden:profile", NOW);
53
54
  // Null preservation: unmeasured stays null, NEVER 0.
55
  const nullElapsed = measurements.find((m) => rows[measurements.indexOf(m)].elapsed_ms === null);
56
  assert.strictEqual(nullElapsed.elapsed_ms, null);
57
  const nullTokens = measurements.filter((m) => m.estimated_tokens_total === null);
58
  assert.ok(nullTokens.length >= 1, "null estimated_tokens_total preserved");
59
60
  const aggregates = computeAggregates(measurements, "golden", NOW);
61
  assert.strictEqual(aggregates.length, 1);
62
  const agg = aggregates[0];
63
  assert.strictEqual(agg.p50_ms, expected.median);
64
  assert.strictEqual(agg.p90_ms, expected.p90);
65
  // Tail honesty: at n=5 (< TAIL_MIN_SAMPLES) the aggregate reports p95/p99
66
  // as null — the fixture's expected.p95/p99 remain the RAW nearest-rank
67
  // values, still asserted below so the formula itself cannot drift.
68
  assert.strictEqual(agg.p95_ms, null);
69
  assert.strictEqual(agg.p99_ms, null);
70
  const elapsedValues = rows
71
    .filter((row) => row.returncode === 0 && row.elapsed_ms !== null)
72
    .map((row) => row.elapsed_ms);
73
  assert.strictEqual(percentileNearest(elapsedValues, 95), expected.p95);
74
  assert.strictEqual(percentileNearest(elapsedValues, 99), expected.p99);
75
  assert.strictEqual(agg.n, expected.n);
76
  assert.ok(expected.n < TAIL_MIN_SAMPLES.p95, "fixture group stays below the p95 gate");
77
  // 1 failure among 6 counted samples (skip row excluded entirely).
78
  assert.strictEqual(agg.failure_count, 1);
79
  assert.strictEqual(agg.sample_count, 7);
80
});
81
82
test("skip-only aggregate failure_rate is null, not fabricated zero", () => {
83
  const rows = [
84
    {
85
      subject: "git", subject_kind: "git", scenario: "s", operation: "skipped",
86
      run: 1, elapsed_ms: null, returncode: 77, skipped: true,
87
    },
88
    {
89
      subject: "oak_installed", subject_kind: "oak", scenario: "s", operation: "skipped",
90
      run: 1, elapsed_ms: null, returncode: 77, skipped: true,
91
    },
92
  ];
93
  const aggregates = computeAggregates(rowsToMeasurements(rows, "k:p", NOW), "p", NOW);
94
  assert.strictEqual(aggregates.length, 2);
95
  for (const agg of aggregates) {
96
    assert.strictEqual(agg.sample_count, 0);
97
    assert.strictEqual(agg.failure_count, 0);
98
    assert.strictEqual(agg.failure_rate, null);
99
    assert.strictEqual(agg.p50_ms, null);
100
  }
101
  const comparisons = computeComparisons(aggregates);
102
  assert.strictEqual(comparisons.length, 1);
103
  assert.strictEqual(comparisons[0].failure_rate_delta_pp, null);
104
});
105
106
test("oak/git comparison ratio and delta_pct match Python golden", () => {
107
  const { oak_values, git_values } = GOLDEN.comparison;
108
  const rows = [
109
    ...git_values.map((elapsed, run) => ({
110
      subject: "git", subject_kind: "git", scenario: "synthetic", operation: "status",
111
      run: run + 1, elapsed_ms: elapsed, returncode: 0,
112
    })),
113
    ...oak_values.map((elapsed, run) => ({
114
      subject: "oak_installed", subject_kind: "oak", scenario: "synthetic", operation: "status",
115
      run: run + 1, elapsed_ms: elapsed, returncode: 0,
116
    })),
117
  ];
118
  const measurements = rowsToMeasurements(rows, "golden:profile", NOW);
119
  const aggregates = computeAggregates(measurements, "golden", NOW);
120
  const comparisons = computeComparisons(aggregates);
121
  assert.strictEqual(comparisons.length, 1);
122
  const comparison = comparisons[0];
123
  assert.strictEqual(comparison.baseline_kind, "git");
124
  assert.strictEqual(comparison.p50_ratio, GOLDEN.comparison.expected_ratio);
125
  assert.strictEqual(comparison.p50_delta_pct_better, GOLDEN.comparison.expected_delta_pct);
126
127
  const oakAgg = aggregates.find((agg) => agg.subject === "oak_installed");
128
  const gitAgg = aggregates.find((agg) => agg.subject === "git");
129
  assert.strictEqual(oakAgg.p50_ms, GOLDEN.comparison.expected_oak_median);
130
  assert.strictEqual(gitAgg.p50_ms, GOLDEN.comparison.expected_git_median);
131
});
132
133
test("zero-baseline comparison: ratio is null, delta_pct guard yields 0", () => {
134
  const rows = [
135
    { subject: "git", subject_kind: "git", scenario: "z", operation: "noop", run: 1, elapsed_ms: 0.0, returncode: 0 },
136
    { subject: "oak_installed", subject_kind: "oak", scenario: "z", operation: "noop", run: 1, elapsed_ms: 3.0, returncode: 0 },
137
  ];
138
  const comparisons = computeComparisons(computeAggregates(rowsToMeasurements(rows, "k:p", NOW), "p", NOW));
139
  assert.strictEqual(comparisons.length, 1);
140
  assert.strictEqual(comparisons[0].p50_ratio, null);
141
  assert.strictEqual(comparisons[0].p50_delta_pct_better, 0.0);
142
  assert.strictEqual(comparisons[0].p50_slower_pct, null);
143
});
144
145
test("upsertSubjects keys on (subject, subject_kind, binary_sha256)", async () => {
146
  const rows = [
147
    { subject: "git", subject_kind: "git", binary_sha256: "aa" },
148
    { subject: "git", subject_kind: "git", binary_sha256: "aa" },
149
    { subject: "git", subject_kind: "git", binary_sha256: "bb" },
150
    { subject: "oak_installed", subject_kind: "oak", binary_sha256: null },
151
  ];
152
  const { records, idsByKey } = await upsertSubjects(rows, "run:profile", {
153
    git: { kind: "git", label: "Git", bin: "git", version: "git version 2.49.0" },
154
  });
155
  assert.strictEqual(records.length, 3);
156
  assert.strictEqual(idsByKey.size, 3);
157
  assert.strictEqual(records[0].version, "git version 2.49.0");
158
  const again = await upsertSubjects(rows, "run:profile", {});
159
  assert.deepStrictEqual(
160
    [...again.idsByKey.values()],
161
    [...idsByKey.values()],
162
    "subject ids are deterministic",
163
  );
164
});
165
166
test("platform rows (no subject field) ingest under platform/driver identity", async () => {
167
  // Platform-lane rows (rows.py LANE_REQUIRED_KEYS["platform"]) carry
168
  // platform/driver instead of subject; effective subject is "github/cli"
169
  // with subject_kind "github". A row with neither subject nor platform is
170
  // skipped from subject-keyed tables and counted, never silent.
171
  const rows = [
172
    {
173
      platform: "github", driver: "cli", scenario: "pr_flow", operation: "pr.create",
174
      run: 1, elapsed_ms: 120.0, returncode: 0, runner_id: "r-1", runner_class: "hosted",
175
    },
176
    {
177
      platform: "github", driver: "cli", scenario: "pr_flow", operation: "pr.create",
178
      run: 2, elapsed_ms: 140.0, returncode: 0, runner_id: "r-1", runner_class: "hosted",
179
    },
180
    { scenario: "pr_flow", operation: "pr.create", run: 3, elapsed_ms: 1.0, returncode: 0 },
181
  ];
182
  assert.strictEqual(effectiveSubject(rows[0]), "github/cli");
183
  assert.strictEqual(effectiveSubjectKind(rows[0]), "github");
184
  assert.strictEqual(effectiveSubject(rows[2]), null);
185
186
  const { records, idsByKey, skipped } = await upsertSubjects(rows, "run:platform");
187
  assert.strictEqual(records.length, 1);
188
  assert.strictEqual(records[0].name, "github/cli");
189
  assert.strictEqual(records[0].kind, "github");
190
  assert.strictEqual(skipped, 1);
191
  const subjectId = idsByKey.get(subjectKey(rows[0]));
192
  assert.ok(subjectId, "platform identity present in idsByKey");
193
194
  const report = {};
195
  const measurements = rowsToMeasurements(rows, "run:platform", NOW, idsByKey, report);
196
  assert.strictEqual(measurements.length, 2, "identity-less row excluded from measurements");
197
  assert.strictEqual(report.skipped_no_subject, 1);
198
  for (const m of measurements) {
199
    assert.strictEqual(m.subject, "github/cli");
200
    assert.strictEqual(m.subject_kind, "github");
201
    assert.strictEqual(m.subject_id, subjectId);
202
  }
203
204
  const aggregates = computeAggregates(measurements, "platform", NOW);
205
  assert.strictEqual(aggregates.length, 1, "both platform rows group under one identity");
206
  assert.strictEqual(aggregates[0].subject, "github/cli");
207
  assert.strictEqual(aggregates[0].subject_kind, "github");
208
  assert.strictEqual(aggregates[0].subject_id, subjectId);
209
  assert.strictEqual(aggregates[0].n, 2);
210
});
211
212
test("same subject name with different binary_sha256 stays two identities", async () => {
213
  const rows = [
214
    {
215
      subject: "oak_local", subject_kind: "oak", binary_sha256: "sha-a",
216
      scenario: "s", operation: "status", run: 1, elapsed_ms: 10.0, returncode: 0,
217
    },
218
    {
219
      subject: "oak_local", subject_kind: "oak", binary_sha256: "sha-b",
220
      scenario: "s", operation: "status", run: 1, elapsed_ms: 20.0, returncode: 0,
221
    },
222
  ];
223
  const { records, idsByKey } = await upsertSubjects(rows, "run:profile");
224
  assert.strictEqual(records.length, 2, "two subject records for two shas");
225
  assert.notStrictEqual(records[0].subject_id, records[1].subject_id);
226
  assert.deepStrictEqual(records.map((r) => r.name), ["oak_local", "oak_local"], "name stays the subject string");
227
228
  const measurements = rowsToMeasurements(rows, "run:profile", NOW, idsByKey);
229
  const aggregates = computeAggregates(measurements, "p", NOW);
230
  assert.strictEqual(aggregates.length, 2, "two aggregate groups, one per sha");
231
  assert.deepStrictEqual(
232
    new Set(aggregates.map((a) => a.subject_id)),
233
    new Set(records.map((r) => r.subject_id)),
234
  );
235
  assert.deepStrictEqual(aggregates.map((a) => a.p50_ms).sort((x, y) => x - y), [10, 20]);
236
});
237
238
test("comparisons run against every git-kind baseline, order-independent", () => {
239
  const rows = [
240
    { subject: "git", subject_kind: "git", scenario: "s", operation: "status", run: 1, elapsed_ms: 100.0, returncode: 0 },
241
    { subject: "git", subject_kind: "git", scenario: "s", operation: "status", run: 2, elapsed_ms: 110.0, returncode: 0 },
242
    { subject: "git_fsmonitor", subject_kind: "git", scenario: "s", operation: "status", run: 1, elapsed_ms: 50.0, returncode: 0 },
243
    { subject: "git_fsmonitor", subject_kind: "git", scenario: "s", operation: "status", run: 2, elapsed_ms: 60.0, returncode: 0 },
244
    { subject: "oak_local", subject_kind: "oak", scenario: "s", operation: "status", run: 1, elapsed_ms: 40.0, returncode: 0 },
245
    { subject: "oak_local", subject_kind: "oak", scenario: "s", operation: "status", run: 2, elapsed_ms: 45.0, returncode: 0 },
246
  ];
247
  const build = (input) => computeComparisons(computeAggregates(rowsToMeasurements(input, "k:p", NOW), "p", NOW));
248
249
  const comparisons = build(rows);
250
  assert.strictEqual(comparisons.length, 2, "one comparison per git-kind baseline");
251
  assert.ok(comparisons.every((c) => c.target_subject_id === "oak_local"));
252
  assert.deepStrictEqual(
253
    comparisons.map((c) => c.baseline_subject_id).sort(),
254
    ["git", "git_fsmonitor"],
255
    "distinct baseline ids",
256
  );
257
  assert.deepStrictEqual(comparisons.map((c) => c.baseline_subject).sort(), ["git", "git_fsmonitor"]);
258
259
  const vsGit = comparisons.find((c) => c.baseline_subject_id === "git");
260
  const vsFsmonitor = comparisons.find((c) => c.baseline_subject_id === "git_fsmonitor");
261
  assert.strictEqual(vsGit.p50_ratio, 42.5 / 105.0);
262
  assert.strictEqual(vsFsmonitor.p50_ratio, 42.5 / 55.0);
263
264
  // Deterministic regardless of input order: reversed and seeded-shuffled
265
  // permutations produce byte-identical comparison arrays.
266
  assert.deepStrictEqual(build([...rows].reverse()), comparisons);
267
  let seed = 1337;
268
  const next = () => {
269
    seed = (seed * 1103515245 + 12345) % 2147483648;
270
    return seed / 2147483648;
271
  };
272
  for (let round = 0; round < 5; round += 1) {
273
    const shuffled = [...rows];
274
    for (let i = shuffled.length - 1; i > 0; i -= 1) {
275
      const j = Math.floor(next() * (i + 1));
276
      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
277
    }
278
    assert.deepStrictEqual(build(shuffled), comparisons, `shuffle round ${round}`);
279
  }
280
});
281
282
test("scorecardFromCells summarizes verdicts with worst-first overall", () => {
283
  const summary = scorecardFromCells([
284
    { verdict: "GREEN" },
285
    { verdict: "AMBER" },
286
    { verdict: "RED" },
287
    { verdict: "GREEN" },
288
  ]);
289
  assert.deepStrictEqual(summary, {
290
    total: 4,
291
    counts: { GREEN: 2, AMBER: 1, RED: 1 },
292
    overall: "RED",
293
  });
294
  assert.strictEqual(scorecardFromCells([]).overall, "UNMEASURED");
295
});
296
297
test("trendSeries groups aggregate rows per subject in time order", () => {
298
  const rows = [
299
    { timestamp_utc: "2026-06-02T00:00:00Z", run_key: "b:p", subject: "git", scenario: "s", operation: "o", p50_ms: 21 },
300
    { timestamp_utc: "2026-06-01T00:00:00Z", run_key: "a:p", subject: "git", scenario: "s", operation: "o", p50_ms: 20 },
301
    { timestamp_utc: "2026-06-01T00:00:00Z", run_key: "a:p", subject: "oak_installed", scenario: "s", operation: "o", p50_ms: 11 },
302
    { timestamp_utc: "2026-06-01T00:00:00Z", run_key: "a:p", subject: "git", scenario: "other", operation: "o", p50_ms: 99 },
303
  ];
304
  const series = trendSeries(rows, { scenario: "s", operation: "o", metric: "p50" });
305
  assert.strictEqual(series.metric, "p50_ms");
306
  const git = series.series.find((entry) => entry.subject === "git");
307
  assert.deepStrictEqual(git.points.map((point) => point.value), [20, 21]);
308
  assert.strictEqual(trendSeries(rows, { scenario: "s", operation: "o", metric: "bogus" }), null);
309
});
310
311
test("aggregate tail gating matches golden: p95 null below n=20, p99 null below n=100", () => {
312
  // Parity note: Python gates tail percentiles in
313
  // oakbench.reporting.tail_latency_summary (and at the rendering layer in
314
  // scripts/benchmark_stats.py) — there is no shared Python aggregate
315
  // function identical to computeAggregates. tests/test_reporting_golden.py
316
  // asserts these same golden cases against tail_latency_summary, so the
317
  // gate thresholds and gated values cannot drift between the two sides.
318
  assert.strictEqual(TAIL_MIN_SAMPLES.p95, 20);
319
  assert.strictEqual(TAIL_MIN_SAMPLES.p99, 100);
320
  const sizes = GOLDEN.aggregate_tail_gating.cases.map((c) => c.values.length);
321
  assert.deepStrictEqual(sizes, [1, 20, 100], "fixture covers n=1 and both gate boundaries");
322
323
  for (const { values, expected } of GOLDEN.aggregate_tail_gating.cases) {
324
    const rows = values.map((elapsed, index) => ({
325
      subject: "oak_installed", subject_kind: "oak", scenario: "tail", operation: "status",
326
      run: index + 1, elapsed_ms: elapsed, returncode: 0,
327
    }));
328
    const aggregates = computeAggregates(rowsToMeasurements(rows, "k:p", NOW), "p", NOW);
329
    assert.strictEqual(aggregates.length, 1);
330
    const agg = aggregates[0];
331
    assert.strictEqual(agg.n, values.length);
332
    assert.strictEqual(agg.p50_ms, expected.p50, `p50 stays ungated at n=${values.length}`);
333
    assert.notStrictEqual(agg.p90_ms, null, `p90 stays ungated at n=${values.length}`);
334
    assert.strictEqual(agg.p95_ms, expected.p95, `p95 at n=${values.length}`);
335
    assert.strictEqual(agg.p99_ms, expected.p99, `p99 at n=${values.length}`);
336
  }
337
});
338
339
test("p90-only regression flags is_regression and regression_dimensions", () => {
340
  const makeAgg = (subjectId, kind, p50, p90) => ({
341
    run_key: "k:p", subject_id: subjectId, subject: subjectId, subject_kind: kind,
342
    profile: "p", scenario: "s", operation: "status", is_canonical: 1,
343
    sample_count: 5, failure_count: 0, failure_rate: 0, n: 5,
344
    p50_ms: p50, p90_ms: p90, p95_ms: null, p99_ms: null,
345
    mean_ms: p50, min_ms: p50, max_ms: p90,
346
    avg_tokens: null, avg_output_bytes: null, truncated_count: 0, avg_tool_calls: null,
347
    updated_at: NOW,
348
  });
349
  const compare = (basP50, basP90, tgtP50, tgtP90) => {
350
    const comparisons = computeComparisons([
351
      makeAgg("git", "git", basP50, basP90),
352
      makeAgg("oak_installed", "oak", tgtP50, tgtP90),
353
    ]);
354
    assert.strictEqual(comparisons.length, 1);
355
    return comparisons[0];
356
  };
357
358
  // Reviewer repro: p50 flat (10 -> 10) but p90 blows up (10 -> 100).
359
  const p90Only = compare(10, 10, 10, 100);
360
  assert.strictEqual(p90Only.is_regression, 1);
361
  assert.deepStrictEqual(p90Only.regression_dimensions, ["p90"]);
362
  assert.strictEqual(p90Only.state, "regression");
363
  assert.strictEqual(p90Only.severity_score, 900, "severity follows the breaching dimension");
364
365
  // p50-only breach: p90 delta (1ms) stays under minDeltaMs (5ms).
366
  const p50Only = compare(10, 100, 100, 101);
367
  assert.strictEqual(p50Only.is_regression, 1);
368
  assert.deepStrictEqual(p50Only.regression_dimensions, ["p50"]);
369
370
  // Neither dimension breaches (4% slower, under the 10% threshold).
371
  const clean = compare(10, 10, 10.4, 10.4);
372
  assert.strictEqual(clean.is_regression, 0);
373
  assert.deepStrictEqual(clean.regression_dimensions, []);
374
  assert.strictEqual(clean.state, "ok");
375
  assert.strictEqual(clean.severity_score, 0);
376
});
377
378
// Minimal D1 stub for driving ingestRows: prepare(...).bind(...) statements
379
// carry their SQL + bound args; batch()/run() route to an in-memory table
380
// keyed by run_key (args[0] for measurements/aggregates/comparisons inserts,
381
// args[1] for subjects whose first bind is subject_id).
382
class StubD1 {
383
  constructor() {
384
    this.tables = { subjects: [], measurements: [], aggregates: [], comparisons: [] };
385
    this.failInsertsForTable = null;
386
    this.batchCalls = [];
387
  }
388
389
  prepare(sql) {
390
    const stub = this;
391
    return {
392
      bind(...args) {
393
        return {
394
          sql,
395
          args,
396
          run: async () => stub.execute(sql, args),
397
          first: async () => null,
398
        };
399
      },
400
    };
401
  }
402
403
  async batch(statements) {
404
    this.batchCalls.push(statements);
405
    const snapshot = Object.fromEntries(
406
      Object.entries(this.tables).map(([table, rows]) => [
407
        table,
408
        rows.map((row) => ({ runKey: row.runKey, args: [...row.args] })),
409
      ]),
410
    );
411
    try {
412
      for (const statement of statements) {
413
        this.execute(statement.sql, statement.args ?? []);
414
      }
415
    } catch (error) {
416
      this.tables = snapshot;
417
      throw error;
418
    }
419
  }
420
421
  execute(sql, args) {
422
    const del = sql.match(/DELETE FROM (\w+) WHERE run_key = \?/);
423
    if (del) {
424
      this.tables[del[1]] = this.tables[del[1]].filter((row) => row.runKey !== args[0]);
425
      return { success: true };
426
    }
427
    const ins = sql.match(/INSERT INTO (\w+)/);
428
    if (ins) {
429
      const table = ins[1];
430
      if (this.failInsertsForTable === table) {
431
        throw new Error(`injected insert failure for ${table}`);
432
      }
433
      const runKey = table === "subjects" ? args[1] : args[0];
434
      if (table === "subjects" && sql.includes("ON CONFLICT(subject_id)")) {
435
        this.tables.subjects = this.tables.subjects.filter((row) => row.args[0] !== args[0]);
436
      }
437
      this.tables[table].push({ runKey, args });
438
      return { success: true };
439
    }
440
    throw new Error(`StubD1: unsupported SQL: ${sql}`);
441
  }
442
}
443
444
test("re-upload with fewer groups leaves no stale aggregates/comparisons", async () => {
445
  const groupRows = (operations) => operations.flatMap((operation) => [
446
    { subject: "git", subject_kind: "git", scenario: "s", operation, run: 1, elapsed_ms: 10.0, returncode: 0 },
447
    { subject: "oak_installed", subject_kind: "oak", scenario: "s", operation, run: 1, elapsed_ms: 12.0, returncode: 0 },
448
  ]);
449
  const db = new StubD1();
450
451
  // A different run_key seeded first must survive the re-upload deletes.
452
  await ingestRows(db, groupRows(["keep"]), "other:p", "p", NOW);
453
454
  const first = await ingestRows(db, groupRows(["o1", "o2", "o3"]), "rk:p", "p", NOW);
455
  assert.strictEqual(first.aggregates, 6, "3 groups x 2 subjects");
456
  assert.strictEqual(first.comparisons, 3);
457
  assert.strictEqual(db.tables.aggregates.filter((r) => r.runKey === "rk:p").length, 6);
458
  assert.strictEqual(db.tables.comparisons.filter((r) => r.runKey === "rk:p").length, 3);
459
460
  // Re-upload the SAME run_key with only 1 of the 3 groups.
461
  await ingestRows(db, groupRows(["o1"]), "rk:p", "p", NOW);
462
  const aggs = db.tables.aggregates.filter((r) => r.runKey === "rk:p");
463
  const comps = db.tables.comparisons.filter((r) => r.runKey === "rk:p");
464
  assert.strictEqual(aggs.length, 2, "exactly the re-uploaded group's aggregates");
465
  assert.strictEqual(comps.length, 1, "exactly the re-uploaded group's comparison");
466
  // Aggregates bind order: run_key, subject_id, profile, scenario, operation.
467
  assert.deepStrictEqual(new Set(aggs.map((r) => r.args[4])), new Set(["o1"]));
468
  // Comparisons bind order: ..., profile, scenario, operation at index 6.
469
  assert.deepStrictEqual(comps.map((r) => r.args[6]), ["o1"]);
470
  assert.strictEqual(db.tables.measurements.filter((r) => r.runKey === "rk:p").length, 2);
471
472
  // Other run untouched.
473
  assert.strictEqual(db.tables.aggregates.filter((r) => r.runKey === "other:p").length, 2);
474
  assert.strictEqual(db.tables.comparisons.filter((r) => r.runKey === "other:p").length, 1);
475
});
476
477
test("failed run rewrite leaves previous run rows intact", async () => {
478
  const groupRows = (operations) => operations.flatMap((operation) => [
479
    { subject: "git", subject_kind: "git", scenario: "s", operation, run: 1, elapsed_ms: 10.0, returncode: 0 },
480
    { subject: "oak_installed", subject_kind: "oak", scenario: "s", operation, run: 1, elapsed_ms: 12.0, returncode: 0 },
481
  ]);
482
  const db = new StubD1();
483
  await ingestRows(db, groupRows(["old"]), "rk:p", "p", NOW);
484
485
  db.failInsertsForTable = "measurements";
486
  await assert.rejects(
487
    () => ingestRows(db, groupRows(["new"]), "rk:p", "p", NOW),
488
    /injected insert failure for measurements/,
489
  );
490
491
  const measurements = db.tables.measurements.filter((r) => r.runKey === "rk:p");
492
  const aggs = db.tables.aggregates.filter((r) => r.runKey === "rk:p");
493
  const comps = db.tables.comparisons.filter((r) => r.runKey === "rk:p");
494
  assert.strictEqual(measurements.length, 2);
495
  assert.strictEqual(aggs.length, 2);
496
  assert.strictEqual(comps.length, 1);
497
  assert.deepStrictEqual(new Set(measurements.map((r) => r.args[3])), new Set(["old"]));
498
  assert.deepStrictEqual(new Set(aggs.map((r) => r.args[4])), new Set(["old"]));
499
  assert.deepStrictEqual(new Set(comps.map((r) => r.args[6])), new Set(["old"]));
500
});
501
502
test("regression_dimensions is persisted through the comparisons insert", async () => {
503
  // p90-only regression: oak medians match git (10), but oak's p90 hits the
504
  // 100ms outlier (nearest-rank p90 of 3 samples = the 3rd).
505
  const rows = [
506
    ...[1, 2, 3].map((run) => ({
507
      subject: "git", subject_kind: "git", scenario: "s", operation: "o", run,
508
      elapsed_ms: 10.0, returncode: 0,
509
    })),
510
    ...[10.0, 10.0, 100.0].map((elapsed, i) => ({
511
      subject: "oak_installed", subject_kind: "oak", scenario: "s", operation: "o", run: i + 1,
512
      elapsed_ms: elapsed, returncode: 0,
513
    })),
514
  ];
515
  const db = new StubD1();
516
  await ingestRows(db, rows, "rk:dims", "p", NOW);
517
518
  const comps = db.tables.comparisons.filter((r) => r.runKey === "rk:dims");
519
  assert.strictEqual(comps.length, 1);
520
  const insertSql = `run_key, target_subject_id, baseline_subject_id, baseline_kind, profile,
521
      scenario, operation,
522
      p50_delta_ms, p50_slower_pct, p90_delta_ms, p90_slower_pct,
523
      p99_delta_ms, p99_slower_pct,
524
      avg_token_delta, token_higher_pct, avg_output_byte_delta, output_byte_higher_pct,
525
      avg_tool_call_delta, tool_call_higher_pct, failure_rate_delta_pp,
526
      threshold_pct, min_delta_ms, is_regression, regression_dimensions, state, severity_score, updated_at`;
527
  const columns = insertSql.replace(/\s+/g, " ").split(", ").map((c) => c.trim());
528
  const dimsIndex = columns.indexOf("regression_dimensions");
529
  assert.ok(dimsIndex >= 0, "insert must include the regression_dimensions column");
530
  const persisted = comps[0].args[dimsIndex];
531
  assert.strictEqual(typeof persisted, "string", "dimensions persist as a JSON string");
532
  assert.deepStrictEqual(JSON.parse(persisted), ["p90"]);
533
  const isRegressionIndex = columns.indexOf("is_regression");
534
  assert.strictEqual(comps[0].args[isRegressionIndex], 1);
535
});
536
537
test("verifySignature accepts a valid X-Hub-Signature-256 and rejects tampering", async () => {
538
  const secret = "shhh-bench";
539
  const body = JSON.stringify({ action: "push", ref: "refs/heads/main" });
540
  const header = `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`;
541
  assert.strictEqual(await verifySignature(secret, body, header), true);
542
  assert.strictEqual(await verifySignature(secret, body + "x", header), false);
543
  assert.strictEqual(await verifySignature("wrong", body, header), false);
544
  assert.strictEqual(await verifySignature(secret, body, "sha256=deadbeef"), false);
545
  assert.strictEqual(await verifySignature(secret, body, ""), false);
546
});
547
548
test("write routes fail closed without UPLOAD_TOKEN but GitHub webhook does not require it", async () => {
549
  const writeResponse = await worker.fetch(
550
    new Request("https://bench.example/api/scorecard", {
551
      method: "POST",
552
      body: JSON.stringify({ bench_id: "b", profile: "p", scorecard: [] }),
553
    }),
554
    {},
555
  );
556
  assert.strictEqual(writeResponse.status, 503);
557
  assert.deepStrictEqual(await writeResponse.json(), {
558
    ok: false,
559
    error: "upload_token_not_configured",
560
  });
561
562
  const secret = "webhook-secret";
563
  const body = JSON.stringify({ action: "push" });
564
  const signature = `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`;
565
  const webhookResponse = await worker.fetch(
566
    new Request("https://bench.example/webhook/github", {
567
      method: "POST",
568
      body,
569
      headers: {
570
        "x-hub-signature-256": signature,
571
        "x-github-event": "push",
572
        "x-github-delivery": "delivery-1",
573
      },
574
    }),
575
    { GITHUB_WEBHOOK_SECRET: secret },
576
  );
577
  assert.strictEqual(webhookResponse.status, 200);
578
  assert.deepStrictEqual(await webhookResponse.json(), { ok: true, recorded: false });
579
});
580
581
test("upload limit is enforced on UTF-8 bytes, not UTF-16 string length", async () => {
582
  const body = "é".repeat(Math.floor(MAX_UPLOAD_BYTES / 2) + 1);
583
  assert.ok(body.length <= MAX_UPLOAD_BYTES, "fixture must be below the old UTF-16 length guard");
584
  assert.ok(new TextEncoder().encode(body).byteLength > MAX_UPLOAD_BYTES);
585
586
  const response = await worker.fetch(
587
    new Request("https://bench.example/api/upload", {
588
      method: "POST",
589
      body,
590
      headers: {
591
        authorization: "Bearer upload-token",
592
        "content-length": String(body.length),
593
      },
594
    }),
595
    { UPLOAD_TOKEN: "upload-token" },
596
  );
597
598
  assert.strictEqual(response.status, 413);
599
  assert.deepStrictEqual(await response.json(), {
600
    ok: false,
601
    error: "upload_too_large",
602
    max_bytes: MAX_UPLOAD_BYTES,
603
  });
604
});