| 1 | // Python/JS reporting parity: assert the shared golden fixture
|
| 2 | // (tests/fixtures/reporting_golden.json) against the worker's exported
|
| 3 | // formulas. The same fixture is asserted from the Python side in
|
| 4 | // tests/test_reporting_golden.py, so the two implementations cannot drift.
|
| 5 | //
|
| 6 | // Run: node --test cloudflare/reporting_parity.test.mjs
|
| 7 |
|
| 8 | import assert from "node:assert/strict";
|
| 9 | import { createHmac } from "node:crypto";
|
| 10 | import { readFileSync } from "node:fs";
|
| 11 | import { dirname, join } from "node:path";
|
| 12 | import { test } from "node:test";
|
| 13 | import { fileURLToPath } from "node:url";
|
| 14 |
|
| 15 | import worker, {
|
| 16 | computeAggregates,
|
| 17 | computeComparisons,
|
| 18 | effectiveSubject,
|
| 19 | effectiveSubjectKind,
|
| 20 | ingestRows,
|
| 21 | MAX_UPLOAD_BYTES,
|
| 22 | rowsToMeasurements,
|
| 23 | scorecardFromCells,
|
| 24 | trendSeries,
|
| 25 | upsertSubjects,
|
| 26 | verifySignature,
|
| 27 | } from "./worker.js";
|
| 28 | import { TAIL_MIN_SAMPLES, deltaPct, percentileNearest, subjectKey } from "./reporting.mjs";
|
| 29 |
|
| 30 | const HERE = dirname(fileURLToPath(import.meta.url));
|
| 31 | const GOLDEN = JSON.parse(readFileSync(join(HERE, "..", "tests", "fixtures", "reporting_golden.json"), "utf8"));
|
| 32 | const NOW = "2026-06-12T00:00:00Z";
|
| 33 |
|
| 34 | test("percentile_nearest matches Python golden cases (empty, single, ties)", () => {
|
| 35 | for (const { values, p, expected } of GOLDEN.percentile_nearest) {
|
| 36 | assert.strictEqual(
|
| 37 | percentileNearest(values, p),
|
| 38 | expected,
|
| 39 | `percentileNearest(${JSON.stringify(values)}, ${p})`,
|
| 40 | );
|
| 41 | }
|
| 42 | });
|
| 43 |
|
| 44 | test("delta_pct matches Python golden cases including base == 0 guard", () => {
|
| 45 | for (const { base, value, expected } of GOLDEN.delta_pct) {
|
| 46 | assert.strictEqual(deltaPct(base, value), expected, `deltaPct(${base}, ${value})`);
|
| 47 | }
|
| 48 | });
|
| 49 |
|
| 50 | test("aggregate group matches Python golden (nulls preserved, skips/failures excluded)", () => {
|
| 51 | const { rows, expected } = GOLDEN.aggregate_group;
|
| 52 | const measurements = rowsToMeasurements(rows, "golden:profile", NOW);
|
| 53 |
|
| 54 | // Null preservation: unmeasured stays null, NEVER 0.
|
| 55 | const nullElapsed = measurements.find((m) => rows[measurements.indexOf(m)].elapsed_ms === null);
|
| 56 | assert.strictEqual(nullElapsed.elapsed_ms, null);
|
| 57 | const nullTokens = measurements.filter((m) => m.estimated_tokens_total === null);
|
| 58 | assert.ok(nullTokens.length >= 1, "null estimated_tokens_total preserved");
|
| 59 |
|
| 60 | const aggregates = computeAggregates(measurements, "golden", NOW);
|
| 61 | assert.strictEqual(aggregates.length, 1);
|
| 62 | const agg = aggregates[0];
|
| 63 | assert.strictEqual(agg.p50_ms, expected.median);
|
| 64 | assert.strictEqual(agg.p90_ms, expected.p90);
|
| 65 | // Tail honesty: at n=5 (< TAIL_MIN_SAMPLES) the aggregate reports p95/p99
|
| 66 | // as null — the fixture's expected.p95/p99 remain the RAW nearest-rank
|
| 67 | // values, still asserted below so the formula itself cannot drift.
|
| 68 | assert.strictEqual(agg.p95_ms, null);
|
| 69 | assert.strictEqual(agg.p99_ms, null);
|
| 70 | const elapsedValues = rows
|
| 71 | .filter((row) => row.returncode === 0 && row.elapsed_ms !== null)
|
| 72 | .map((row) => row.elapsed_ms);
|
| 73 | assert.strictEqual(percentileNearest(elapsedValues, 95), expected.p95);
|
| 74 | assert.strictEqual(percentileNearest(elapsedValues, 99), expected.p99);
|
| 75 | assert.strictEqual(agg.n, expected.n);
|
| 76 | assert.ok(expected.n < TAIL_MIN_SAMPLES.p95, "fixture group stays below the p95 gate");
|
| 77 | // 1 failure among 6 counted samples (skip row excluded entirely).
|
| 78 | assert.strictEqual(agg.failure_count, 1);
|
| 79 | assert.strictEqual(agg.sample_count, 7);
|
| 80 | });
|
| 81 |
|
| 82 | test("skip-only aggregate failure_rate is null, not fabricated zero", () => {
|
| 83 | const rows = [
|
| 84 | {
|
| 85 | subject: "git", subject_kind: "git", scenario: "s", operation: "skipped",
|
| 86 | run: 1, elapsed_ms: null, returncode: 77, skipped: true,
|
| 87 | },
|
| 88 | {
|
| 89 | subject: "oak_installed", subject_kind: "oak", scenario: "s", operation: "skipped",
|
| 90 | run: 1, elapsed_ms: null, returncode: 77, skipped: true,
|
| 91 | },
|
| 92 | ];
|
| 93 | const aggregates = computeAggregates(rowsToMeasurements(rows, "k:p", NOW), "p", NOW);
|
| 94 | assert.strictEqual(aggregates.length, 2);
|
| 95 | for (const agg of aggregates) {
|
| 96 | assert.strictEqual(agg.sample_count, 0);
|
| 97 | assert.strictEqual(agg.failure_count, 0);
|
| 98 | assert.strictEqual(agg.failure_rate, null);
|
| 99 | assert.strictEqual(agg.p50_ms, null);
|
| 100 | }
|
| 101 | const comparisons = computeComparisons(aggregates);
|
| 102 | assert.strictEqual(comparisons.length, 1);
|
| 103 | assert.strictEqual(comparisons[0].failure_rate_delta_pp, null);
|
| 104 | });
|
| 105 |
|
| 106 | test("oak/git comparison ratio and delta_pct match Python golden", () => {
|
| 107 | const { oak_values, git_values } = GOLDEN.comparison;
|
| 108 | const rows = [
|
| 109 | ...git_values.map((elapsed, run) => ({
|
| 110 | subject: "git", subject_kind: "git", scenario: "synthetic", operation: "status",
|
| 111 | run: run + 1, elapsed_ms: elapsed, returncode: 0,
|
| 112 | })),
|
| 113 | ...oak_values.map((elapsed, run) => ({
|
| 114 | subject: "oak_installed", subject_kind: "oak", scenario: "synthetic", operation: "status",
|
| 115 | run: run + 1, elapsed_ms: elapsed, returncode: 0,
|
| 116 | })),
|
| 117 | ];
|
| 118 | const measurements = rowsToMeasurements(rows, "golden:profile", NOW);
|
| 119 | const aggregates = computeAggregates(measurements, "golden", NOW);
|
| 120 | const comparisons = computeComparisons(aggregates);
|
| 121 | assert.strictEqual(comparisons.length, 1);
|
| 122 | const comparison = comparisons[0];
|
| 123 | assert.strictEqual(comparison.baseline_kind, "git");
|
| 124 | assert.strictEqual(comparison.p50_ratio, GOLDEN.comparison.expected_ratio);
|
| 125 | assert.strictEqual(comparison.p50_delta_pct_better, GOLDEN.comparison.expected_delta_pct);
|
| 126 |
|
| 127 | const oakAgg = aggregates.find((agg) => agg.subject === "oak_installed");
|
| 128 | const gitAgg = aggregates.find((agg) => agg.subject === "git");
|
| 129 | assert.strictEqual(oakAgg.p50_ms, GOLDEN.comparison.expected_oak_median);
|
| 130 | assert.strictEqual(gitAgg.p50_ms, GOLDEN.comparison.expected_git_median);
|
| 131 | });
|
| 132 |
|
| 133 | test("zero-baseline comparison: ratio is null, delta_pct guard yields 0", () => {
|
| 134 | const rows = [
|
| 135 | { subject: "git", subject_kind: "git", scenario: "z", operation: "noop", run: 1, elapsed_ms: 0.0, returncode: 0 },
|
| 136 | { subject: "oak_installed", subject_kind: "oak", scenario: "z", operation: "noop", run: 1, elapsed_ms: 3.0, returncode: 0 },
|
| 137 | ];
|
| 138 | const comparisons = computeComparisons(computeAggregates(rowsToMeasurements(rows, "k:p", NOW), "p", NOW));
|
| 139 | assert.strictEqual(comparisons.length, 1);
|
| 140 | assert.strictEqual(comparisons[0].p50_ratio, null);
|
| 141 | assert.strictEqual(comparisons[0].p50_delta_pct_better, 0.0);
|
| 142 | assert.strictEqual(comparisons[0].p50_slower_pct, null);
|
| 143 | });
|
| 144 |
|
| 145 | test("upsertSubjects keys on (subject, subject_kind, binary_sha256)", async () => {
|
| 146 | const rows = [
|
| 147 | { subject: "git", subject_kind: "git", binary_sha256: "aa" },
|
| 148 | { subject: "git", subject_kind: "git", binary_sha256: "aa" },
|
| 149 | { subject: "git", subject_kind: "git", binary_sha256: "bb" },
|
| 150 | { subject: "oak_installed", subject_kind: "oak", binary_sha256: null },
|
| 151 | ];
|
| 152 | const { records, idsByKey } = await upsertSubjects(rows, "run:profile", {
|
| 153 | git: { kind: "git", label: "Git", bin: "git", version: "git version 2.49.0" },
|
| 154 | });
|
| 155 | assert.strictEqual(records.length, 3);
|
| 156 | assert.strictEqual(idsByKey.size, 3);
|
| 157 | assert.strictEqual(records[0].version, "git version 2.49.0");
|
| 158 | const again = await upsertSubjects(rows, "run:profile", {});
|
| 159 | assert.deepStrictEqual(
|
| 160 | [...again.idsByKey.values()],
|
| 161 | [...idsByKey.values()],
|
| 162 | "subject ids are deterministic",
|
| 163 | );
|
| 164 | });
|
| 165 |
|
| 166 | test("platform rows (no subject field) ingest under platform/driver identity", async () => {
|
| 167 | // Platform-lane rows (rows.py LANE_REQUIRED_KEYS["platform"]) carry
|
| 168 | // platform/driver instead of subject; effective subject is "github/cli"
|
| 169 | // with subject_kind "github". A row with neither subject nor platform is
|
| 170 | // skipped from subject-keyed tables and counted, never silent.
|
| 171 | const rows = [
|
| 172 | {
|
| 173 | platform: "github", driver: "cli", scenario: "pr_flow", operation: "pr.create",
|
| 174 | run: 1, elapsed_ms: 120.0, returncode: 0, runner_id: "r-1", runner_class: "hosted",
|
| 175 | },
|
| 176 | {
|
| 177 | platform: "github", driver: "cli", scenario: "pr_flow", operation: "pr.create",
|
| 178 | run: 2, elapsed_ms: 140.0, returncode: 0, runner_id: "r-1", runner_class: "hosted",
|
| 179 | },
|
| 180 | { scenario: "pr_flow", operation: "pr.create", run: 3, elapsed_ms: 1.0, returncode: 0 },
|
| 181 | ];
|
| 182 | assert.strictEqual(effectiveSubject(rows[0]), "github/cli");
|
| 183 | assert.strictEqual(effectiveSubjectKind(rows[0]), "github");
|
| 184 | assert.strictEqual(effectiveSubject(rows[2]), null);
|
| 185 |
|
| 186 | const { records, idsByKey, skipped } = await upsertSubjects(rows, "run:platform");
|
| 187 | assert.strictEqual(records.length, 1);
|
| 188 | assert.strictEqual(records[0].name, "github/cli");
|
| 189 | assert.strictEqual(records[0].kind, "github");
|
| 190 | assert.strictEqual(skipped, 1);
|
| 191 | const subjectId = idsByKey.get(subjectKey(rows[0]));
|
| 192 | assert.ok(subjectId, "platform identity present in idsByKey");
|
| 193 |
|
| 194 | const report = {};
|
| 195 | const measurements = rowsToMeasurements(rows, "run:platform", NOW, idsByKey, report);
|
| 196 | assert.strictEqual(measurements.length, 2, "identity-less row excluded from measurements");
|
| 197 | assert.strictEqual(report.skipped_no_subject, 1);
|
| 198 | for (const m of measurements) {
|
| 199 | assert.strictEqual(m.subject, "github/cli");
|
| 200 | assert.strictEqual(m.subject_kind, "github");
|
| 201 | assert.strictEqual(m.subject_id, subjectId);
|
| 202 | }
|
| 203 |
|
| 204 | const aggregates = computeAggregates(measurements, "platform", NOW);
|
| 205 | assert.strictEqual(aggregates.length, 1, "both platform rows group under one identity");
|
| 206 | assert.strictEqual(aggregates[0].subject, "github/cli");
|
| 207 | assert.strictEqual(aggregates[0].subject_kind, "github");
|
| 208 | assert.strictEqual(aggregates[0].subject_id, subjectId);
|
| 209 | assert.strictEqual(aggregates[0].n, 2);
|
| 210 | });
|
| 211 |
|
| 212 | test("same subject name with different binary_sha256 stays two identities", async () => {
|
| 213 | const rows = [
|
| 214 | {
|
| 215 | subject: "oak_local", subject_kind: "oak", binary_sha256: "sha-a",
|
| 216 | scenario: "s", operation: "status", run: 1, elapsed_ms: 10.0, returncode: 0,
|
| 217 | },
|
| 218 | {
|
| 219 | subject: "oak_local", subject_kind: "oak", binary_sha256: "sha-b",
|
| 220 | scenario: "s", operation: "status", run: 1, elapsed_ms: 20.0, returncode: 0,
|
| 221 | },
|
| 222 | ];
|
| 223 | const { records, idsByKey } = await upsertSubjects(rows, "run:profile");
|
| 224 | assert.strictEqual(records.length, 2, "two subject records for two shas");
|
| 225 | assert.notStrictEqual(records[0].subject_id, records[1].subject_id);
|
| 226 | assert.deepStrictEqual(records.map((r) => r.name), ["oak_local", "oak_local"], "name stays the subject string");
|
| 227 |
|
| 228 | const measurements = rowsToMeasurements(rows, "run:profile", NOW, idsByKey);
|
| 229 | const aggregates = computeAggregates(measurements, "p", NOW);
|
| 230 | assert.strictEqual(aggregates.length, 2, "two aggregate groups, one per sha");
|
| 231 | assert.deepStrictEqual(
|
| 232 | new Set(aggregates.map((a) => a.subject_id)),
|
| 233 | new Set(records.map((r) => r.subject_id)),
|
| 234 | );
|
| 235 | assert.deepStrictEqual(aggregates.map((a) => a.p50_ms).sort((x, y) => x - y), [10, 20]);
|
| 236 | });
|
| 237 |
|
| 238 | test("comparisons run against every git-kind baseline, order-independent", () => {
|
| 239 | const rows = [
|
| 240 | { subject: "git", subject_kind: "git", scenario: "s", operation: "status", run: 1, elapsed_ms: 100.0, returncode: 0 },
|
| 241 | { subject: "git", subject_kind: "git", scenario: "s", operation: "status", run: 2, elapsed_ms: 110.0, returncode: 0 },
|
| 242 | { subject: "git_fsmonitor", subject_kind: "git", scenario: "s", operation: "status", run: 1, elapsed_ms: 50.0, returncode: 0 },
|
| 243 | { subject: "git_fsmonitor", subject_kind: "git", scenario: "s", operation: "status", run: 2, elapsed_ms: 60.0, returncode: 0 },
|
| 244 | { subject: "oak_local", subject_kind: "oak", scenario: "s", operation: "status", run: 1, elapsed_ms: 40.0, returncode: 0 },
|
| 245 | { subject: "oak_local", subject_kind: "oak", scenario: "s", operation: "status", run: 2, elapsed_ms: 45.0, returncode: 0 },
|
| 246 | ];
|
| 247 | const build = (input) => computeComparisons(computeAggregates(rowsToMeasurements(input, "k:p", NOW), "p", NOW));
|
| 248 |
|
| 249 | const comparisons = build(rows);
|
| 250 | assert.strictEqual(comparisons.length, 2, "one comparison per git-kind baseline");
|
| 251 | assert.ok(comparisons.every((c) => c.target_subject_id === "oak_local"));
|
| 252 | assert.deepStrictEqual(
|
| 253 | comparisons.map((c) => c.baseline_subject_id).sort(),
|
| 254 | ["git", "git_fsmonitor"],
|
| 255 | "distinct baseline ids",
|
| 256 | );
|
| 257 | assert.deepStrictEqual(comparisons.map((c) => c.baseline_subject).sort(), ["git", "git_fsmonitor"]);
|
| 258 |
|
| 259 | const vsGit = comparisons.find((c) => c.baseline_subject_id === "git");
|
| 260 | const vsFsmonitor = comparisons.find((c) => c.baseline_subject_id === "git_fsmonitor");
|
| 261 | assert.strictEqual(vsGit.p50_ratio, 42.5 / 105.0);
|
| 262 | assert.strictEqual(vsFsmonitor.p50_ratio, 42.5 / 55.0);
|
| 263 |
|
| 264 | // Deterministic regardless of input order: reversed and seeded-shuffled
|
| 265 | // permutations produce byte-identical comparison arrays.
|
| 266 | assert.deepStrictEqual(build([...rows].reverse()), comparisons);
|
| 267 | let seed = 1337;
|
| 268 | const next = () => {
|
| 269 | seed = (seed * 1103515245 + 12345) % 2147483648;
|
| 270 | return seed / 2147483648;
|
| 271 | };
|
| 272 | for (let round = 0; round < 5; round += 1) {
|
| 273 | const shuffled = [...rows];
|
| 274 | for (let i = shuffled.length - 1; i > 0; i -= 1) {
|
| 275 | const j = Math.floor(next() * (i + 1));
|
| 276 | [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
| 277 | }
|
| 278 | assert.deepStrictEqual(build(shuffled), comparisons, `shuffle round ${round}`);
|
| 279 | }
|
| 280 | });
|
| 281 |
|
| 282 | test("scorecardFromCells summarizes verdicts with worst-first overall", () => {
|
| 283 | const summary = scorecardFromCells([
|
| 284 | { verdict: "GREEN" },
|
| 285 | { verdict: "AMBER" },
|
| 286 | { verdict: "RED" },
|
| 287 | { verdict: "GREEN" },
|
| 288 | ]);
|
| 289 | assert.deepStrictEqual(summary, {
|
| 290 | total: 4,
|
| 291 | counts: { GREEN: 2, AMBER: 1, RED: 1 },
|
| 292 | overall: "RED",
|
| 293 | });
|
| 294 | assert.strictEqual(scorecardFromCells([]).overall, "UNMEASURED");
|
| 295 | });
|
| 296 |
|
| 297 | test("trendSeries groups aggregate rows per subject in time order", () => {
|
| 298 | const rows = [
|
| 299 | { timestamp_utc: "2026-06-02T00:00:00Z", run_key: "b:p", subject: "git", scenario: "s", operation: "o", p50_ms: 21 },
|
| 300 | { timestamp_utc: "2026-06-01T00:00:00Z", run_key: "a:p", subject: "git", scenario: "s", operation: "o", p50_ms: 20 },
|
| 301 | { timestamp_utc: "2026-06-01T00:00:00Z", run_key: "a:p", subject: "oak_installed", scenario: "s", operation: "o", p50_ms: 11 },
|
| 302 | { timestamp_utc: "2026-06-01T00:00:00Z", run_key: "a:p", subject: "git", scenario: "other", operation: "o", p50_ms: 99 },
|
| 303 | ];
|
| 304 | const series = trendSeries(rows, { scenario: "s", operation: "o", metric: "p50" });
|
| 305 | assert.strictEqual(series.metric, "p50_ms");
|
| 306 | const git = series.series.find((entry) => entry.subject === "git");
|
| 307 | assert.deepStrictEqual(git.points.map((point) => point.value), [20, 21]);
|
| 308 | assert.strictEqual(trendSeries(rows, { scenario: "s", operation: "o", metric: "bogus" }), null);
|
| 309 | });
|
| 310 |
|
| 311 | test("aggregate tail gating matches golden: p95 null below n=20, p99 null below n=100", () => {
|
| 312 | // Parity note: Python gates tail percentiles in
|
| 313 | // oakbench.reporting.tail_latency_summary (and at the rendering layer in
|
| 314 | // scripts/benchmark_stats.py) — there is no shared Python aggregate
|
| 315 | // function identical to computeAggregates. tests/test_reporting_golden.py
|
| 316 | // asserts these same golden cases against tail_latency_summary, so the
|
| 317 | // gate thresholds and gated values cannot drift between the two sides.
|
| 318 | assert.strictEqual(TAIL_MIN_SAMPLES.p95, 20);
|
| 319 | assert.strictEqual(TAIL_MIN_SAMPLES.p99, 100);
|
| 320 | const sizes = GOLDEN.aggregate_tail_gating.cases.map((c) => c.values.length);
|
| 321 | assert.deepStrictEqual(sizes, [1, 20, 100], "fixture covers n=1 and both gate boundaries");
|
| 322 |
|
| 323 | for (const { values, expected } of GOLDEN.aggregate_tail_gating.cases) {
|
| 324 | const rows = values.map((elapsed, index) => ({
|
| 325 | subject: "oak_installed", subject_kind: "oak", scenario: "tail", operation: "status",
|
| 326 | run: index + 1, elapsed_ms: elapsed, returncode: 0,
|
| 327 | }));
|
| 328 | const aggregates = computeAggregates(rowsToMeasurements(rows, "k:p", NOW), "p", NOW);
|
| 329 | assert.strictEqual(aggregates.length, 1);
|
| 330 | const agg = aggregates[0];
|
| 331 | assert.strictEqual(agg.n, values.length);
|
| 332 | assert.strictEqual(agg.p50_ms, expected.p50, `p50 stays ungated at n=${values.length}`);
|
| 333 | assert.notStrictEqual(agg.p90_ms, null, `p90 stays ungated at n=${values.length}`);
|
| 334 | assert.strictEqual(agg.p95_ms, expected.p95, `p95 at n=${values.length}`);
|
| 335 | assert.strictEqual(agg.p99_ms, expected.p99, `p99 at n=${values.length}`);
|
| 336 | }
|
| 337 | });
|
| 338 |
|
| 339 | test("p90-only regression flags is_regression and regression_dimensions", () => {
|
| 340 | const makeAgg = (subjectId, kind, p50, p90) => ({
|
| 341 | run_key: "k:p", subject_id: subjectId, subject: subjectId, subject_kind: kind,
|
| 342 | profile: "p", scenario: "s", operation: "status", is_canonical: 1,
|
| 343 | sample_count: 5, failure_count: 0, failure_rate: 0, n: 5,
|
| 344 | p50_ms: p50, p90_ms: p90, p95_ms: null, p99_ms: null,
|
| 345 | mean_ms: p50, min_ms: p50, max_ms: p90,
|
| 346 | avg_tokens: null, avg_output_bytes: null, truncated_count: 0, avg_tool_calls: null,
|
| 347 | updated_at: NOW,
|
| 348 | });
|
| 349 | const compare = (basP50, basP90, tgtP50, tgtP90) => {
|
| 350 | const comparisons = computeComparisons([
|
| 351 | makeAgg("git", "git", basP50, basP90),
|
| 352 | makeAgg("oak_installed", "oak", tgtP50, tgtP90),
|
| 353 | ]);
|
| 354 | assert.strictEqual(comparisons.length, 1);
|
| 355 | return comparisons[0];
|
| 356 | };
|
| 357 |
|
| 358 | // Reviewer repro: p50 flat (10 -> 10) but p90 blows up (10 -> 100).
|
| 359 | const p90Only = compare(10, 10, 10, 100);
|
| 360 | assert.strictEqual(p90Only.is_regression, 1);
|
| 361 | assert.deepStrictEqual(p90Only.regression_dimensions, ["p90"]);
|
| 362 | assert.strictEqual(p90Only.state, "regression");
|
| 363 | assert.strictEqual(p90Only.severity_score, 900, "severity follows the breaching dimension");
|
| 364 |
|
| 365 | // p50-only breach: p90 delta (1ms) stays under minDeltaMs (5ms).
|
| 366 | const p50Only = compare(10, 100, 100, 101);
|
| 367 | assert.strictEqual(p50Only.is_regression, 1);
|
| 368 | assert.deepStrictEqual(p50Only.regression_dimensions, ["p50"]);
|
| 369 |
|
| 370 | // Neither dimension breaches (4% slower, under the 10% threshold).
|
| 371 | const clean = compare(10, 10, 10.4, 10.4);
|
| 372 | assert.strictEqual(clean.is_regression, 0);
|
| 373 | assert.deepStrictEqual(clean.regression_dimensions, []);
|
| 374 | assert.strictEqual(clean.state, "ok");
|
| 375 | assert.strictEqual(clean.severity_score, 0);
|
| 376 | });
|
| 377 |
|
| 378 | // Minimal D1 stub for driving ingestRows: prepare(...).bind(...) statements
|
| 379 | // carry their SQL + bound args; batch()/run() route to an in-memory table
|
| 380 | // keyed by run_key (args[0] for measurements/aggregates/comparisons inserts,
|
| 381 | // args[1] for subjects whose first bind is subject_id).
|
| 382 | class StubD1 {
|
| 383 | constructor() {
|
| 384 | this.tables = { subjects: [], measurements: [], aggregates: [], comparisons: [] };
|
| 385 | this.failInsertsForTable = null;
|
| 386 | this.batchCalls = [];
|
| 387 | }
|
| 388 |
|
| 389 | prepare(sql) {
|
| 390 | const stub = this;
|
| 391 | return {
|
| 392 | bind(...args) {
|
| 393 | return {
|
| 394 | sql,
|
| 395 | args,
|
| 396 | run: async () => stub.execute(sql, args),
|
| 397 | first: async () => null,
|
| 398 | };
|
| 399 | },
|
| 400 | };
|
| 401 | }
|
| 402 |
|
| 403 | async batch(statements) {
|
| 404 | this.batchCalls.push(statements);
|
| 405 | const snapshot = Object.fromEntries(
|
| 406 | Object.entries(this.tables).map(([table, rows]) => [
|
| 407 | table,
|
| 408 | rows.map((row) => ({ runKey: row.runKey, args: [...row.args] })),
|
| 409 | ]),
|
| 410 | );
|
| 411 | try {
|
| 412 | for (const statement of statements) {
|
| 413 | this.execute(statement.sql, statement.args ?? []);
|
| 414 | }
|
| 415 | } catch (error) {
|
| 416 | this.tables = snapshot;
|
| 417 | throw error;
|
| 418 | }
|
| 419 | }
|
| 420 |
|
| 421 | execute(sql, args) {
|
| 422 | const del = sql.match(/DELETE FROM (\w+) WHERE run_key = \?/);
|
| 423 | if (del) {
|
| 424 | this.tables[del[1]] = this.tables[del[1]].filter((row) => row.runKey !== args[0]);
|
| 425 | return { success: true };
|
| 426 | }
|
| 427 | const ins = sql.match(/INSERT INTO (\w+)/);
|
| 428 | if (ins) {
|
| 429 | const table = ins[1];
|
| 430 | if (this.failInsertsForTable === table) {
|
| 431 | throw new Error(`injected insert failure for ${table}`);
|
| 432 | }
|
| 433 | const runKey = table === "subjects" ? args[1] : args[0];
|
| 434 | if (table === "subjects" && sql.includes("ON CONFLICT(subject_id)")) {
|
| 435 | this.tables.subjects = this.tables.subjects.filter((row) => row.args[0] !== args[0]);
|
| 436 | }
|
| 437 | this.tables[table].push({ runKey, args });
|
| 438 | return { success: true };
|
| 439 | }
|
| 440 | throw new Error(`StubD1: unsupported SQL: ${sql}`);
|
| 441 | }
|
| 442 | }
|
| 443 |
|
| 444 | test("re-upload with fewer groups leaves no stale aggregates/comparisons", async () => {
|
| 445 | const groupRows = (operations) => operations.flatMap((operation) => [
|
| 446 | { subject: "git", subject_kind: "git", scenario: "s", operation, run: 1, elapsed_ms: 10.0, returncode: 0 },
|
| 447 | { subject: "oak_installed", subject_kind: "oak", scenario: "s", operation, run: 1, elapsed_ms: 12.0, returncode: 0 },
|
| 448 | ]);
|
| 449 | const db = new StubD1();
|
| 450 |
|
| 451 | // A different run_key seeded first must survive the re-upload deletes.
|
| 452 | await ingestRows(db, groupRows(["keep"]), "other:p", "p", NOW);
|
| 453 |
|
| 454 | const first = await ingestRows(db, groupRows(["o1", "o2", "o3"]), "rk:p", "p", NOW);
|
| 455 | assert.strictEqual(first.aggregates, 6, "3 groups x 2 subjects");
|
| 456 | assert.strictEqual(first.comparisons, 3);
|
| 457 | assert.strictEqual(db.tables.aggregates.filter((r) => r.runKey === "rk:p").length, 6);
|
| 458 | assert.strictEqual(db.tables.comparisons.filter((r) => r.runKey === "rk:p").length, 3);
|
| 459 |
|
| 460 | // Re-upload the SAME run_key with only 1 of the 3 groups.
|
| 461 | await ingestRows(db, groupRows(["o1"]), "rk:p", "p", NOW);
|
| 462 | const aggs = db.tables.aggregates.filter((r) => r.runKey === "rk:p");
|
| 463 | const comps = db.tables.comparisons.filter((r) => r.runKey === "rk:p");
|
| 464 | assert.strictEqual(aggs.length, 2, "exactly the re-uploaded group's aggregates");
|
| 465 | assert.strictEqual(comps.length, 1, "exactly the re-uploaded group's comparison");
|
| 466 | // Aggregates bind order: run_key, subject_id, profile, scenario, operation.
|
| 467 | assert.deepStrictEqual(new Set(aggs.map((r) => r.args[4])), new Set(["o1"]));
|
| 468 | // Comparisons bind order: ..., profile, scenario, operation at index 6.
|
| 469 | assert.deepStrictEqual(comps.map((r) => r.args[6]), ["o1"]);
|
| 470 | assert.strictEqual(db.tables.measurements.filter((r) => r.runKey === "rk:p").length, 2);
|
| 471 |
|
| 472 | // Other run untouched.
|
| 473 | assert.strictEqual(db.tables.aggregates.filter((r) => r.runKey === "other:p").length, 2);
|
| 474 | assert.strictEqual(db.tables.comparisons.filter((r) => r.runKey === "other:p").length, 1);
|
| 475 | });
|
| 476 |
|
| 477 | test("failed run rewrite leaves previous run rows intact", async () => {
|
| 478 | const groupRows = (operations) => operations.flatMap((operation) => [
|
| 479 | { subject: "git", subject_kind: "git", scenario: "s", operation, run: 1, elapsed_ms: 10.0, returncode: 0 },
|
| 480 | { subject: "oak_installed", subject_kind: "oak", scenario: "s", operation, run: 1, elapsed_ms: 12.0, returncode: 0 },
|
| 481 | ]);
|
| 482 | const db = new StubD1();
|
| 483 | await ingestRows(db, groupRows(["old"]), "rk:p", "p", NOW);
|
| 484 |
|
| 485 | db.failInsertsForTable = "measurements";
|
| 486 | await assert.rejects(
|
| 487 | () => ingestRows(db, groupRows(["new"]), "rk:p", "p", NOW),
|
| 488 | /injected insert failure for measurements/,
|
| 489 | );
|
| 490 |
|
| 491 | const measurements = db.tables.measurements.filter((r) => r.runKey === "rk:p");
|
| 492 | const aggs = db.tables.aggregates.filter((r) => r.runKey === "rk:p");
|
| 493 | const comps = db.tables.comparisons.filter((r) => r.runKey === "rk:p");
|
| 494 | assert.strictEqual(measurements.length, 2);
|
| 495 | assert.strictEqual(aggs.length, 2);
|
| 496 | assert.strictEqual(comps.length, 1);
|
| 497 | assert.deepStrictEqual(new Set(measurements.map((r) => r.args[3])), new Set(["old"]));
|
| 498 | assert.deepStrictEqual(new Set(aggs.map((r) => r.args[4])), new Set(["old"]));
|
| 499 | assert.deepStrictEqual(new Set(comps.map((r) => r.args[6])), new Set(["old"]));
|
| 500 | });
|
| 501 |
|
| 502 | test("regression_dimensions is persisted through the comparisons insert", async () => {
|
| 503 | // p90-only regression: oak medians match git (10), but oak's p90 hits the
|
| 504 | // 100ms outlier (nearest-rank p90 of 3 samples = the 3rd).
|
| 505 | const rows = [
|
| 506 | ...[1, 2, 3].map((run) => ({
|
| 507 | subject: "git", subject_kind: "git", scenario: "s", operation: "o", run,
|
| 508 | elapsed_ms: 10.0, returncode: 0,
|
| 509 | })),
|
| 510 | ...[10.0, 10.0, 100.0].map((elapsed, i) => ({
|
| 511 | subject: "oak_installed", subject_kind: "oak", scenario: "s", operation: "o", run: i + 1,
|
| 512 | elapsed_ms: elapsed, returncode: 0,
|
| 513 | })),
|
| 514 | ];
|
| 515 | const db = new StubD1();
|
| 516 | await ingestRows(db, rows, "rk:dims", "p", NOW);
|
| 517 |
|
| 518 | const comps = db.tables.comparisons.filter((r) => r.runKey === "rk:dims");
|
| 519 | assert.strictEqual(comps.length, 1);
|
| 520 | const insertSql = `run_key, target_subject_id, baseline_subject_id, baseline_kind, profile,
|
| 521 | scenario, operation,
|
| 522 | p50_delta_ms, p50_slower_pct, p90_delta_ms, p90_slower_pct,
|
| 523 | p99_delta_ms, p99_slower_pct,
|
| 524 | avg_token_delta, token_higher_pct, avg_output_byte_delta, output_byte_higher_pct,
|
| 525 | avg_tool_call_delta, tool_call_higher_pct, failure_rate_delta_pp,
|
| 526 | threshold_pct, min_delta_ms, is_regression, regression_dimensions, state, severity_score, updated_at`;
|
| 527 | const columns = insertSql.replace(/\s+/g, " ").split(", ").map((c) => c.trim());
|
| 528 | const dimsIndex = columns.indexOf("regression_dimensions");
|
| 529 | assert.ok(dimsIndex >= 0, "insert must include the regression_dimensions column");
|
| 530 | const persisted = comps[0].args[dimsIndex];
|
| 531 | assert.strictEqual(typeof persisted, "string", "dimensions persist as a JSON string");
|
| 532 | assert.deepStrictEqual(JSON.parse(persisted), ["p90"]);
|
| 533 | const isRegressionIndex = columns.indexOf("is_regression");
|
| 534 | assert.strictEqual(comps[0].args[isRegressionIndex], 1);
|
| 535 | });
|
| 536 |
|
| 537 | test("verifySignature accepts a valid X-Hub-Signature-256 and rejects tampering", async () => {
|
| 538 | const secret = "shhh-bench";
|
| 539 | const body = JSON.stringify({ action: "push", ref: "refs/heads/main" });
|
| 540 | const header = `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`;
|
| 541 | assert.strictEqual(await verifySignature(secret, body, header), true);
|
| 542 | assert.strictEqual(await verifySignature(secret, body + "x", header), false);
|
| 543 | assert.strictEqual(await verifySignature("wrong", body, header), false);
|
| 544 | assert.strictEqual(await verifySignature(secret, body, "sha256=deadbeef"), false);
|
| 545 | assert.strictEqual(await verifySignature(secret, body, ""), false);
|
| 546 | });
|
| 547 |
|
| 548 | test("write routes fail closed without UPLOAD_TOKEN but GitHub webhook does not require it", async () => {
|
| 549 | const writeResponse = await worker.fetch(
|
| 550 | new Request("https://bench.example/api/scorecard", {
|
| 551 | method: "POST",
|
| 552 | body: JSON.stringify({ bench_id: "b", profile: "p", scorecard: [] }),
|
| 553 | }),
|
| 554 | {},
|
| 555 | );
|
| 556 | assert.strictEqual(writeResponse.status, 503);
|
| 557 | assert.deepStrictEqual(await writeResponse.json(), {
|
| 558 | ok: false,
|
| 559 | error: "upload_token_not_configured",
|
| 560 | });
|
| 561 |
|
| 562 | const secret = "webhook-secret";
|
| 563 | const body = JSON.stringify({ action: "push" });
|
| 564 | const signature = `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`;
|
| 565 | const webhookResponse = await worker.fetch(
|
| 566 | new Request("https://bench.example/webhook/github", {
|
| 567 | method: "POST",
|
| 568 | body,
|
| 569 | headers: {
|
| 570 | "x-hub-signature-256": signature,
|
| 571 | "x-github-event": "push",
|
| 572 | "x-github-delivery": "delivery-1",
|
| 573 | },
|
| 574 | }),
|
| 575 | { GITHUB_WEBHOOK_SECRET: secret },
|
| 576 | );
|
| 577 | assert.strictEqual(webhookResponse.status, 200);
|
| 578 | assert.deepStrictEqual(await webhookResponse.json(), { ok: true, recorded: false });
|
| 579 | });
|
| 580 |
|
| 581 | test("upload limit is enforced on UTF-8 bytes, not UTF-16 string length", async () => {
|
| 582 | const body = "é".repeat(Math.floor(MAX_UPLOAD_BYTES / 2) + 1);
|
| 583 | assert.ok(body.length <= MAX_UPLOAD_BYTES, "fixture must be below the old UTF-16 length guard");
|
| 584 | assert.ok(new TextEncoder().encode(body).byteLength > MAX_UPLOAD_BYTES);
|
| 585 |
|
| 586 | const response = await worker.fetch(
|
| 587 | new Request("https://bench.example/api/upload", {
|
| 588 | method: "POST",
|
| 589 | body,
|
| 590 | headers: {
|
| 591 | authorization: "Bearer upload-token",
|
| 592 | "content-length": String(body.length),
|
| 593 | },
|
| 594 | }),
|
| 595 | { UPLOAD_TOKEN: "upload-token" },
|
| 596 | );
|
| 597 |
|
| 598 | assert.strictEqual(response.status, 413);
|
| 599 | assert.deepStrictEqual(await response.json(), {
|
| 600 | ok: false,
|
| 601 | error: "upload_too_large",
|
| 602 | max_bytes: MAX_UPLOAD_BYTES,
|
| 603 | });
|
| 604 | });
|