tests/fixtures/reporting_golden.json · oak/benchmarks

Viewing files at commit 62758990fe31Commit details View latest

Browse files

tests/fixtures/reporting_golden.json 289 lines · 6.1 KB · json

{
  "_comment": "Python/JS reporting parity contract. Expected values were computed by running the REAL Python functions (oakbench.stats.percentile_nearest, oakbench.reporting.delta_pct/median). delta_pct with base==0 returns 0.0 by definition (reporting.delta_pct guard); percentile of an empty list is null; ratio uses target/baseline with median statistics.",
  "percentile_nearest": [
    {
      "values": [],
      "p": 50,
      "expected": null
    },
    {
      "values": [
        7.5
      ],
      "p": 99,
      "expected": 7.5
    },
    {
      "values": [
        1,
        2,
        3,
        4
      ],
      "p": 50,
      "expected": 2.0
    },
    {
      "values": [
        1,
        2,
        3,
        4
      ],
      "p": 90,
      "expected": 4.0
    },
    {
      "values": [
        1,
        2,
        3,
        4
      ],
      "p": 95,
      "expected": 4.0
    },
    {
      "values": [
        5,
        5,
        5,
        5,
        5
      ],
      "p": 90,
      "expected": 5.0
    },
    {
      "values": [
        10,
        20,
        30,
        40,
        50,
        60,
        70,
        80,
        90,
        100
      ],
      "p": 90,
      "expected": 90.0
    },
    {
      "values": [
        10,
        20,
        30,
        40,
        50,
        60,
        70,
        80,
        90,
        100
      ],
      "p": 99,
      "expected": 100.0
    },
    {
      "values": [
        3.5,
        1.25,
        2.75
      ],
      "p": 50,
      "expected": 2.75
    },
    {
      "values": [
        0.0,
        0.0,
        0.0
      ],
      "p": 99,
      "expected": 0.0
    }
  ],
  "delta_pct": [
    {
      "base": 0,
      "value": 123.4,
      "expected": 0.0
    },
    {
      "base": 0.0,
      "value": 0.0,
      "expected": 0.0
    },
    {
      "base": 100.0,
      "value": 80.0,
      "expected": 20.0
    },
    {
      "base": 100.0,
      "value": 125.0,
      "expected": -25.0
    },
    {
      "base": 50.0,
      "value": 50.0,
      "expected": 0.0
    },
    {
      "base": 200.0,
      "value": 0.0,
      "expected": 100.0
    },
    {
      "base": 12.5,
      "value": 11.0,
      "expected": 12.0
    }
  ],
  "aggregate_group": {
    "rows": [
      {
        "subject": "oak_installed",
        "subject_kind": "oak",
        "scenario": "synthetic",
        "operation": "status",
        "run": 1,
        "elapsed_ms": 12.0,
        "returncode": 0,
        "estimated_tokens_total": 100
      },
      {
        "subject": "oak_installed",
        "subject_kind": "oak",
        "scenario": "synthetic",
        "operation": "status",
        "run": 2,
        "elapsed_ms": 15.5,
        "returncode": 0,
        "estimated_tokens_total": 110
      },
      {
        "subject": "oak_installed",
        "subject_kind": "oak",
        "scenario": "synthetic",
        "operation": "status",
        "run": 3,
        "elapsed_ms": 11.0,
        "returncode": 0,
        "estimated_tokens_total": null
      },
      {
        "subject": "oak_installed",
        "subject_kind": "oak",
        "scenario": "synthetic",
        "operation": "status",
        "run": 4,
        "elapsed_ms": 30.0,
        "returncode": 0,
        "estimated_tokens_total": 120
      },
      {
        "subject": "oak_installed",
        "subject_kind": "oak",
        "scenario": "synthetic",
        "operation": "status",
        "run": 5,
        "elapsed_ms": 14.0,
        "returncode": 0,
        "estimated_tokens_total": 90
      },
      {
        "subject": "oak_installed",
        "subject_kind": "oak",
        "scenario": "synthetic",
        "operation": "status",
        "run": 6,
        "elapsed_ms": null,
        "returncode": 0,
        "estimated_tokens_total": 105
      },
      {
        "subject": "oak_installed",
        "subject_kind": "oak",
        "scenario": "synthetic",
        "operation": "status",
        "run": 7,
        "elapsed_ms": 99.0,
        "returncode": 1,
        "estimated_tokens_total": 500
      },
      {
        "subject": "oak_installed",
        "subject_kind": "oak",
        "scenario": "synthetic",
        "operation": "status",
        "run": 8,
        "elapsed_ms": null,
        "returncode": 77,
        "skipped": true,
        "skip_reason": "synthetic skip"
      }
    ],
    "expected": {
      "median": 14.0,
      "p90": 30.0,
      "p95": 30.0,
      "p99": 30.0,
      "n": 5
    }
  },
  "aggregate_tail_gating": {
    "_comment": "Tail honesty parity (oakbench.reporting.TAIL_PERCENTILE_MIN_SAMPLES / benchmark_stats TAIL_MIN_SAMPLES: p95 needs n>=20, p99 needs n>=100; below that the nearest-rank percentile is just the max relabelled). Python asserts via oakbench.reporting.tail_latency_summary; JS asserts via computeAggregates. null means unmeasured, never the lone sample. p50 (and p90 on the JS aggregate) stay ungated.",
    "cases": [
      {
        "values": [7.5],
        "expected": {
          "p50": 7.5,
          "p95": null,
          "p99": null,
          "tail_supported": []
        }
      },
      {
        "values": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
        "expected": {
          "p50": 10.5,
          "p95": 19.0,
          "p99": null,
          "tail_supported": ["p95"]
        }
      },
      {
        "values": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100],
        "expected": {
          "p50": 50.5,
          "p95": 95.0,
          "p99": 99.0,
          "tail_supported": ["p95", "p99"]
        }
      }
    ]
  },
  "comparison": {
    "oak_values": [
      12.0,
      15.5,
      11.0,
      30.0,
      14.0
    ],
    "git_values": [
      20.0,
      26.0,
      19.0,
      41.0,
      24.0
    ],
    "expected_oak_median": 14.0,
    "expected_git_median": 24.0,
    "expected_ratio": 0.5833333333333334,
    "expected_delta_pct": 41.66666666666667
  }
}

About

Reversible clean-root candidate rebuilt from oak/benchmarks main@009dead264ca; original repository retained.

Download

RunnersDetails

No runs on main yet.

Open branchesHistory

mrmrs-516758agent_loop_hosted_v1: hosted agent-loop gap map lane (Oak vs GitHub vs ideal)checking…6 behind1mo ago

mrmrs-b26e28Add stale_branch_land_v1: baseline for landing branches that are behind mainchecking…6 behind1mo ago

mrmrs-fad256task_loop: shallow_worker_branch_refresh_v1 (fresh shallow worker -> task branch -> first parent refresh, Oak vs Git, HTTP request counts)checking…6 behind1mo ago

oakmark-throughput-benchmark-transplantAdd oakmark: per-repo push/clone throughput benchmark for oak (ForgeMark analogue)checking…11 behind3mo ago

All branches →

Activity12 commits to main

May 2026now

1	`{`
2	`"_comment": "Python/JS reporting parity contract. Expected values were computed by running the REAL Python functions (oakbench.stats.percentile_nearest, oakbench.reporting.delta_pct/median). delta_pct with base==0 returns 0.0 by definition (reporting.delta_pct guard); percentile of an empty list is null; ratio uses target/baseline with median statistics.",`
3	`"percentile_nearest": [`
4	`{`
5	`"values": [],`
6	`"p": 50,`
7	`"expected": null`
8	`},`
9	`{`
10	`"values": [`
11	`7.5`
12	`],`
13	`"p": 99,`
14	`"expected": 7.5`
15	`},`
16	`{`
17	`"values": [`
18	`1,`
19	`2,`
20	`3,`
21	`4`
22	`],`
23	`"p": 50,`
24	`"expected": 2.0`
25	`},`
26	`{`
27	`"values": [`
28	`1,`
29	`2,`
30	`3,`
31	`4`
32	`],`
33	`"p": 90,`
34	`"expected": 4.0`
35	`},`
36	`{`
37	`"values": [`
38	`1,`
39	`2,`
40	`3,`
41	`4`
42	`],`
43	`"p": 95,`
44	`"expected": 4.0`
45	`},`
46	`{`
47	`"values": [`
48	`5,`
49	`5,`
50	`5,`
51	`5,`
52	`5`
53	`],`
54	`"p": 90,`
55	`"expected": 5.0`
56	`},`
57	`{`
58	`"values": [`
59	`10,`
60	`20,`
61	`30,`
62	`40,`
63	`50,`
64	`60,`
65	`70,`
66	`80,`
67	`90,`
68	`100`
69	`],`
70	`"p": 90,`
71	`"expected": 90.0`
72	`},`
73	`{`
74	`"values": [`
75	`10,`
76	`20,`
77	`30,`
78	`40,`
79	`50,`
80	`60,`
81	`70,`
82	`80,`
83	`90,`
84	`100`
85	`],`
86	`"p": 99,`
87	`"expected": 100.0`
88	`},`
89	`{`
90	`"values": [`
91	`3.5,`
92	`1.25,`
93	`2.75`
94	`],`
95	`"p": 50,`
96	`"expected": 2.75`
97	`},`
98	`{`
99	`"values": [`
100	`0.0,`
101	`0.0,`
102	`0.0`
103	`],`
104	`"p": 99,`
105	`"expected": 0.0`
106	`}`
107	`],`
108	`"delta_pct": [`
109	`{`
110	`"base": 0,`
111	`"value": 123.4,`
112	`"expected": 0.0`
113	`},`
114	`{`
115	`"base": 0.0,`
116	`"value": 0.0,`
117	`"expected": 0.0`
118	`},`
119	`{`
120	`"base": 100.0,`
121	`"value": 80.0,`
122	`"expected": 20.0`
123	`},`
124	`{`
125	`"base": 100.0,`
126	`"value": 125.0,`
127	`"expected": -25.0`
128	`},`
129	`{`
130	`"base": 50.0,`
131	`"value": 50.0,`
132	`"expected": 0.0`
133	`},`
134	`{`
135	`"base": 200.0,`
136	`"value": 0.0,`
137	`"expected": 100.0`
138	`},`
139	`{`
140	`"base": 12.5,`
141	`"value": 11.0,`
142	`"expected": 12.0`
143	`}`
144	`],`
145	`"aggregate_group": {`
146	`"rows": [`
147	`{`
148	`"subject": "oak_installed",`
149	`"subject_kind": "oak",`
150	`"scenario": "synthetic",`
151	`"operation": "status",`
152	`"run": 1,`
153	`"elapsed_ms": 12.0,`
154	`"returncode": 0,`
155	`"estimated_tokens_total": 100`
156	`},`
157	`{`
158	`"subject": "oak_installed",`
159	`"subject_kind": "oak",`
160	`"scenario": "synthetic",`
161	`"operation": "status",`
162	`"run": 2,`
163	`"elapsed_ms": 15.5,`
164	`"returncode": 0,`
165	`"estimated_tokens_total": 110`
166	`},`
167	`{`
168	`"subject": "oak_installed",`
169	`"subject_kind": "oak",`
170	`"scenario": "synthetic",`
171	`"operation": "status",`
172	`"run": 3,`
173	`"elapsed_ms": 11.0,`
174	`"returncode": 0,`
175	`"estimated_tokens_total": null`
176	`},`
177	`{`
178	`"subject": "oak_installed",`
179	`"subject_kind": "oak",`
180	`"scenario": "synthetic",`
181	`"operation": "status",`
182	`"run": 4,`
183	`"elapsed_ms": 30.0,`
184	`"returncode": 0,`
185	`"estimated_tokens_total": 120`
186	`},`
187	`{`
188	`"subject": "oak_installed",`
189	`"subject_kind": "oak",`
190	`"scenario": "synthetic",`
191	`"operation": "status",`
192	`"run": 5,`
193	`"elapsed_ms": 14.0,`
194	`"returncode": 0,`
195	`"estimated_tokens_total": 90`
196	`},`
197	`{`
198	`"subject": "oak_installed",`
199	`"subject_kind": "oak",`
200	`"scenario": "synthetic",`
201	`"operation": "status",`
202	`"run": 6,`
203	`"elapsed_ms": null,`
204	`"returncode": 0,`
205	`"estimated_tokens_total": 105`
206	`},`
207	`{`
208	`"subject": "oak_installed",`
209	`"subject_kind": "oak",`
210	`"scenario": "synthetic",`
211	`"operation": "status",`
212	`"run": 7,`
213	`"elapsed_ms": 99.0,`
214	`"returncode": 1,`
215	`"estimated_tokens_total": 500`
216	`},`
217	`{`
218	`"subject": "oak_installed",`
219	`"subject_kind": "oak",`
220	`"scenario": "synthetic",`
221	`"operation": "status",`
222	`"run": 8,`
223	`"elapsed_ms": null,`
224	`"returncode": 77,`
225	`"skipped": true,`
226	`"skip_reason": "synthetic skip"`
227	`}`
228	`],`
229	`"expected": {`
230	`"median": 14.0,`
231	`"p90": 30.0,`
232	`"p95": 30.0,`
233	`"p99": 30.0,`
234	`"n": 5`
235	`}`
236	`},`
237	`"aggregate_tail_gating": {`
238	`"_comment": "Tail honesty parity (oakbench.reporting.TAIL_PERCENTILE_MIN_SAMPLES / benchmark_stats TAIL_MIN_SAMPLES: p95 needs n>=20, p99 needs n>=100; below that the nearest-rank percentile is just the max relabelled). Python asserts via oakbench.reporting.tail_latency_summary; JS asserts via computeAggregates. null means unmeasured, never the lone sample. p50 (and p90 on the JS aggregate) stay ungated.",`
239	`"cases": [`
240	`{`
241	`"values": [7.5],`
242	`"expected": {`
243	`"p50": 7.5,`
244	`"p95": null,`
245	`"p99": null,`
246	`"tail_supported": []`
247	`}`
248	`},`
249	`{`
250	`"values": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20],`
251	`"expected": {`
252	`"p50": 10.5,`
253	`"p95": 19.0,`
254	`"p99": null,`
255	`"tail_supported": ["p95"]`
256	`}`
257	`},`
258	`{`
259	`"values": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100],`
260	`"expected": {`
261	`"p50": 50.5,`
262	`"p95": 95.0,`
263	`"p99": 99.0,`
264	`"tail_supported": ["p95", "p99"]`
265	`}`
266	`}`
267	`]`
268	`},`
269	`"comparison": {`
270	`"oak_values": [`
271	`12.0,`
272	`15.5,`
273	`11.0,`
274	`30.0,`
275	`14.0`
276	`],`
277	`"git_values": [`
278	`20.0,`
279	`26.0,`
280	`19.0,`
281	`41.0,`
282	`24.0`
283	`],`
284	`"expected_oak_median": 14.0,`
285	`"expected_git_median": 24.0,`
286	`"expected_ratio": 0.5833333333333334,`
287	`"expected_delta_pct": 41.66666666666667`
288	`}`
289	`}`