| 1 | """Tests for the opportunity report: ranking, honesty rules, missing inputs."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import contextlib
|
| 6 | import io
|
| 7 | import json
|
| 8 | import sys
|
| 9 | import tempfile
|
| 10 | import unittest
|
| 11 | from pathlib import Path
|
| 12 | from typing import Any, Optional
|
| 13 |
|
| 14 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 15 |
|
| 16 | import opportunities
|
| 17 |
|
| 18 |
|
| 19 | def core_row(
|
| 20 | subject: str,
|
| 21 | operation: str,
|
| 22 | *,
|
| 23 | scenario: str = "tiny_text",
|
| 24 | returncode: int = 0,
|
| 25 | elapsed_ms: float = 10.0,
|
| 26 | tokens: Optional[float] = None,
|
| 27 | tokens_with_envelope: Optional[float] = None,
|
| 28 | output_bytes: Optional[int] = None,
|
| 29 | tool_calls: Optional[int] = None,
|
| 30 | **extra: Any,
|
| 31 | ) -> dict[str, Any]:
|
| 32 | row: dict[str, Any] = {
|
| 33 | "subject": subject,
|
| 34 | "subject_kind": "oak" if subject.startswith("oak") else "git",
|
| 35 | "scenario": scenario,
|
| 36 | "operation": operation,
|
| 37 | "returncode": returncode,
|
| 38 | "elapsed_ms": elapsed_ms,
|
| 39 | "run": 0,
|
| 40 | }
|
| 41 | if tokens is not None:
|
| 42 | row["estimated_cost_weighted_tokens"] = tokens
|
| 43 | if tokens_with_envelope is not None:
|
| 44 | row["estimated_cost_weighted_tokens_with_envelope"] = tokens_with_envelope
|
| 45 | if output_bytes is not None:
|
| 46 | row["raw_output_bytes"] = output_bytes
|
| 47 | if tool_calls is not None:
|
| 48 | row["tool_call_count"] = tool_calls
|
| 49 | row.update(extra)
|
| 50 | return row
|
| 51 |
|
| 52 |
|
| 53 | def skip_row(subject: str, operation: str, reason: str, scenario: str = "tiny_text") -> dict[str, Any]:
|
| 54 | return {
|
| 55 | "subject": subject,
|
| 56 | "subject_kind": "oak" if subject.startswith("oak") else "git",
|
| 57 | "scenario": scenario,
|
| 58 | "operation": operation,
|
| 59 | "returncode": 77,
|
| 60 | "skipped": True,
|
| 61 | "skip_reason": reason,
|
| 62 | "elapsed_ms": 0.0,
|
| 63 | "run": 0,
|
| 64 | }
|
| 65 |
|
| 66 |
|
| 67 | def write_results(
|
| 68 | results_dir: Path,
|
| 69 | core: Optional[list[dict[str, Any]]] = None,
|
| 70 | mount: Optional[list[dict[str, Any]]] = None,
|
| 71 | contention: Optional[list[dict[str, Any]]] = None,
|
| 72 | ) -> None:
|
| 73 | results_dir.mkdir(parents=True, exist_ok=True)
|
| 74 | if core is not None:
|
| 75 | write_jsonl(results_dir / "latest.jsonl", core)
|
| 76 | if mount is not None:
|
| 77 | write_jsonl(results_dir / "latest.mount.jsonl", mount)
|
| 78 | if contention is not None:
|
| 79 | contention_dir = results_dir / "parallel-contention"
|
| 80 | contention_dir.mkdir(parents=True, exist_ok=True)
|
| 81 | write_jsonl(contention_dir / "latest.jsonl", contention)
|
| 82 |
|
| 83 |
|
| 84 | def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
| 85 | with path.open("w") as fh:
|
| 86 | for row in rows:
|
| 87 | fh.write(json.dumps(row) + "\n")
|
| 88 |
|
| 89 |
|
| 90 | def entries_for(entries: list[dict[str, Any]], section: str) -> list[dict[str, Any]]:
|
| 91 | return [entry for entry in entries if entry["section"] == section]
|
| 92 |
|
| 93 |
|
| 94 | class OpportunitiesTest(unittest.TestCase):
|
| 95 | def setUp(self) -> None:
|
| 96 | self._tmp = tempfile.TemporaryDirectory()
|
| 97 | self.addCleanup(self._tmp.cleanup)
|
| 98 | self.results = Path(self._tmp.name) / "results"
|
| 99 |
|
| 100 | def test_oak_worse_on_tokens_is_ranked_by_score(self) -> None:
|
| 101 | # Same +100% gap on both ops, but status.dirty carries frequency
|
| 102 | # weight 10 vs branch.create's 5, so it must rank first.
|
| 103 | rows = []
|
| 104 | for _ in range(3):
|
| 105 | rows.append(core_row("git", "status.dirty", tokens=100.0))
|
| 106 | rows.append(core_row("oak_installed", "status.dirty", tokens=200.0))
|
| 107 | rows.append(core_row("git", "branch.create", tokens=100.0))
|
| 108 | rows.append(core_row("oak_installed", "branch.create", tokens=200.0))
|
| 109 | write_results(self.results, core=rows)
|
| 110 |
|
| 111 | report, entries = opportunities.generate_report(self.results)
|
| 112 |
|
| 113 | losses = [
|
| 114 | entry
|
| 115 | for entry in entries_for(entries, "losses")
|
| 116 | if entry["metric"] == "estimated_cost_weighted_tokens"
|
| 117 | ]
|
| 118 | self.assertEqual([entry["operation"] for entry in losses], ["status.dirty", "branch.create"])
|
| 119 | self.assertEqual(losses[0]["rank"], 1)
|
| 120 | self.assertAlmostEqual(losses[0]["pct_gap"], 100.0)
|
| 121 | self.assertAlmostEqual(losses[0]["score"], 1000.0)
|
| 122 | self.assertAlmostEqual(losses[1]["score"], 500.0)
|
| 123 | self.assertFalse(losses[0]["low_n"])
|
| 124 | self.assertEqual(losses[0]["n"], 3)
|
| 125 | self.assertIn("## 1. Where Oak loses to Git", report)
|
| 126 | self.assertIn("status.dirty", report)
|
| 127 |
|
| 128 | def test_oak_better_than_git_is_not_an_opportunity(self) -> None:
|
| 129 | rows = [
|
| 130 | core_row("git", "status.dirty", tokens=200.0),
|
| 131 | core_row("oak_installed", "status.dirty", tokens=100.0),
|
| 132 | ]
|
| 133 | write_results(self.results, core=rows)
|
| 134 | _, entries = opportunities.generate_report(self.results)
|
| 135 | self.assertEqual(entries_for(entries, "losses"), [])
|
| 136 |
|
| 137 | def test_skip_rows_excluded_from_medians_and_listed_in_section_4(self) -> None:
|
| 138 | rows = []
|
| 139 | for _ in range(3):
|
| 140 | rows.append(core_row("git", "status.dirty", elapsed_ms=10.0))
|
| 141 | rows.append(core_row("oak_installed", "status.dirty", elapsed_ms=20.0))
|
| 142 | # A skip row with a huge elapsed_ms must never enter the median.
|
| 143 | rows.append(
|
| 144 | dict(
|
| 145 | skip_row("oak_installed", "status.dirty", "remote_not_configured"),
|
| 146 | elapsed_ms=99999.0,
|
| 147 | )
|
| 148 | )
|
| 149 | write_results(self.results, core=rows)
|
| 150 |
|
| 151 | report, entries = opportunities.generate_report(self.results)
|
| 152 |
|
| 153 | losses = entries_for(entries, "losses")
|
| 154 | self.assertEqual(len(losses), 1)
|
| 155 | self.assertEqual(losses[0]["metric"], "elapsed_ms")
|
| 156 | self.assertAlmostEqual(losses[0]["value"], 20.0)
|
| 157 | self.assertAlmostEqual(losses[0]["pct_gap"], 100.0)
|
| 158 |
|
| 159 | unmeasured = entries_for(entries, "unmeasured")
|
| 160 | self.assertEqual(len(unmeasured), 1)
|
| 161 | self.assertEqual(unmeasured[0]["metric"], "skip:remote_not_configured")
|
| 162 | self.assertEqual(unmeasured[0]["value"], 1.0)
|
| 163 | self.assertIn("remote_not_configured", report)
|
| 164 | self.assertIn("## 4. Unmeasured (coverage to unlock)", report)
|
| 165 |
|
| 166 | def test_null_metric_is_unmeasured_not_zero(self) -> None:
|
| 167 | # The oak row's tokens are null: no token comparison must be emitted,
|
| 168 | # and the null must not drag a median toward zero.
|
| 169 | rows = [
|
| 170 | core_row("git", "status.dirty", tokens=100.0),
|
| 171 | core_row("oak_installed", "status.dirty"),
|
| 172 | ]
|
| 173 | write_results(self.results, core=rows)
|
| 174 | _, entries = opportunities.generate_report(self.results)
|
| 175 | token_losses = [
|
| 176 | entry
|
| 177 | for entry in entries_for(entries, "losses")
|
| 178 | if entry["metric"] == "estimated_cost_weighted_tokens"
|
| 179 | ]
|
| 180 | self.assertEqual(token_losses, [])
|
| 181 |
|
| 182 | def test_envelope_tokens_preferred_when_present(self) -> None:
|
| 183 | rows = [
|
| 184 | core_row("git", "status.dirty", tokens=100.0, tokens_with_envelope=150.0),
|
| 185 | core_row("oak_installed", "status.dirty", tokens=100.0, tokens_with_envelope=300.0),
|
| 186 | ]
|
| 187 | write_results(self.results, core=rows)
|
| 188 | _, entries = opportunities.generate_report(self.results)
|
| 189 | token_losses = [
|
| 190 | entry
|
| 191 | for entry in entries_for(entries, "losses")
|
| 192 | if entry["metric"] == "estimated_cost_weighted_tokens"
|
| 193 | ]
|
| 194 | self.assertEqual(len(token_losses), 1)
|
| 195 | self.assertAlmostEqual(token_losses[0]["value"], 300.0)
|
| 196 | self.assertAlmostEqual(token_losses[0]["baseline_value"], 150.0)
|
| 197 | self.assertAlmostEqual(token_losses[0]["pct_gap"], 100.0)
|
| 198 |
|
| 199 | def test_quality_bar_detects_pipe_incompatibility_on_oak_only(self) -> None:
|
| 200 | rows = [
|
| 201 | core_row(
|
| 202 | "oak_installed",
|
| 203 | "diff.full.inforecall",
|
| 204 | pipe_compatible_unified=False,
|
| 205 | information_recall=1.0,
|
| 206 | ),
|
| 207 | # Git exhibiting the same flaw is context, not a target.
|
| 208 | core_row(
|
| 209 | "git",
|
| 210 | "diff.full.inforecall",
|
| 211 | pipe_compatible_unified=False,
|
| 212 | information_recall=1.0,
|
| 213 | ),
|
| 214 | ]
|
| 215 | write_results(self.results, core=rows)
|
| 216 | report, entries = opportunities.generate_report(self.results)
|
| 217 | quality = entries_for(entries, "quality")
|
| 218 | self.assertEqual(len(quality), 1)
|
| 219 | self.assertEqual(quality[0]["subject"], "oak_installed")
|
| 220 | self.assertEqual(quality[0]["metric"], "pipe_compatible_unified")
|
| 221 | self.assertEqual(quality[0]["value"], 0.0)
|
| 222 | self.assertIn("## 2. Absolute quality bar", report)
|
| 223 |
|
| 224 | def test_quality_bar_detects_recall_instability_and_ansi(self) -> None:
|
| 225 | rows = [
|
| 226 | core_row("oak_installed", "status.dirty.inforecall", information_recall=0.5),
|
| 227 | core_row("oak_installed", "status.dirty.inforecall", information_precision=0.25),
|
| 228 | core_row("oak_installed", "status.dirty.determinism", output_stable=False),
|
| 229 | core_row("oak_installed", "diff.dirty.determinism", output_stable=True, ansi_escape_count_non_tty=7),
|
| 230 | ]
|
| 231 | write_results(self.results, core=rows)
|
| 232 | _, entries = opportunities.generate_report(self.results)
|
| 233 | metrics = {entry["metric"]: entry for entry in entries_for(entries, "quality")}
|
| 234 | self.assertEqual(
|
| 235 | set(metrics),
|
| 236 | {"information_recall", "information_precision", "output_stable", "ansi_escape_count_non_tty"},
|
| 237 | )
|
| 238 | self.assertAlmostEqual(metrics["information_recall"]["value"], 0.5)
|
| 239 | self.assertAlmostEqual(metrics["information_precision"]["value"], 0.25)
|
| 240 | self.assertEqual(metrics["output_stable"]["value"], 0.0)
|
| 241 | self.assertEqual(metrics["ansi_escape_count_non_tty"]["value"], 7.0)
|
| 242 |
|
| 243 | def test_low_n_marked_inline_and_repeated_in_section_5(self) -> None:
|
| 244 | rows = [
|
| 245 | core_row("git", "status.dirty", tokens=100.0),
|
| 246 | core_row("oak_installed", "status.dirty", tokens=200.0),
|
| 247 | ]
|
| 248 | write_results(self.results, core=rows)
|
| 249 | report, entries = opportunities.generate_report(self.results)
|
| 250 |
|
| 251 | losses = entries_for(entries, "losses")
|
| 252 | self.assertTrue(all(entry["low_n"] for entry in losses))
|
| 253 | self.assertIn("(n=1, low-n)", report)
|
| 254 |
|
| 255 | low = entries_for(entries, "low_confidence")
|
| 256 | self.assertEqual(len(low), len(losses))
|
| 257 | self.assertIn("## 5. Low confidence", report)
|
| 258 | self.assertIn("- [losses] `tiny_text/status.dirty`", report)
|
| 259 |
|
| 260 | def test_missing_input_files_produce_lane_not_run_lines(self) -> None:
|
| 261 | write_results(self.results, core=[core_row("git", "status.dirty", tokens=100.0)])
|
| 262 | report, entries = opportunities.generate_report(self.results)
|
| 263 | self.assertIn("latest.mount.jsonl` — missing (lane not run)", report)
|
| 264 | self.assertIn("parallel-contention", report)
|
| 265 | self.assertNotIn("Traceback", report)
|
| 266 | self.assertEqual(entries_for(entries, "unmeasured"), [])
|
| 267 |
|
| 268 | def test_all_inputs_missing_still_reports(self) -> None:
|
| 269 | self.results.mkdir(parents=True)
|
| 270 | report, entries = opportunities.generate_report(self.results)
|
| 271 | self.assertEqual(report.count("missing (lane not run)"), 3)
|
| 272 | self.assertIn("- Oak baseline: missing", report)
|
| 273 | self.assertEqual(entries, [])
|
| 274 |
|
| 275 | def test_mount_and_remote_net_rows_become_trends(self) -> None:
|
| 276 | core = [core_row("oak_installed", "remote.net.push.first", elapsed_ms=400.0, tokens=350.0)]
|
| 277 | mount = [
|
| 278 | {
|
| 279 | "subject": "oak_mount",
|
| 280 | "scenario": "capability_probe",
|
| 281 | "operation": "oak.version",
|
| 282 | "returncode": 0,
|
| 283 | "elapsed_ms": 5.0,
|
| 284 | "run": 0,
|
| 285 | }
|
| 286 | ]
|
| 287 | write_results(self.results, core=core, mount=mount)
|
| 288 | report, entries = opportunities.generate_report(self.results)
|
| 289 | trends = entries_for(entries, "trends")
|
| 290 | ops = {(entry["operation"], entry["metric"]) for entry in trends}
|
| 291 | self.assertIn(("remote.net.push.first", "elapsed_ms"), ops)
|
| 292 | self.assertIn(("remote.net.push.first", "estimated_cost_weighted_tokens"), ops)
|
| 293 | self.assertIn(("oak.version", "elapsed_ms"), ops)
|
| 294 | self.assertIn("## 3. Oak-only trends (no Git comparator)", report)
|
| 295 |
|
| 296 | def test_deterministic_output(self) -> None:
|
| 297 | rows = [
|
| 298 | core_row("git", "status.dirty", tokens=100.0, output_bytes=50, tool_calls=1),
|
| 299 | core_row("oak_installed", "status.dirty", tokens=200.0, output_bytes=80, tool_calls=2),
|
| 300 | skip_row("oak_installed", "remote.push.first", "no remote configured"),
|
| 301 | ]
|
| 302 | write_results(self.results, core=rows)
|
| 303 | first = opportunities.generate_report(self.results)
|
| 304 | second = opportunities.generate_report(self.results)
|
| 305 | self.assertEqual(first, second)
|
| 306 |
|
| 307 | def test_main_writes_json(self) -> None:
|
| 308 | write_results(
|
| 309 | self.results,
|
| 310 | core=[
|
| 311 | core_row("git", "status.dirty", tokens=100.0),
|
| 312 | core_row("oak_installed", "status.dirty", tokens=200.0),
|
| 313 | ],
|
| 314 | )
|
| 315 | json_path = Path(self._tmp.name) / "out.json"
|
| 316 | stdout = io.StringIO()
|
| 317 | with contextlib.redirect_stdout(stdout):
|
| 318 | exit_code = opportunities.main(["--results", str(self.results), "--json", str(json_path)])
|
| 319 | self.assertEqual(exit_code, 0)
|
| 320 | self.assertIn("# Oak Opportunity Report", stdout.getvalue())
|
| 321 | entries = json.loads(json_path.read_text())
|
| 322 | self.assertTrue(entries)
|
| 323 | expected_keys = {
|
| 324 | "section",
|
| 325 | "rank",
|
| 326 | "backlog_rank",
|
| 327 | "scenario",
|
| 328 | "operation",
|
| 329 | "metric",
|
| 330 | "subject",
|
| 331 | "value",
|
| 332 | "baseline_value",
|
| 333 | "pct_gap",
|
| 334 | "score",
|
| 335 | "n",
|
| 336 | "low_n",
|
| 337 | "evidence_file",
|
| 338 | }
|
| 339 | for entry in entries:
|
| 340 | self.assertEqual(set(entry), expected_keys)
|
| 341 | self.assertEqual([entry["backlog_rank"] for entry in entries], [1, 1])
|
| 342 |
|
| 343 | def test_main_accepts_explicit_jsonl_paths_for_publish_readiness(self) -> None:
|
| 344 | core_rows = []
|
| 345 | mount_rows = []
|
| 346 | for _ in range(3):
|
| 347 | core_rows.append(core_row("git", "status.dirty", tokens=100.0))
|
| 348 | core_rows.append(core_row("oak_installed", "status.dirty", tokens=200.0))
|
| 349 | mount_rows.append(
|
| 350 | {
|
| 351 | "subject": "oak_mount",
|
| 352 | "subject_kind": "oak",
|
| 353 | "scenario": "capability_probe",
|
| 354 | "operation": "oak.version",
|
| 355 | "returncode": 0,
|
| 356 | "elapsed_ms": 5.0,
|
| 357 | "run": 0,
|
| 358 | }
|
| 359 | )
|
| 360 | write_results(
|
| 361 | self.results,
|
| 362 | core=core_rows,
|
| 363 | mount=mount_rows,
|
| 364 | )
|
| 365 | json_path = Path(self._tmp.name) / "backlog.json"
|
| 366 | stdout = io.StringIO()
|
| 367 | with contextlib.redirect_stdout(stdout):
|
| 368 | exit_code = opportunities.main(
|
| 369 | [
|
| 370 | str(self.results / "latest.jsonl"),
|
| 371 | str(self.results / "latest.mount.jsonl"),
|
| 372 | "--json",
|
| 373 | str(json_path),
|
| 374 | ]
|
| 375 | )
|
| 376 | self.assertEqual(exit_code, 0)
|
| 377 | report = stdout.getvalue()
|
| 378 | self.assertIn(f"- core: `{self.results / 'latest.jsonl'}` — read, 6 rows", report)
|
| 379 | self.assertIn(f"- mount: `{self.results / 'latest.mount.jsonl'}` — read, 3 rows", report)
|
| 380 | self.assertNotIn("parallel-contention", report)
|
| 381 |
|
| 382 | entries = json.loads(json_path.read_text())
|
| 383 | self.assertEqual([entry["backlog_rank"] for entry in entries], [1, 2])
|
| 384 | self.assertEqual(entries[0]["section"], "losses")
|
| 385 | self.assertEqual(entries[0]["operation"], "status.dirty")
|
| 386 | self.assertEqual(entries[1]["section"], "trends")
|
| 387 | self.assertEqual(entries[1]["operation"], "oak.version")
|
| 388 |
|
| 389 |
|
| 390 | class FrequencyWeightTest(unittest.TestCase):
|
| 391 | def test_documented_families(self) -> None:
|
| 392 | self.assertEqual(opportunities.frequency_weight("status.dirty"), 10)
|
| 393 | self.assertEqual(opportunities.frequency_weight("proc.spawn"), 10)
|
| 394 | self.assertEqual(opportunities.frequency_weight("task.snapshot"), 10)
|
| 395 | self.assertEqual(opportunities.frequency_weight("branch.create"), 5)
|
| 396 | self.assertEqual(opportunities.frequency_weight("repo.init"), 2)
|
| 397 | self.assertEqual(opportunities.frequency_weight("remote.push.first"), 2)
|
| 398 | self.assertEqual(opportunities.frequency_weight("parallel.total"), 1)
|
| 399 |
|
| 400 |
|
| 401 | if __name__ == "__main__":
|
| 402 | unittest.main()
|