| 1 | """Tests for oakbench.calibration: versioned reporting-layer token factors."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import json
|
| 6 | import sys
|
| 7 | import tempfile
|
| 8 | import unittest
|
| 9 | from pathlib import Path
|
| 10 |
|
| 11 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 12 |
|
| 13 | from oakbench.calibration import (
|
| 14 | DEFAULT_CALIBRATION_PATH,
|
| 15 | available_token_counters,
|
| 16 | calibrated_tokens,
|
| 17 | load_calibration,
|
| 18 | )
|
| 19 |
|
| 20 |
|
| 21 | class LoadCalibrationTest(unittest.TestCase):
|
| 22 | def test_placeholder_config_loads(self) -> None:
|
| 23 | calibration = load_calibration(DEFAULT_CALIBRATION_PATH)
|
| 24 | self.assertEqual(calibration["schema_version"], 1)
|
| 25 | self.assertEqual(calibration["calibration_version"], "uncalibrated-v0")
|
| 26 | self.assertEqual(calibration["method"], "composition_heuristic")
|
| 27 | for style in ("status_output", "diff_output", "log_output", "prose"):
|
| 28 | entry = calibration["factors"][style]
|
| 29 | # Honest placeholder: usable heuristic factor, but n=0 and null CIs.
|
| 30 | self.assertEqual(entry["n"], 0)
|
| 31 | self.assertIsNone(entry["ci_low"])
|
| 32 | self.assertIsNone(entry["ci_high"])
|
| 33 |
|
| 34 | def test_bad_schema_version_raises_value_error(self) -> None:
|
| 35 | with tempfile.TemporaryDirectory() as tmp:
|
| 36 | path = Path(tmp) / "token_calibration.json"
|
| 37 | path.write_text(json.dumps({"schema_version": 99, "calibration_version": "x", "factors": {}}))
|
| 38 | with self.assertRaises(ValueError) as ctx:
|
| 39 | load_calibration(path)
|
| 40 | self.assertIn("schema_version", str(ctx.exception))
|
| 41 |
|
| 42 | def test_missing_schema_version_raises_value_error(self) -> None:
|
| 43 | with tempfile.TemporaryDirectory() as tmp:
|
| 44 | path = Path(tmp) / "token_calibration.json"
|
| 45 | path.write_text(json.dumps({"calibration_version": "x", "factors": {}}))
|
| 46 | with self.assertRaises(ValueError):
|
| 47 | load_calibration(path)
|
| 48 |
|
| 49 |
|
| 50 | class CalibratedTokensTest(unittest.TestCase):
|
| 51 | CALIBRATION = {
|
| 52 | "schema_version": 1,
|
| 53 | "calibration_version": "cal-test-v1",
|
| 54 | "method": "composition_heuristic",
|
| 55 | "factors": {
|
| 56 | "prose": {"chars_per_token": 4.0, "ci_low": 3.8, "ci_high": 4.2, "n": 5},
|
| 57 | "diff_output": {"chars_per_token": 3.2, "ci_low": 3.0, "ci_high": 3.4, "n": 5},
|
| 58 | "log_output": {"chars_per_token": None, "ci_low": None, "ci_high": None, "n": 0},
|
| 59 | },
|
| 60 | }
|
| 61 |
|
| 62 | def test_math_and_version_citation(self) -> None:
|
| 63 | result = calibrated_tokens(400, "prose", self.CALIBRATION)
|
| 64 | self.assertIsNotNone(result)
|
| 65 | self.assertEqual(result["tokens_estimated"], 100.0)
|
| 66 | self.assertEqual(result["calibration_version"], "cal-test-v1")
|
| 67 | self.assertEqual(result["method"], "composition_heuristic")
|
| 68 | self.assertEqual(result["style"], "prose")
|
| 69 |
|
| 70 | def test_diff_style_uses_its_own_factor(self) -> None:
|
| 71 | result = calibrated_tokens(320, "diff_output", self.CALIBRATION)
|
| 72 | self.assertIsNotNone(result)
|
| 73 | self.assertAlmostEqual(result["tokens_estimated"], 100.0)
|
| 74 |
|
| 75 | def test_unknown_style_is_unmeasured(self) -> None:
|
| 76 | # Unknown style never gets a silent default factor (ADR-0002).
|
| 77 | self.assertIsNone(calibrated_tokens(400, "interpretive_dance", self.CALIBRATION))
|
| 78 |
|
| 79 | def test_null_factor_style_is_unmeasured(self) -> None:
|
| 80 | self.assertIsNone(calibrated_tokens(400, "log_output", self.CALIBRATION))
|
| 81 |
|
| 82 | def test_none_chars_is_unmeasured(self) -> None:
|
| 83 | self.assertIsNone(calibrated_tokens(None, "prose", self.CALIBRATION))
|
| 84 |
|
| 85 |
|
| 86 | class AvailableTokenCountersTest(unittest.TestCase):
|
| 87 | def test_feature_detection_shape(self) -> None:
|
| 88 | counters = available_token_counters()
|
| 89 | self.assertEqual(set(counters), {"tiktoken", "anthropic_api"})
|
| 90 | self.assertIsInstance(counters["tiktoken"], bool)
|
| 91 | self.assertIsInstance(counters["anthropic_api"], bool)
|
| 92 |
|
| 93 |
|
| 94 | if __name__ == "__main__":
|
| 95 | unittest.main()
|