Viewing files at commit 3d84c25e26dbCommit detailsView latest

Browse files
tests/test_oakbench_calibration.py 95 lines · 3.9 KB · python
1
"""Tests for oakbench.calibration: versioned reporting-layer token factors."""
2
3
from __future__ import annotations
4
5
import json
6
import sys
7
import tempfile
8
import unittest
9
from pathlib import Path
10
11
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
12
13
from oakbench.calibration import (
14
    DEFAULT_CALIBRATION_PATH,
15
    available_token_counters,
16
    calibrated_tokens,
17
    load_calibration,
18
)
19
20
21
class LoadCalibrationTest(unittest.TestCase):
22
    def test_placeholder_config_loads(self) -> None:
23
        calibration = load_calibration(DEFAULT_CALIBRATION_PATH)
24
        self.assertEqual(calibration["schema_version"], 1)
25
        self.assertEqual(calibration["calibration_version"], "uncalibrated-v0")
26
        self.assertEqual(calibration["method"], "composition_heuristic")
27
        for style in ("status_output", "diff_output", "log_output", "prose"):
28
            entry = calibration["factors"][style]
29
            # Honest placeholder: usable heuristic factor, but n=0 and null CIs.
30
            self.assertEqual(entry["n"], 0)
31
            self.assertIsNone(entry["ci_low"])
32
            self.assertIsNone(entry["ci_high"])
33
34
    def test_bad_schema_version_raises_value_error(self) -> None:
35
        with tempfile.TemporaryDirectory() as tmp:
36
            path = Path(tmp) / "token_calibration.json"
37
            path.write_text(json.dumps({"schema_version": 99, "calibration_version": "x", "factors": {}}))
38
            with self.assertRaises(ValueError) as ctx:
39
                load_calibration(path)
40
            self.assertIn("schema_version", str(ctx.exception))
41
42
    def test_missing_schema_version_raises_value_error(self) -> None:
43
        with tempfile.TemporaryDirectory() as tmp:
44
            path = Path(tmp) / "token_calibration.json"
45
            path.write_text(json.dumps({"calibration_version": "x", "factors": {}}))
46
            with self.assertRaises(ValueError):
47
                load_calibration(path)
48
49
50
class CalibratedTokensTest(unittest.TestCase):
51
    CALIBRATION = {
52
        "schema_version": 1,
53
        "calibration_version": "cal-test-v1",
54
        "method": "composition_heuristic",
55
        "factors": {
56
            "prose": {"chars_per_token": 4.0, "ci_low": 3.8, "ci_high": 4.2, "n": 5},
57
            "diff_output": {"chars_per_token": 3.2, "ci_low": 3.0, "ci_high": 3.4, "n": 5},
58
            "log_output": {"chars_per_token": None, "ci_low": None, "ci_high": None, "n": 0},
59
        },
60
    }
61
62
    def test_math_and_version_citation(self) -> None:
63
        result = calibrated_tokens(400, "prose", self.CALIBRATION)
64
        self.assertIsNotNone(result)
65
        self.assertEqual(result["tokens_estimated"], 100.0)
66
        self.assertEqual(result["calibration_version"], "cal-test-v1")
67
        self.assertEqual(result["method"], "composition_heuristic")
68
        self.assertEqual(result["style"], "prose")
69
70
    def test_diff_style_uses_its_own_factor(self) -> None:
71
        result = calibrated_tokens(320, "diff_output", self.CALIBRATION)
72
        self.assertIsNotNone(result)
73
        self.assertAlmostEqual(result["tokens_estimated"], 100.0)
74
75
    def test_unknown_style_is_unmeasured(self) -> None:
76
        # Unknown style never gets a silent default factor (ADR-0002).
77
        self.assertIsNone(calibrated_tokens(400, "interpretive_dance", self.CALIBRATION))
78
79
    def test_null_factor_style_is_unmeasured(self) -> None:
80
        self.assertIsNone(calibrated_tokens(400, "log_output", self.CALIBRATION))
81
82
    def test_none_chars_is_unmeasured(self) -> None:
83
        self.assertIsNone(calibrated_tokens(None, "prose", self.CALIBRATION))
84
85
86
class AvailableTokenCountersTest(unittest.TestCase):
87
    def test_feature_detection_shape(self) -> None:
88
        counters = available_token_counters()
89
        self.assertEqual(set(counters), {"tiktoken", "anthropic_api"})
90
        self.assertIsInstance(counters["tiktoken"], bool)
91
        self.assertIsInstance(counters["anthropic_api"], bool)
92
93
94
if __name__ == "__main__":
95
    unittest.main()