Viewing files at commit 21316fe59317Commit detailsView latest

Browse files
tests/test_pure_codecs.py 112 lines · 4.8 KB · python
1
"""Pure-Python BLAKE3 / zstd / XXH64 used by the independent Oak reader.
2
3
Vectors: tests/data/blake3_vectors.json holds BLAKE3 of the spec's test input
4
pattern (byte i = i % 251) at chunk/block boundaries, produced once with the
5
reference ``blake3`` package; tests/data/zstd_vectors.json holds frames made
6
by the ``zstd`` CLI (levels 1-19, multi-block, RLE, no-checksum). When the
7
``zstd`` CLI is installed, a live differential also runs.
8
"""
9
import hashlib
10
import json
11
import os
12
from pathlib import Path
13
import shutil
14
import subprocess
15
import sys
16
import unittest
17
18
ROOT = Path(__file__).resolve().parents[1]
19
sys.path.insert(0, str(ROOT / "scripts"))
20
from oakbench import pureblake3, purezstd  # noqa: E402
21
22
DATA = ROOT / "tests" / "data"
23
24
25
def pattern(n):
26
    return bytes(i % 251 for i in range(n))
27
28
29
class Blake3Tests(unittest.TestCase):
30
    def test_reference_vectors_across_chunk_and_tree_boundaries(self):
31
        vectors = json.loads((DATA / "blake3_vectors.json").read_text())
32
        self.assertGreaterEqual(len(vectors), 20)
33
        for length, expected in vectors.items():
34
            with self.subTest(length=length):
35
                self.assertEqual(pureblake3.hexdigest(pattern(int(length))), expected)
36
37
    def test_single_bit_flip_changes_digest(self):
38
        data = bytearray(pattern(2049))
39
        before = pureblake3.hexdigest(bytes(data))
40
        data[1500] ^= 1
41
        self.assertNotEqual(before, pureblake3.hexdigest(bytes(data)))
42
43
44
class Xxh64Tests(unittest.TestCase):
45
    def test_known_values(self):
46
        self.assertEqual(purezstd.xxh64(b""), 0xEF46DB3751D8E999)
47
        self.assertEqual(purezstd.xxh64(b"a"), 0xD24EC4F1A98C6E5B)
48
        self.assertEqual(purezstd.xxh64(b"abc"), 0x44BC2CF5AD770999)
49
50
51
class ZstdVectorTests(unittest.TestCase):
52
    def setUp(self):
53
        self.vectors = json.loads((DATA / "zstd_vectors.json").read_text())["vectors"]
54
55
    def test_cli_generated_frames_decode_exactly(self):
56
        for vector in self.vectors:
57
            with self.subTest(vector=vector["name"]):
58
                out = purezstd.decompress(bytes.fromhex(vector["frame_hex"]))
59
                self.assertEqual(len(out), vector["size"])
60
                self.assertEqual(hashlib.sha256(out).hexdigest(), vector["sha256"])
61
62
    def test_every_single_byte_corruption_is_rejected_or_detected(self):
63
        # Flip each byte of a checksummed compressed frame: the decoder must
64
        # raise or (never) silently return the original plaintext's digest
65
        # for different input bytes being accepted as the same content.
66
        vector = next(v for v in self.vectors if v["name"] == "small-text-l19")
67
        frame = bytearray(bytes.fromhex(vector["frame_hex"]))
68
        silent = 0
69
        for index in range(len(frame)):
70
            corrupt = bytearray(frame)
71
            corrupt[index] ^= 0x41
72
            try:
73
                out = purezstd.decompress(bytes(corrupt))
74
            except (purezstd.ZstdError, purezstd.ZstdBoundError):
75
                continue
76
            if hashlib.sha256(out).hexdigest() != vector["sha256"]:
77
                silent += 1
78
        # The XXH64 content checksum must catch every altered output.
79
        self.assertEqual(silent, 0)
80
81
    def test_truncation_and_trailing_garbage_fail(self):
82
        frame = bytes.fromhex(self.vectors[0]["frame_hex"])
83
        for cut in (1, 4, 5, len(frame) // 2, len(frame) - 1):
84
            with self.assertRaises(purezstd.ZstdError):
85
                purezstd.decompress(frame[:cut])
86
        with self.assertRaises(purezstd.ZstdError):
87
            purezstd.decompress(frame + b"\x00garbage")
88
89
    def test_declared_size_and_window_bounds_are_bound_errors(self):
90
        frame = b"\x28\xb5\x2f\xfd\xa0" + (1 << 30).to_bytes(4, "little") + b"\x01\x00\x00"
91
        with self.assertRaises(purezstd.ZstdBoundError):
92
            purezstd.decompress(frame, max_output=1 << 20, max_window=1 << 20)
93
94
    def test_raw_encoder_roundtrip_and_multiblock(self):
95
        for payload in (b"", b"x", os.urandom(300_000)):
96
            self.assertEqual(purezstd.decompress(purezstd.encode_raw(payload)), payload)
97
        self.assertEqual(purezstd.frame_content_size(purezstd.encode_raw(b"abc")), 3)
98
99
    @unittest.skipUnless(shutil.which("zstd"), "zstd CLI not installed")
100
    def test_live_differential_against_zstd_cli(self):
101
        corpus = [pattern(70_000), os.urandom(140_000), b"oak tree blob " * 20_000,
102
                  (ROOT / "scripts" / "bench.py").read_bytes()]
103
        for data in corpus:
104
            for level in ("-1", "-6", "-19", "--fast=3", "--long=20"):
105
                with self.subTest(size=len(data), level=level):
106
                    frame = subprocess.run(["zstd", "-q", "-c", level], input=data, capture_output=True,
107
                                           check=True).stdout
108
                    self.assertEqual(purezstd.decompress(frame), data)
109
110
111
if __name__ == "__main__":
112
    unittest.main()