Viewing files at commit 19b295c14414Commit detailsView latest

Browse files
tests/test_pure_codecs.py 134 lines · 6.1 KB · python
1
"""Pure-Python BLAKE3 / zstd / XXH64 used by the independent Oak reader.
2
3
Vectors: tests/data/blake3_vectors.json holds BLAKE3 of the spec's test input
4
pattern (byte i = i % 251) at chunk/block boundaries, produced once with the
5
reference ``blake3`` package; tests/data/zstd_vectors.json holds frames made
6
by the ``zstd`` CLI (levels 1-19, multi-block, RLE, no-checksum). When the
7
``zstd`` CLI is installed, a live differential also runs.
8
"""
9
import hashlib
10
import json
11
import os
12
from pathlib import Path
13
import shutil
14
import subprocess
15
import sys
16
import unittest
17
18
ROOT = Path(__file__).resolve().parents[1]
19
sys.path.insert(0, str(ROOT / "scripts"))
20
from oakbench import pureblake3, purezstd  # noqa: E402
21
22
DATA = ROOT / "tests" / "data"
23
24
25
def pattern(n):
26
    return bytes(i % 251 for i in range(n))
27
28
29
class Blake3Tests(unittest.TestCase):
30
    def test_reference_vectors_across_chunk_and_tree_boundaries(self):
31
        vectors = json.loads((DATA / "blake3_vectors.json").read_text())
32
        self.assertGreaterEqual(len(vectors), 20)
33
        for length, expected in vectors.items():
34
            with self.subTest(length=length):
35
                self.assertEqual(pureblake3.hexdigest(pattern(int(length))), expected)
36
37
    def test_single_bit_flip_changes_digest(self):
38
        data = bytearray(pattern(2049))
39
        before = pureblake3.hexdigest(bytes(data))
40
        data[1500] ^= 1
41
        self.assertNotEqual(before, pureblake3.hexdigest(bytes(data)))
42
43
44
class Xxh64Tests(unittest.TestCase):
45
    def test_known_values(self):
46
        self.assertEqual(purezstd.xxh64(b""), 0xEF46DB3751D8E999)
47
        self.assertEqual(purezstd.xxh64(b"a"), 0xD24EC4F1A98C6E5B)
48
        self.assertEqual(purezstd.xxh64(b"abc"), 0x44BC2CF5AD770999)
49
50
51
class ZstdVectorTests(unittest.TestCase):
52
    def setUp(self):
53
        self.vectors = json.loads((DATA / "zstd_vectors.json").read_text())["vectors"]
54
55
    def test_cli_generated_frames_decode_exactly(self):
56
        for vector in self.vectors:
57
            with self.subTest(vector=vector["name"]):
58
                out = purezstd.decompress(bytes.fromhex(vector["frame_hex"]))
59
                self.assertEqual(len(out), vector["size"])
60
                self.assertEqual(hashlib.sha256(out).hexdigest(), vector["sha256"])
61
62
    def test_every_single_byte_corruption_is_rejected_or_detected(self):
63
        # Flip each byte of a checksummed compressed frame: the decoder must
64
        # raise or (never) silently return the original plaintext's digest
65
        # for different input bytes being accepted as the same content.
66
        vector = next(v for v in self.vectors if v["name"] == "small-text-l19")
67
        frame = bytearray(bytes.fromhex(vector["frame_hex"]))
68
        silent = 0
69
        for index in range(len(frame)):
70
            corrupt = bytearray(frame)
71
            corrupt[index] ^= 0x41
72
            try:
73
                out = purezstd.decompress(bytes(corrupt))
74
            except (purezstd.ZstdError, purezstd.ZstdBoundError):
75
                continue
76
            if hashlib.sha256(out).hexdigest() != vector["sha256"]:
77
                silent += 1
78
        # The XXH64 content checksum must catch every altered output.
79
        self.assertEqual(silent, 0)
80
81
    def test_truncation_and_trailing_garbage_fail(self):
82
        frame = bytes.fromhex(self.vectors[0]["frame_hex"])
83
        for cut in (1, 4, 5, len(frame) // 2, len(frame) - 1):
84
            with self.assertRaises(purezstd.ZstdError):
85
                purezstd.decompress(frame[:cut])
86
        with self.assertRaises(purezstd.ZstdError):
87
            purezstd.decompress(frame + b"\x00garbage")
88
89
    def test_declared_size_and_window_bounds_are_bound_errors(self):
90
        frame = b"\x28\xb5\x2f\xfd\xa0" + (1 << 30).to_bytes(4, "little") + b"\x01\x00\x00"
91
        with self.assertRaises(purezstd.ZstdBoundError):
92
            purezstd.decompress(frame, max_output=1 << 20, max_window=1 << 20)
93
94
    def test_raw_encoder_roundtrip_and_multiblock(self):
95
        for payload in (b"", b"x", os.urandom(300_000)):
96
            self.assertEqual(purezstd.decompress(purezstd.encode_raw(payload)), payload)
97
        self.assertEqual(purezstd.frame_content_size(purezstd.encode_raw(b"abc")), 3)
98
99
    # A hand-built compressed block: raw literals of size 0, one sequence with
100
    # RLE tables (LL=0, OF code 3 -> offset 5, ML=3) and a 3-bit bitstream.
101
    MATCH_BLOCK = bytes([0x3D, 0x00, 0x00, 0x00, 0x01, 0x54, 0x00, 0x03, 0x00, 0x08])
102
    FRAME_HEADER = b"\x28\xb5\x2f\xfd\x00\x00"  # no content size, 1 KiB window
103
104
    def test_match_within_frame_decodes(self):
105
        raw_block = bytes([(6 << 3), 0x00, 0x00]) + b"abcdef"
106
        self.assertEqual(purezstd.decompress(self.FRAME_HEADER + raw_block + self.MATCH_BLOCK), b"abcdefbcd")
107
108
    def test_match_into_previous_frame_is_rejected(self):
109
        with self.assertRaisesRegex(purezstd.ZstdError, "outside frame window"):
110
            purezstd.decompress(purezstd.encode_raw(b"abcdef") + self.FRAME_HEADER + self.MATCH_BLOCK)
111
112
    def test_match_beyond_declared_window_is_rejected(self):
113
        # Single-segment frame whose window (= content size 9) is fine, then a
114
        # frame declaring content size 4 but reaching back 5 bytes.
115
        frame = b"\x28\xb5\x2f\xfd\x20\x09" + bytes([(6 << 3), 0x00, 0x00]) + b"abcdef" + self.MATCH_BLOCK
116
        self.assertEqual(purezstd.decompress(frame), b"abcdefbcd")
117
        small = b"\x28\xb5\x2f\xfd\x20\x04" + bytes([(6 << 3), 0x00, 0x00]) + b"abcdef" + self.MATCH_BLOCK
118
        with self.assertRaises(purezstd.ZstdError):
119
            purezstd.decompress(small)
120
121
    @unittest.skipUnless(shutil.which("zstd"), "zstd CLI not installed")
122
    def test_live_differential_against_zstd_cli(self):
123
        corpus = [pattern(70_000), os.urandom(140_000), b"oak tree blob " * 20_000,
124
                  (ROOT / "scripts" / "bench.py").read_bytes()]
125
        for data in corpus:
126
            for level in ("-1", "-6", "-19", "--fast=3", "--long=20"):
127
                with self.subTest(size=len(data), level=level):
128
                    frame = subprocess.run(["zstd", "-q", "-c", level], input=data, capture_output=True,
129
                                           check=True).stdout
130
                    self.assertEqual(purezstd.decompress(frame), data)
131
132
133
if __name__ == "__main__":
134
    unittest.main()