| 1 | """Pure-Python BLAKE3 / zstd / XXH64 used by the independent Oak reader.
|
| 2 |
|
| 3 | Vectors: tests/data/blake3_vectors.json holds BLAKE3 of the spec's test input
|
| 4 | pattern (byte i = i % 251) at chunk/block boundaries, produced once with the
|
| 5 | reference ``blake3`` package; tests/data/zstd_vectors.json holds frames made
|
| 6 | by the ``zstd`` CLI (levels 1-19, multi-block, RLE, no-checksum). When the
|
| 7 | ``zstd`` CLI is installed, a live differential also runs.
|
| 8 | """
|
| 9 | import hashlib
|
| 10 | import json
|
| 11 | import os
|
| 12 | from pathlib import Path
|
| 13 | import shutil
|
| 14 | import subprocess
|
| 15 | import sys
|
| 16 | import unittest
|
| 17 |
|
| 18 | ROOT = Path(__file__).resolve().parents[1]
|
| 19 | sys.path.insert(0, str(ROOT / "scripts"))
|
| 20 | from oakbench import pureblake3, purezstd # noqa: E402
|
| 21 |
|
| 22 | DATA = ROOT / "tests" / "data"
|
| 23 |
|
| 24 |
|
| 25 | def pattern(n):
|
| 26 | return bytes(i % 251 for i in range(n))
|
| 27 |
|
| 28 |
|
| 29 | class Blake3Tests(unittest.TestCase):
|
| 30 | def test_reference_vectors_across_chunk_and_tree_boundaries(self):
|
| 31 | vectors = json.loads((DATA / "blake3_vectors.json").read_text())
|
| 32 | self.assertGreaterEqual(len(vectors), 20)
|
| 33 | for length, expected in vectors.items():
|
| 34 | with self.subTest(length=length):
|
| 35 | self.assertEqual(pureblake3.hexdigest(pattern(int(length))), expected)
|
| 36 |
|
| 37 | def test_single_bit_flip_changes_digest(self):
|
| 38 | data = bytearray(pattern(2049))
|
| 39 | before = pureblake3.hexdigest(bytes(data))
|
| 40 | data[1500] ^= 1
|
| 41 | self.assertNotEqual(before, pureblake3.hexdigest(bytes(data)))
|
| 42 |
|
| 43 |
|
| 44 | class Xxh64Tests(unittest.TestCase):
|
| 45 | def test_known_values(self):
|
| 46 | self.assertEqual(purezstd.xxh64(b""), 0xEF46DB3751D8E999)
|
| 47 | self.assertEqual(purezstd.xxh64(b"a"), 0xD24EC4F1A98C6E5B)
|
| 48 | self.assertEqual(purezstd.xxh64(b"abc"), 0x44BC2CF5AD770999)
|
| 49 |
|
| 50 |
|
| 51 | class ZstdVectorTests(unittest.TestCase):
|
| 52 | def setUp(self):
|
| 53 | self.vectors = json.loads((DATA / "zstd_vectors.json").read_text())["vectors"]
|
| 54 |
|
| 55 | def test_cli_generated_frames_decode_exactly(self):
|
| 56 | for vector in self.vectors:
|
| 57 | with self.subTest(vector=vector["name"]):
|
| 58 | out = purezstd.decompress(bytes.fromhex(vector["frame_hex"]))
|
| 59 | self.assertEqual(len(out), vector["size"])
|
| 60 | self.assertEqual(hashlib.sha256(out).hexdigest(), vector["sha256"])
|
| 61 |
|
| 62 | def test_every_single_byte_corruption_is_rejected_or_detected(self):
|
| 63 | # Flip each byte of a checksummed compressed frame: the decoder must
|
| 64 | # raise or (never) silently return the original plaintext's digest
|
| 65 | # for different input bytes being accepted as the same content.
|
| 66 | vector = next(v for v in self.vectors if v["name"] == "small-text-l19")
|
| 67 | frame = bytearray(bytes.fromhex(vector["frame_hex"]))
|
| 68 | silent = 0
|
| 69 | for index in range(len(frame)):
|
| 70 | corrupt = bytearray(frame)
|
| 71 | corrupt[index] ^= 0x41
|
| 72 | try:
|
| 73 | out = purezstd.decompress(bytes(corrupt))
|
| 74 | except (purezstd.ZstdError, purezstd.ZstdBoundError):
|
| 75 | continue
|
| 76 | if hashlib.sha256(out).hexdigest() != vector["sha256"]:
|
| 77 | silent += 1
|
| 78 | # The XXH64 content checksum must catch every altered output.
|
| 79 | self.assertEqual(silent, 0)
|
| 80 |
|
| 81 | def test_truncation_and_trailing_garbage_fail(self):
|
| 82 | frame = bytes.fromhex(self.vectors[0]["frame_hex"])
|
| 83 | for cut in (1, 4, 5, len(frame) // 2, len(frame) - 1):
|
| 84 | with self.assertRaises(purezstd.ZstdError):
|
| 85 | purezstd.decompress(frame[:cut])
|
| 86 | with self.assertRaises(purezstd.ZstdError):
|
| 87 | purezstd.decompress(frame + b"\x00garbage")
|
| 88 |
|
| 89 | def test_declared_size_and_window_bounds_are_bound_errors(self):
|
| 90 | frame = b"\x28\xb5\x2f\xfd\xa0" + (1 << 30).to_bytes(4, "little") + b"\x01\x00\x00"
|
| 91 | with self.assertRaises(purezstd.ZstdBoundError):
|
| 92 | purezstd.decompress(frame, max_output=1 << 20, max_window=1 << 20)
|
| 93 |
|
| 94 | def test_raw_encoder_roundtrip_and_multiblock(self):
|
| 95 | for payload in (b"", b"x", os.urandom(300_000)):
|
| 96 | self.assertEqual(purezstd.decompress(purezstd.encode_raw(payload)), payload)
|
| 97 | self.assertEqual(purezstd.frame_content_size(purezstd.encode_raw(b"abc")), 3)
|
| 98 |
|
| 99 | # A hand-built compressed block: raw literals of size 0, one sequence with
|
| 100 | # RLE tables (LL=0, OF code 3 -> offset 5, ML=3) and a 3-bit bitstream.
|
| 101 | MATCH_BLOCK = bytes([0x3D, 0x00, 0x00, 0x00, 0x01, 0x54, 0x00, 0x03, 0x00, 0x08])
|
| 102 | FRAME_HEADER = b"\x28\xb5\x2f\xfd\x00\x00" # no content size, 1 KiB window
|
| 103 |
|
| 104 | def test_match_within_frame_decodes(self):
|
| 105 | raw_block = bytes([(6 << 3), 0x00, 0x00]) + b"abcdef"
|
| 106 | self.assertEqual(purezstd.decompress(self.FRAME_HEADER + raw_block + self.MATCH_BLOCK), b"abcdefbcd")
|
| 107 |
|
| 108 | def test_match_into_previous_frame_is_rejected(self):
|
| 109 | with self.assertRaisesRegex(purezstd.ZstdError, "outside frame window"):
|
| 110 | purezstd.decompress(purezstd.encode_raw(b"abcdef") + self.FRAME_HEADER + self.MATCH_BLOCK)
|
| 111 |
|
| 112 | def test_match_beyond_declared_window_is_rejected(self):
|
| 113 | # Single-segment frame whose window (= content size 9) is fine, then a
|
| 114 | # frame declaring content size 4 but reaching back 5 bytes.
|
| 115 | frame = b"\x28\xb5\x2f\xfd\x20\x09" + bytes([(6 << 3), 0x00, 0x00]) + b"abcdef" + self.MATCH_BLOCK
|
| 116 | self.assertEqual(purezstd.decompress(frame), b"abcdefbcd")
|
| 117 | small = b"\x28\xb5\x2f\xfd\x20\x04" + bytes([(6 << 3), 0x00, 0x00]) + b"abcdef" + self.MATCH_BLOCK
|
| 118 | with self.assertRaises(purezstd.ZstdError):
|
| 119 | purezstd.decompress(small)
|
| 120 |
|
| 121 | @unittest.skipUnless(shutil.which("zstd"), "zstd CLI not installed")
|
| 122 | def test_live_differential_against_zstd_cli(self):
|
| 123 | corpus = [pattern(70_000), os.urandom(140_000), b"oak tree blob " * 20_000,
|
| 124 | (ROOT / "scripts" / "bench.py").read_bytes()]
|
| 125 | for data in corpus:
|
| 126 | for level in ("-1", "-6", "-19", "--fast=3", "--long=20"):
|
| 127 | with self.subTest(size=len(data), level=level):
|
| 128 | frame = subprocess.run(["zstd", "-q", "-c", level], input=data, capture_output=True,
|
| 129 | check=True).stdout
|
| 130 | self.assertEqual(purezstd.decompress(frame), data)
|
| 131 |
|
| 132 |
|
| 133 | if __name__ == "__main__":
|
| 134 | unittest.main()
|