| 1 | """Optional independent decoder for local Oak SQLite v1 committed trees.
|
| 2 |
|
| 3 | No Oak code or executable is imported or invoked. Optional blake3/zstandard
|
| 4 | packages belong to a pinned reader environment, not the base harness. Unknown
|
| 5 | formats and missing instruments are unavailable evidence; corrupt objects fail.
|
| 6 | """
|
| 7 | from __future__ import annotations
|
| 8 | import hashlib
|
| 9 | import importlib.metadata
|
| 10 | import json
|
| 11 | from pathlib import Path
|
| 12 | import re
|
| 13 | import sqlite3
|
| 14 | import sys
|
| 15 | import time
|
| 16 | import tempfile
|
| 17 |
|
| 18 | READER_VERSION = 'oak-sqlite-v1-2026-09-18.1'
|
| 19 | MAX_OBJECT = 32 * 1024 * 1024
|
| 20 | MAX_TOTAL = 128 * 1024 * 1024
|
| 21 | MAX_ENTRIES = 8192
|
| 22 | MAX_DATABASE = 512 * 1024 * 1024
|
| 23 |
|
| 24 | class Unavailable(Exception):
|
| 25 | pass
|
| 26 |
|
| 27 | class Invalid(Exception):
|
| 28 | pass
|
| 29 |
|
| 30 |
|
| 31 | def read_committed_tree(repo: Path, destination: Path) -> dict:
|
| 32 | try:
|
| 33 | import blake3
|
| 34 | import zstandard
|
| 35 | except ImportError as exc:
|
| 36 | raise Unavailable('reader dependencies unavailable: blake3 and zstandard required') from exc
|
| 37 | db_path = repo / '.oak' / 'oak.db'
|
| 38 | if (repo.is_symlink() or (repo / '.oak').is_symlink() or db_path.is_symlink()
|
| 39 | or not db_path.is_file()):
|
| 40 | raise Unavailable('local Oak SQLite database missing or linked')
|
| 41 | if destination.exists() or destination.is_symlink():
|
| 42 | raise Invalid('reader destination must not exist')
|
| 43 | deadline = time.monotonic() + 60
|
| 44 | destination.mkdir(parents=True)
|
| 45 |
|
| 46 | def hashed(raw, expected):
|
| 47 | if not isinstance(expected, str) or not re.fullmatch('[0-9a-f]{64}', expected):
|
| 48 | raise Invalid('invalid object hash')
|
| 49 | if blake3.blake3(raw).hexdigest() != expected:
|
| 50 | raise Invalid('object hash mismatch')
|
| 51 | return raw
|
| 52 |
|
| 53 | def decode(content, codec):
|
| 54 | if not isinstance(content, bytes) or len(content) > MAX_OBJECT:
|
| 55 | raise Invalid('stored object exceeds reader bound')
|
| 56 | if codec == 0:
|
| 57 | return content
|
| 58 | if codec != 1:
|
| 59 | raise Unavailable('unsupported object codec')
|
| 60 | try:
|
| 61 | size = zstandard.frame_content_size(content)
|
| 62 | if zstandard.get_frame_parameters(content).window_size > MAX_OBJECT:
|
| 63 | raise Invalid('decompression window exceeds reader bound')
|
| 64 | if size not in (-1, zstandard.CONTENTSIZE_UNKNOWN) and not 0 <= size <= MAX_OBJECT:
|
| 65 | raise Invalid('decompressed object exceeds reader bound')
|
| 66 | raw = zstandard.ZstdDecompressor(max_window_size=MAX_OBJECT).decompress(
|
| 67 | content, max_output_size=MAX_OBJECT, allow_extra_data=False)
|
| 68 | except zstandard.ZstdError as exc:
|
| 69 | raise Invalid('invalid compressed object') from exc
|
| 70 | if len(raw) > MAX_OBJECT:
|
| 71 | raise Invalid('decompressed object exceeds reader bound')
|
| 72 | return raw
|
| 73 |
|
| 74 | def database_snapshot():
|
| 75 | if any(Path(str(db_path) + suffix).exists() for suffix in ('-wal', '-journal')):
|
| 76 | raise Unavailable('active SQLite journal; quiescent database required')
|
| 77 | with db_path.open('rb') as stream:
|
| 78 | content = stream.read(MAX_DATABASE + 1)
|
| 79 | if len(content) > MAX_DATABASE:
|
| 80 | raise Unavailable('database exceeds independent reader bound')
|
| 81 | return content
|
| 82 |
|
| 83 | # Snapshot only quiescent databases. immutable=1 is safe on this private copy,
|
| 84 | # never on a live WAL database; the original bytes are checked again below.
|
| 85 | snapshot = database_snapshot()
|
| 86 | database_sha256 = hashlib.sha256(snapshot).hexdigest()
|
| 87 | with tempfile.TemporaryDirectory(prefix='oakbench-reader-db-') as temp:
|
| 88 | snapshot_path = Path(temp) / 'snapshot.db'
|
| 89 | snapshot_path.write_bytes(snapshot)
|
| 90 | del snapshot
|
| 91 | return _read_snapshot(snapshot_path, database_sha256, database_snapshot,
|
| 92 | destination, deadline, hashed, decode, blake3)
|
| 93 |
|
| 94 |
|
| 95 | def _read_snapshot(snapshot_path, database_sha256, database_snapshot, destination,
|
| 96 | deadline, hashed, decode, blake3):
|
| 97 | total = 0
|
| 98 | entries_seen = 0
|
| 99 | with sqlite3.connect(snapshot_path.as_uri() + '?mode=ro&immutable=1', uri=True, timeout=5) as db:
|
| 100 | db.execute('PRAGMA query_only=ON')
|
| 101 | db.execute('PRAGMA trusted_schema=OFF')
|
| 102 | db.set_progress_handler(lambda: int(time.monotonic() > deadline), 1000)
|
| 103 | db.execute('BEGIN')
|
| 104 |
|
| 105 | def metadata(key):
|
| 106 | row = db.execute('SELECT value FROM metadata WHERE key=? AND length(value)<=4096', (key,)).fetchone()
|
| 107 | return row[0] if row else None
|
| 108 |
|
| 109 | if metadata('hash_format') not in (None, 'v1'):
|
| 110 | raise Unavailable('unsupported Oak hash format')
|
| 111 | head = metadata('head')
|
| 112 | branch = metadata('current_branch')
|
| 113 | if not head or not branch:
|
| 114 | raise Unavailable('committed HEAD unavailable')
|
| 115 | commit = db.execute('SELECT branch_name,parent_hash,merge_parent_hash,manifest_hash,author,message,timestamp FROM commits WHERE hash=? AND length(CAST(branch_name AS BLOB))<=4096 AND coalesce(length(CAST(parent_hash AS BLOB)),0)<=64 AND coalesce(length(CAST(merge_parent_hash AS BLOB)),0)<=64 AND length(CAST(manifest_hash AS BLOB))=64 AND length(CAST(author AS BLOB))<=4096 AND coalesce(length(CAST(message AS BLOB)),0)<=1048576 AND length(CAST(timestamp AS BLOB))<=128', (head,)).fetchone()
|
| 116 | if not commit or any(v is not None and (not isinstance(v, str) or len(v) > MAX_OBJECT) for v in commit):
|
| 117 | raise Invalid('committed HEAD object missing or invalid')
|
| 118 | # Oak v1 stores the same RFC3339 timestamp text used in its hash preimage.
|
| 119 | hashed('\n'.join(v or '' for v in commit).encode(), head)
|
| 120 | root_hash = commit[3]
|
| 121 |
|
| 122 | def blob(object_hash):
|
| 123 | row = db.execute('SELECT content,size,codec FROM blobs WHERE hash=? AND length(content)<=?', (object_hash, MAX_OBJECT)).fetchone()
|
| 124 | if row is None:
|
| 125 | if db.execute('SELECT 1 FROM blobs WHERE hash=?', (object_hash,)).fetchone():
|
| 126 | raise Invalid('stored blob exceeds reader bound')
|
| 127 | # Chunk-only hydration is not yet qualified by this reader.
|
| 128 | raise Unavailable('complete blob unavailable; chunk-only storage unsupported')
|
| 129 | content, size, codec = row
|
| 130 | if type(size) is not int or not 0 <= size <= MAX_OBJECT:
|
| 131 | raise Invalid('blob size exceeds reader bound')
|
| 132 | raw = decode(content, codec)
|
| 133 | if len(raw) != size:
|
| 134 | raise Invalid('blob size mismatch')
|
| 135 | return hashed(raw, object_hash)
|
| 136 |
|
| 137 | def visit(tree_hash, relative, ancestors):
|
| 138 | nonlocal total, entries_seen
|
| 139 | if len(ancestors) >= 32 or tree_hash in ancestors or time.monotonic() > deadline:
|
| 140 | raise Invalid('tree depth, cycle or time bound exceeded')
|
| 141 | if tree_hash == blake3.blake3(b'').hexdigest():
|
| 142 | return
|
| 143 | row = db.execute('SELECT content FROM trees WHERE hash=? AND (content IS NULL OR length(content)<=?)', (tree_hash, MAX_OBJECT)).fetchone()
|
| 144 | if row is None:
|
| 145 | raise Invalid('committed tree missing or oversized')
|
| 146 | if row[0] is None:
|
| 147 | if db.execute('SELECT 1 FROM tree_entries WHERE tree_hash=? AND (length(CAST(name AS BLOB))>4096 OR length(CAST(hash AS BLOB))!=64 OR length(CAST(kind AS BLOB))>4 OR length(CAST(mode AS BLOB))>10) LIMIT 1', (tree_hash,)).fetchone():
|
| 148 | raise Invalid('stored tree entry exceeds reader bound')
|
| 149 | stored = db.execute('SELECT kind,name,hash,mode FROM tree_entries WHERE tree_hash=? ORDER BY name LIMIT ?', (tree_hash, MAX_ENTRIES + 1)).fetchall()
|
| 150 | raw = '\n'.join('\t'.join((kind, name, obj, 'tree' if kind == 'tree' else mode)) for kind, name, obj, mode in stored).encode()
|
| 151 | else:
|
| 152 | raw = decode(row[0], 1)
|
| 153 | hashed(raw, tree_hash)
|
| 154 | try:
|
| 155 | lines = raw.decode('utf-8').split('\n') if raw else []
|
| 156 | except UnicodeError as exc:
|
| 157 | raise Invalid('invalid tree encoding') from exc
|
| 158 | previous = None
|
| 159 | for line in lines:
|
| 160 | parts = line.split('\t')
|
| 161 | if len(parts) != 4:
|
| 162 | raise Invalid('invalid tree entry')
|
| 163 | kind, name, object_hash, mode = parts
|
| 164 | if (not name or name in ('.', '..', '.oak', '.git') or any(c in name for c in '/\\\x00\n\r')
|
| 165 | or (previous is not None and name <= previous)):
|
| 166 | raise Invalid('unsafe or unordered tree path')
|
| 167 | previous = name
|
| 168 | entries_seen += 1
|
| 169 | if entries_seen > MAX_ENTRIES:
|
| 170 | raise Invalid('tree entry bound exceeded')
|
| 171 | path = relative / name
|
| 172 | if (destination / path).exists() or (destination / path).is_symlink():
|
| 173 | raise Unavailable('filesystem path collision on reader host')
|
| 174 | if kind == 'tree' and mode == 'tree':
|
| 175 | (destination / path).mkdir()
|
| 176 | visit(object_hash, path, ancestors | {tree_hash})
|
| 177 | elif kind == 'blob' and mode in ('regular', 'executable'):
|
| 178 | content = blob(object_hash)
|
| 179 | total += len(content)
|
| 180 | if total > MAX_TOTAL:
|
| 181 | raise Invalid('total payload bound exceeded')
|
| 182 | target = destination / path
|
| 183 | target.write_bytes(content)
|
| 184 | target.chmod(0o755 if mode == 'executable' else 0o644)
|
| 185 | else:
|
| 186 | raise Unavailable('unsupported tree entry kind or mode')
|
| 187 | visit(root_hash, Path(), set())
|
| 188 | db.commit()
|
| 189 | if hashlib.sha256(database_snapshot()).hexdigest() != database_sha256:
|
| 190 | raise Unavailable('database changed during independent read')
|
| 191 | return {'head': head, 'database_sha256': database_sha256, 'reader_version': READER_VERSION,
|
| 192 | 'reader_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
| 193 | 'blake3_version': importlib.metadata.version('blake3'),
|
| 194 | 'zstandard_version': importlib.metadata.version('zstandard')}
|
| 195 |
|
| 196 |
|
| 197 | def main():
|
| 198 | try:
|
| 199 | if len(sys.argv) != 3:
|
| 200 | raise Invalid('usage: oak_sqlite_reader.py REPOSITORY NEW_DESTINATION')
|
| 201 | result = read_committed_tree(Path(sys.argv[1]), Path(sys.argv[2]))
|
| 202 | code = 0
|
| 203 | except Unavailable as exc:
|
| 204 | result, code = {'unavailable': str(exc)}, 77
|
| 205 | except Invalid as exc:
|
| 206 | result, code = {'corrupt': True, 'reason': str(exc)}, 1
|
| 207 | except (OSError, ValueError, TypeError, sqlite3.Error) as exc:
|
| 208 | result, code = {'unavailable': 'reader instrument error: ' + str(exc)}, 77
|
| 209 | print(json.dumps(result, sort_keys=True))
|
| 210 | return code
|
| 211 |
|
| 212 | if __name__ == '__main__':
|
| 213 | raise SystemExit(main())
|