Viewing files at commit f0c34b306c79Commit detailsView latest

Browse files
scripts/oakbench/oak_sqlite_reader.py 213 lines · 10.6 KB · python
1
"""Optional independent decoder for local Oak SQLite v1 committed trees.
2
3
No Oak code or executable is imported or invoked. Optional blake3/zstandard
4
packages belong to a pinned reader environment, not the base harness. Unknown
5
formats and missing instruments are unavailable evidence; corrupt objects fail.
6
"""
7
from __future__ import annotations
8
import hashlib
9
import importlib.metadata
10
import json
11
from pathlib import Path
12
import re
13
import sqlite3
14
import sys
15
import time
16
import tempfile
17
18
READER_VERSION = 'oak-sqlite-v1-2026-09-18.1'
19
MAX_OBJECT = 32 * 1024 * 1024
20
MAX_TOTAL = 128 * 1024 * 1024
21
MAX_ENTRIES = 8192
22
MAX_DATABASE = 512 * 1024 * 1024
23
24
class Unavailable(Exception):
25
    pass
26
27
class Invalid(Exception):
28
    pass
29
30
31
def read_committed_tree(repo: Path, destination: Path) -> dict:
32
    try:
33
        import blake3
34
        import zstandard
35
    except ImportError as exc:
36
        raise Unavailable('reader dependencies unavailable: blake3 and zstandard required') from exc
37
    db_path = repo / '.oak' / 'oak.db'
38
    if (repo.is_symlink() or (repo / '.oak').is_symlink() or db_path.is_symlink()
39
            or not db_path.is_file()):
40
        raise Unavailable('local Oak SQLite database missing or linked')
41
    if destination.exists() or destination.is_symlink():
42
        raise Invalid('reader destination must not exist')
43
    deadline = time.monotonic() + 60
44
    destination.mkdir(parents=True)
45
46
    def hashed(raw, expected):
47
        if not isinstance(expected, str) or not re.fullmatch('[0-9a-f]{64}', expected):
48
            raise Invalid('invalid object hash')
49
        if blake3.blake3(raw).hexdigest() != expected:
50
            raise Invalid('object hash mismatch')
51
        return raw
52
53
    def decode(content, codec):
54
        if not isinstance(content, bytes) or len(content) > MAX_OBJECT:
55
            raise Invalid('stored object exceeds reader bound')
56
        if codec == 0:
57
            return content
58
        if codec != 1:
59
            raise Unavailable('unsupported object codec')
60
        try:
61
            size = zstandard.frame_content_size(content)
62
            if zstandard.get_frame_parameters(content).window_size > MAX_OBJECT:
63
                raise Invalid('decompression window exceeds reader bound')
64
            if size not in (-1, zstandard.CONTENTSIZE_UNKNOWN) and not 0 <= size <= MAX_OBJECT:
65
                raise Invalid('decompressed object exceeds reader bound')
66
            raw = zstandard.ZstdDecompressor(max_window_size=MAX_OBJECT).decompress(
67
                content, max_output_size=MAX_OBJECT, allow_extra_data=False)
68
        except zstandard.ZstdError as exc:
69
            raise Invalid('invalid compressed object') from exc
70
        if len(raw) > MAX_OBJECT:
71
            raise Invalid('decompressed object exceeds reader bound')
72
        return raw
73
74
    def database_snapshot():
75
        if any(Path(str(db_path) + suffix).exists() for suffix in ('-wal', '-journal')):
76
            raise Unavailable('active SQLite journal; quiescent database required')
77
        with db_path.open('rb') as stream:
78
            content = stream.read(MAX_DATABASE + 1)
79
        if len(content) > MAX_DATABASE:
80
            raise Unavailable('database exceeds independent reader bound')
81
        return content
82
83
    # Snapshot only quiescent databases. immutable=1 is safe on this private copy,
84
    # never on a live WAL database; the original bytes are checked again below.
85
    snapshot = database_snapshot()
86
    database_sha256 = hashlib.sha256(snapshot).hexdigest()
87
    with tempfile.TemporaryDirectory(prefix='oakbench-reader-db-') as temp:
88
        snapshot_path = Path(temp) / 'snapshot.db'
89
        snapshot_path.write_bytes(snapshot)
90
        del snapshot
91
        return _read_snapshot(snapshot_path, database_sha256, database_snapshot,
92
                              destination, deadline, hashed, decode, blake3)
93
94
95
def _read_snapshot(snapshot_path, database_sha256, database_snapshot, destination,
96
                   deadline, hashed, decode, blake3):
97
    total = 0
98
    entries_seen = 0
99
    with sqlite3.connect(snapshot_path.as_uri() + '?mode=ro&immutable=1', uri=True, timeout=5) as db:
100
        db.execute('PRAGMA query_only=ON')
101
        db.execute('PRAGMA trusted_schema=OFF')
102
        db.set_progress_handler(lambda: int(time.monotonic() > deadline), 1000)
103
        db.execute('BEGIN')
104
105
        def metadata(key):
106
            row = db.execute('SELECT value FROM metadata WHERE key=? AND length(value)<=4096', (key,)).fetchone()
107
            return row[0] if row else None
108
109
        if metadata('hash_format') not in (None, 'v1'):
110
            raise Unavailable('unsupported Oak hash format')
111
        head = metadata('head')
112
        branch = metadata('current_branch')
113
        if not head or not branch:
114
            raise Unavailable('committed HEAD unavailable')
115
        commit = db.execute('SELECT branch_name,parent_hash,merge_parent_hash,manifest_hash,author,message,timestamp FROM commits WHERE hash=? AND length(CAST(branch_name AS BLOB))<=4096 AND coalesce(length(CAST(parent_hash AS BLOB)),0)<=64 AND coalesce(length(CAST(merge_parent_hash AS BLOB)),0)<=64 AND length(CAST(manifest_hash AS BLOB))=64 AND length(CAST(author AS BLOB))<=4096 AND coalesce(length(CAST(message AS BLOB)),0)<=1048576 AND length(CAST(timestamp AS BLOB))<=128', (head,)).fetchone()
116
        if not commit or any(v is not None and (not isinstance(v, str) or len(v) > MAX_OBJECT) for v in commit):
117
            raise Invalid('committed HEAD object missing or invalid')
118
        # Oak v1 stores the same RFC3339 timestamp text used in its hash preimage.
119
        hashed('\n'.join(v or '' for v in commit).encode(), head)
120
        root_hash = commit[3]
121
122
        def blob(object_hash):
123
            row = db.execute('SELECT content,size,codec FROM blobs WHERE hash=? AND length(content)<=?', (object_hash, MAX_OBJECT)).fetchone()
124
            if row is None:
125
                if db.execute('SELECT 1 FROM blobs WHERE hash=?', (object_hash,)).fetchone():
126
                    raise Invalid('stored blob exceeds reader bound')
127
                # Chunk-only hydration is not yet qualified by this reader.
128
                raise Unavailable('complete blob unavailable; chunk-only storage unsupported')
129
            content, size, codec = row
130
            if type(size) is not int or not 0 <= size <= MAX_OBJECT:
131
                raise Invalid('blob size exceeds reader bound')
132
            raw = decode(content, codec)
133
            if len(raw) != size:
134
                raise Invalid('blob size mismatch')
135
            return hashed(raw, object_hash)
136
137
        def visit(tree_hash, relative, ancestors):
138
            nonlocal total, entries_seen
139
            if len(ancestors) >= 32 or tree_hash in ancestors or time.monotonic() > deadline:
140
                raise Invalid('tree depth, cycle or time bound exceeded')
141
            if tree_hash == blake3.blake3(b'').hexdigest():
142
                return
143
            row = db.execute('SELECT content FROM trees WHERE hash=? AND (content IS NULL OR length(content)<=?)', (tree_hash, MAX_OBJECT)).fetchone()
144
            if row is None:
145
                raise Invalid('committed tree missing or oversized')
146
            if row[0] is None:
147
                if db.execute('SELECT 1 FROM tree_entries WHERE tree_hash=? AND (length(CAST(name AS BLOB))>4096 OR length(CAST(hash AS BLOB))!=64 OR length(CAST(kind AS BLOB))>4 OR length(CAST(mode AS BLOB))>10) LIMIT 1', (tree_hash,)).fetchone():
148
                    raise Invalid('stored tree entry exceeds reader bound')
149
                stored = db.execute('SELECT kind,name,hash,mode FROM tree_entries WHERE tree_hash=? ORDER BY name LIMIT ?', (tree_hash, MAX_ENTRIES + 1)).fetchall()
150
                raw = '\n'.join('\t'.join((kind, name, obj, 'tree' if kind == 'tree' else mode)) for kind, name, obj, mode in stored).encode()
151
            else:
152
                raw = decode(row[0], 1)
153
            hashed(raw, tree_hash)
154
            try:
155
                lines = raw.decode('utf-8').split('\n') if raw else []
156
            except UnicodeError as exc:
157
                raise Invalid('invalid tree encoding') from exc
158
            previous = None
159
            for line in lines:
160
                parts = line.split('\t')
161
                if len(parts) != 4:
162
                    raise Invalid('invalid tree entry')
163
                kind, name, object_hash, mode = parts
164
                if (not name or name in ('.', '..', '.oak', '.git') or any(c in name for c in '/\\\x00\n\r')
165
                        or (previous is not None and name <= previous)):
166
                    raise Invalid('unsafe or unordered tree path')
167
                previous = name
168
                entries_seen += 1
169
                if entries_seen > MAX_ENTRIES:
170
                    raise Invalid('tree entry bound exceeded')
171
                path = relative / name
172
                if (destination / path).exists() or (destination / path).is_symlink():
173
                    raise Unavailable('filesystem path collision on reader host')
174
                if kind == 'tree' and mode == 'tree':
175
                    (destination / path).mkdir()
176
                    visit(object_hash, path, ancestors | {tree_hash})
177
                elif kind == 'blob' and mode in ('regular', 'executable'):
178
                    content = blob(object_hash)
179
                    total += len(content)
180
                    if total > MAX_TOTAL:
181
                        raise Invalid('total payload bound exceeded')
182
                    target = destination / path
183
                    target.write_bytes(content)
184
                    target.chmod(0o755 if mode == 'executable' else 0o644)
185
                else:
186
                    raise Unavailable('unsupported tree entry kind or mode')
187
        visit(root_hash, Path(), set())
188
        db.commit()
189
        if hashlib.sha256(database_snapshot()).hexdigest() != database_sha256:
190
            raise Unavailable('database changed during independent read')
191
    return {'head': head, 'database_sha256': database_sha256, 'reader_version': READER_VERSION,
192
            'reader_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
193
            'blake3_version': importlib.metadata.version('blake3'),
194
            'zstandard_version': importlib.metadata.version('zstandard')}
195
196
197
def main():
198
    try:
199
        if len(sys.argv) != 3:
200
            raise Invalid('usage: oak_sqlite_reader.py REPOSITORY NEW_DESTINATION')
201
        result = read_committed_tree(Path(sys.argv[1]), Path(sys.argv[2]))
202
        code = 0
203
    except Unavailable as exc:
204
        result, code = {'unavailable': str(exc)}, 77
205
    except Invalid as exc:
206
        result, code = {'corrupt': True, 'reason': str(exc)}, 1
207
    except (OSError, ValueError, TypeError, sqlite3.Error) as exc:
208
        result, code = {'unavailable': 'reader instrument error: ' + str(exc)}, 77
209
    print(json.dumps(result, sort_keys=True))
210
    return code
211
212
if __name__ == '__main__':
213
    raise SystemExit(main())