Viewing files at commit e60fa60d3a02Commit detailsView latest

scripts/oakbench/results.py 132 lines · 5.6 KB · python
1
"""Result persistence: one write path for every lane.
2
3
Writes the timestamped raw JSONL (the durable source of truth) first, then
4
validates rows against the lane's row contract, and only then writes the
5
Markdown summary and the latest.* copies. Rows are always persisted before a
6
contract violation is raised, so a contract bug never destroys collected
7
measurements; it fails the run loudly instead (data first, then the alarm).
8
The latest.* pointers are written strictly after validation so they never
9
point at rows that violate the contract.
10
"""
11
12
from __future__ import annotations
13
14
import json
15
import os
16
import shutil
17
from pathlib import Path
18
from typing import Any, Optional
19
20
from oakbench.rows import RowContractViolation, validate_rows
21
22
try:
23
    import fcntl
24
except ImportError:  # pragma: no cover - platforms without fcntl
25
    fcntl = None  # type: ignore[assignment]
26
27
28
def _write_all(fd: int, data: bytes) -> None:
29
    view = memoryview(data)
30
    written_total = 0
31
    while written_total < len(view):
32
        written = os.write(fd, view[written_total:])
33
        if written == 0:
34
            raise OSError("os.write returned 0 bytes")
35
        written_total += written
36
37
38
def _atomic_copy(src: Path, dst: Path) -> None:
39
    tmp = dst.with_name(f"{dst.name}.tmp-{os.getpid()}")
40
    try:
41
        shutil.copyfile(src, tmp)
42
        os.replace(tmp, dst)
43
    finally:
44
        if tmp.exists():
45
            tmp.unlink()
46
47
48
class ResultsStore:
49
    def __init__(
50
        self, results_dir: Path, lane: str, filename_suffix: str = "", flush_partial: bool = False
51
    ) -> None:
52
        """filename_suffix distinguishes lanes sharing one results dir:
53
        suffix "mount" writes <id>.mount.jsonl and latest.mount.jsonl so the
54
        mount lane never clobbers the core lane's latest.jsonl.
55
56
        flush_partial (opt-in, default off so other lanes are untouched)
57
        enables crash insurance for long campaigns: completed rows are appended
58
        incrementally to <id><suffix>.partial.jsonl via append_partial(), and
59
        write() finalizes by rewriting the canonical file and removing the
60
        partial only after the canonical write succeeded. A campaign killed
61
        hours in keeps every flushed row."""
62
        self.results_dir = results_dir
63
        self.lane = lane
64
        self.suffix = f".{filename_suffix}" if filename_suffix else ""
65
        self.flush_partial = flush_partial
66
67
    def partial_path(self, bench_id: str) -> Path:
68
        return self.results_dir / f"{bench_id}{self.suffix}.partial.jsonl"
69
70
    def append_partial(self, bench_id: str, rows: list[dict[str, Any]]) -> Path:
71
        """Append completed rows to the partial JSONL (crash insurance).
72
73
        Appends are flock-guarded os.write() loops of complete buffers (the
74
        oakbench/vcs_shim.py sidecar pattern): with O_APPEND plus the
75
        exclusive lock, concurrent flushers cannot interleave rows even if the
76
        OS accepts only part of the buffer per syscall. Lines use the same
77
        json.dumps(sort_keys=True) encoding as the canonical file, so a
78
        surviving partial is directly consumable by row readers.
79
        """
80
        if not self.flush_partial:
81
            raise RuntimeError(
82
                "partial flushing is disabled; construct ResultsStore(..., flush_partial=True)"
83
            )
84
        self.results_dir.mkdir(parents=True, exist_ok=True)
85
        path = self.partial_path(bench_id)
86
        data = "".join(json.dumps(row, sort_keys=True) + "\n" for row in rows).encode("utf-8")
87
        fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o644)
88
        try:
89
            if fcntl is not None:
90
                fcntl.flock(fd, fcntl.LOCK_EX)
91
            _write_all(fd, data)
92
        finally:
93
            try:
94
                if fcntl is not None:
95
                    fcntl.flock(fd, fcntl.LOCK_UN)
96
            finally:
97
                os.close(fd)
98
        return path
99
100
    def write(
101
        self, bench_id: str, rows: list[dict[str, Any]], summary_text: Optional[str] = None
102
    ) -> tuple[Path, Optional[Path]]:
103
        self.results_dir.mkdir(parents=True, exist_ok=True)
104
        raw_path = self.results_dir / f"{bench_id}{self.suffix}.jsonl"
105
        summary_path: Optional[Path] = None
106
        with raw_path.open("w") as fh:
107
            for row in rows:
108
                fh.write(json.dumps(row, sort_keys=True) + "\n")
109
110
        # The raw {bench_id} file is intentionally written BEFORE validation
111
        # (the exception message points emitter debugging at it), but the
112
        # latest.* pointers must only ever reference contract-clean rows, so
113
        # validation happens here — after the raw write, before any latest copy.
114
        errors = validate_rows(rows, self.lane)
115
        if errors:
116
            preview = "\n  ".join(errors[:20])
117
            more = f"\n  ... and {len(errors) - 20} more" if len(errors) > 20 else ""
118
            raise RowContractViolation(
119
                f"Row contract violation ({self.lane} lane). Rows were written to {raw_path} "
120
                f"but the emitter has a bug:\n  {preview}{more}"
121
            )
122
123
        _atomic_copy(raw_path, self.results_dir / f"latest{self.suffix}.jsonl")
124
        if summary_text is not None:
125
            summary_path = self.results_dir / f"{bench_id}{self.suffix}.summary.md"
126
            summary_path.write_text(summary_text)
127
            _atomic_copy(summary_path, self.results_dir / f"latest{self.suffix}.summary.md")
128
        if self.flush_partial:
129
            # Finalize only after the canonical write AND validation succeeded;
130
            # on any failure the partial survives as the crash-insurance copy.
131
            self.partial_path(bench_id).unlink(missing_ok=True)
132
        return raw_path, summary_path