Viewing files at commit ebd567552629Commit detailsView latest

Browse files
scripts/oakbench/subject_kinds.py 221 lines · 9.4 KB · python
1
"""Subject-kind plugins: the "any VCS can join" structural layer.
2
3
Each file in config/subject_kinds/<kind>.json declares, per subject kind,
4
what the harness needs to measure it: the argv template per operation
5
(null = capability gap, the caller emits a skip row
6
"capability_gap:<kind>:<operation>" — never a substituted command),
7
snapshot call-count semantics, lock-hint patterns, integrity commands,
8
workspace model, server-error patterns, and the remote env var.
9
10
This layer is a PARALLEL READ of existing configuration, not a rewrite:
11
existing lanes keep loading config/command_semantics.json through
12
oakbench.command_semantics, byte-for-byte unchanged. parity_report() is the
13
proof mechanism: it compares a plugin JSON against command_semantics.json
14
for that kind and returns the list of mismatches. An empty list means the
15
plugin layer is measurement-identical to the source of truth, so adding a
16
new VCS (jujutsu, sapling, pijul) is one JSON file plus one subjects.toml
17
entry plus golden output samples — zero lane-code changes.
18
"""
19
20
from __future__ import annotations
21
22
import json
23
from dataclasses import dataclass, field
24
from pathlib import Path
25
from typing import Any
26
27
ROOT = Path(__file__).resolve().parents[2]
28
DEFAULT_DIR = ROOT / "config" / "subject_kinds"
29
DEFAULT_COMMAND_SEMANTICS = ROOT / "config" / "command_semantics.json"
30
SCHEMA_VERSION = 1
31
32
# Operations whose value is a per-track mapping rather than a flat argv
33
# template, mirroring config/command_semantics.json harness_tracks shape.
34
TRACKED_OPERATIONS = ("status", "diff")
35
DEFAULT_TRACK = "agent-default"
36
37
38
@dataclass(frozen=True)
39
class SubjectKind:
40
    """One subject kind's measurement declaration, loaded from JSON."""
41
42
    kind: str
43
    schema_version: int
44
    commands: dict[str, Any]
45
    snapshot_call_semantics: dict[str, Any]
46
    lock_hint_patterns: tuple[str, ...]
47
    integrity_command: tuple[tuple[str, ...], ...] | None
48
    workspace_model: str
49
    server_error_patterns: tuple[str, ...]
50
    remote_env_var: str | None
51
    source_path: Path = field(compare=False, default=Path())
52
53
    def command_for(self, operation: str, track: str = DEFAULT_TRACK) -> list[str] | None:
54
        """Argv template for an operation, or None for a capability gap.
55
56
        None covers both an explicit null and an absent operation; callers
57
        emit a skip row "capability_gap:<kind>:<operation>" for either.
58
        """
59
        value = self.commands.get(operation)
60
        if isinstance(value, dict):
61
            value = value.get(track)
62
        if value is None:
63
            return None
64
        return [str(part) for part in value]
65
66
    def skip_reason(self, operation: str) -> str:
67
        return f"capability_gap:{self.kind}:{operation}"
68
69
70
def _fail(path: Path, message: str) -> ValueError:
71
    return ValueError(f"{path}: {message}")
72
73
74
def _validate_argv(path: Path, operation: str, value: Any) -> None:
75
    if value is None:
76
        return
77
    if not isinstance(value, list) or not value or not all(isinstance(part, str) for part in value):
78
        raise _fail(path, f"commands.{operation} must be null or a non-empty list of strings, got {value!r}")
79
80
81
def _validate(path: Path, data: dict[str, Any]) -> None:
82
    if data.get("schema_version") != SCHEMA_VERSION:
83
        raise _fail(path, f"schema_version must be {SCHEMA_VERSION}, got {data.get('schema_version')!r}")
84
    kind = data.get("kind")
85
    if not isinstance(kind, str) or not kind:
86
        raise _fail(path, "kind must be a non-empty string")
87
    if kind != path.stem:
88
        raise _fail(path, f"kind {kind!r} must match filename stem {path.stem!r}")
89
90
    commands = data.get("commands")
91
    if not isinstance(commands, dict) or not commands:
92
        raise _fail(path, "commands must be a non-empty object of operation -> argv template | null")
93
    for operation, value in commands.items():
94
        if isinstance(value, dict):
95
            for track, tracked in value.items():
96
                _validate_argv(path, f"{operation}.{track}", tracked)
97
        else:
98
            _validate_argv(path, operation, value)
99
100
    snapshot = data.get("snapshot_call_semantics")
101
    if not isinstance(snapshot, dict):
102
        raise _fail(path, "snapshot_call_semantics must be an object")
103
    if not isinstance(snapshot.get("stage_required"), bool):
104
        raise _fail(path, "snapshot_call_semantics.stage_required must be a boolean")
105
    if not isinstance(snapshot.get("calls_per_snapshot"), int):
106
        raise _fail(path, "snapshot_call_semantics.calls_per_snapshot must be an integer")
107
108
    for key in ("lock_hint_patterns", "server_error_patterns"):
109
        value = data.get(key)
110
        if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
111
            raise _fail(path, f"{key} must be a list of strings")
112
113
    integrity = data.get("integrity_command")
114
    if integrity is not None:
115
        ok = isinstance(integrity, list) and integrity and all(
116
            isinstance(cmd, list) and cmd and all(isinstance(part, str) for part in cmd) for cmd in integrity
117
        )
118
        if not ok:
119
            raise _fail(path, "integrity_command must be null or a non-empty list of argv lists")
120
121
    model = data.get("workspace_model")
122
    if not isinstance(model, str) or not model:
123
        raise _fail(path, "workspace_model must be a non-empty string")
124
125
    remote = data.get("remote_env_var")
126
    if remote is not None and (not isinstance(remote, str) or not remote):
127
        raise _fail(path, "remote_env_var must be null or a non-empty string")
128
129
130
def available_kinds(root: Path = DEFAULT_DIR) -> list[str]:
131
    """Sorted kind names with a plugin JSON under config/subject_kinds/."""
132
    if not root.is_dir():
133
        return []
134
    return sorted(path.stem for path in root.glob("*.json"))
135
136
137
def load_subject_kind(kind: str, root: Path = DEFAULT_DIR) -> SubjectKind:
138
    path = root / f"{kind}.json"
139
    if not path.is_file():
140
        known = ", ".join(available_kinds(root)) or "none"
141
        raise ValueError(f"unknown subject kind {kind!r}: no plugin at {path} (available: {known})")
142
    try:
143
        data = json.loads(path.read_text())
144
    except json.JSONDecodeError as exc:
145
        raise _fail(path, f"invalid JSON: {exc}") from exc
146
    if not isinstance(data, dict):
147
        raise _fail(path, "plugin root must be a JSON object")
148
    _validate(path, data)
149
    integrity = data["integrity_command"]
150
    return SubjectKind(
151
        kind=data["kind"],
152
        schema_version=data["schema_version"],
153
        commands=data["commands"],
154
        snapshot_call_semantics=data["snapshot_call_semantics"],
155
        lock_hint_patterns=tuple(data["lock_hint_patterns"]),
156
        integrity_command=tuple(tuple(cmd) for cmd in integrity) if integrity is not None else None,
157
        workspace_model=data["workspace_model"],
158
        server_error_patterns=tuple(data["server_error_patterns"]),
159
        remote_env_var=data["remote_env_var"],
160
        source_path=path,
161
    )
162
163
164
def command_for(kind: str, operation: str, track: str = DEFAULT_TRACK, root: Path = DEFAULT_DIR) -> list[str] | None:
165
    """Argv template for kind+operation, or None = capability gap (skip row)."""
166
    return load_subject_kind(kind, root).command_for(operation, track)
167
168
169
def parity_report(
170
    kind: str,
171
    command_semantics_path: Path = DEFAULT_COMMAND_SEMANTICS,
172
    root: Path = DEFAULT_DIR,
173
) -> list[str]:
174
    """Mismatches between the plugin JSON and command_semantics.json for kind.
175
176
    Empty list = the plugin layer is measurement-identical to the existing
177
    source of truth for every operation the harness runs. This is the
178
    row_parity proof for the config layer: identical commands in, identical
179
    rows out, so lanes can later read either source without drift.
180
    """
181
    plugin = load_subject_kind(kind, root)
182
    data = json.loads(Path(command_semantics_path).read_text())
183
    tracks = data.get("harness_tracks", {})
184
    mismatches: list[str] = []
185
    if kind not in tracks:
186
        return [f"kind_not_in_command_semantics:{kind} ({command_semantics_path})"]
187
    semantics = tracks[kind]
188
189
    for operation in sorted(semantics):
190
        expected = semantics[operation]
191
        if operation not in plugin.commands:
192
            mismatches.append(f"missing_operation:{kind}:{operation} (in command_semantics.json, not in plugin)")
193
            continue
194
        actual = plugin.commands[operation]
195
        if actual != expected:
196
            mismatches.append(
197
                f"command_mismatch:{kind}:{operation} plugin={actual!r} command_semantics={expected!r}"
198
            )
199
    for operation in sorted(set(plugin.commands) - set(semantics)):
200
        mismatches.append(
201
            f"extra_operation:{kind}:{operation} (in plugin, not in command_semantics.json harness_tracks)"
202
        )
203
204
    # Snapshot call-count semantics derive from the stage entry: a non-null
205
    # stage means the kind pays stage+commit as two recorded calls
206
    # (oakbench.command_semantics.snapshot_commands).
207
    stage = semantics.get("stage")
208
    expected_stage_required = stage is not None
209
    expected_calls = 2 if stage is not None else 1
210
    snapshot = plugin.snapshot_call_semantics
211
    if snapshot.get("stage_required") != expected_stage_required:
212
        mismatches.append(
213
            f"snapshot_mismatch:{kind}:stage_required plugin={snapshot.get('stage_required')!r} "
214
            f"derived_from_command_semantics={expected_stage_required!r}"
215
        )
216
    if snapshot.get("calls_per_snapshot") != expected_calls:
217
        mismatches.append(
218
            f"snapshot_mismatch:{kind}:calls_per_snapshot plugin={snapshot.get('calls_per_snapshot')!r} "
219
            f"derived_from_command_semantics={expected_calls!r}"
220
        )
221
    return mismatches