| 1 | """Subject-kind plugins: the "any VCS can join" structural layer.
|
| 2 |
|
| 3 | Each file in config/subject_kinds/<kind>.json declares, per subject kind,
|
| 4 | what the harness needs to measure it: the argv template per operation
|
| 5 | (null = capability gap, the caller emits a skip row
|
| 6 | "capability_gap:<kind>:<operation>" — never a substituted command),
|
| 7 | snapshot call-count semantics, lock-hint patterns, integrity commands,
|
| 8 | workspace model, server-error patterns, and the remote env var.
|
| 9 |
|
| 10 | This layer is a PARALLEL READ of existing configuration, not a rewrite:
|
| 11 | existing lanes keep loading config/command_semantics.json through
|
| 12 | oakbench.command_semantics, byte-for-byte unchanged. parity_report() is the
|
| 13 | proof mechanism: it compares a plugin JSON against command_semantics.json
|
| 14 | for that kind and returns the list of mismatches. An empty list means the
|
| 15 | plugin layer is measurement-identical to the source of truth, so adding a
|
| 16 | new VCS (jujutsu, sapling, pijul) is one JSON file plus one subjects.toml
|
| 17 | entry plus golden output samples — zero lane-code changes.
|
| 18 | """
|
| 19 |
|
| 20 | from __future__ import annotations
|
| 21 |
|
| 22 | import json
|
| 23 | from dataclasses import dataclass, field
|
| 24 | from pathlib import Path
|
| 25 | from typing import Any
|
| 26 |
|
| 27 | ROOT = Path(__file__).resolve().parents[2]
|
| 28 | DEFAULT_DIR = ROOT / "config" / "subject_kinds"
|
| 29 | DEFAULT_COMMAND_SEMANTICS = ROOT / "config" / "command_semantics.json"
|
| 30 | SCHEMA_VERSION = 1
|
| 31 |
|
| 32 | # Operations whose value is a per-track mapping rather than a flat argv
|
| 33 | # template, mirroring config/command_semantics.json harness_tracks shape.
|
| 34 | TRACKED_OPERATIONS = ("status", "diff")
|
| 35 | DEFAULT_TRACK = "agent-default"
|
| 36 |
|
| 37 |
|
| 38 | @dataclass(frozen=True)
|
| 39 | class SubjectKind:
|
| 40 | """One subject kind's measurement declaration, loaded from JSON."""
|
| 41 |
|
| 42 | kind: str
|
| 43 | schema_version: int
|
| 44 | commands: dict[str, Any]
|
| 45 | snapshot_call_semantics: dict[str, Any]
|
| 46 | lock_hint_patterns: tuple[str, ...]
|
| 47 | integrity_command: tuple[tuple[str, ...], ...] | None
|
| 48 | workspace_model: str
|
| 49 | server_error_patterns: tuple[str, ...]
|
| 50 | remote_env_var: str | None
|
| 51 | source_path: Path = field(compare=False, default=Path())
|
| 52 |
|
| 53 | def command_for(self, operation: str, track: str = DEFAULT_TRACK) -> list[str] | None:
|
| 54 | """Argv template for an operation, or None for a capability gap.
|
| 55 |
|
| 56 | None covers both an explicit null and an absent operation; callers
|
| 57 | emit a skip row "capability_gap:<kind>:<operation>" for either.
|
| 58 | """
|
| 59 | value = self.commands.get(operation)
|
| 60 | if isinstance(value, dict):
|
| 61 | value = value.get(track)
|
| 62 | if value is None:
|
| 63 | return None
|
| 64 | return [str(part) for part in value]
|
| 65 |
|
| 66 | def skip_reason(self, operation: str) -> str:
|
| 67 | return f"capability_gap:{self.kind}:{operation}"
|
| 68 |
|
| 69 |
|
| 70 | def _fail(path: Path, message: str) -> ValueError:
|
| 71 | return ValueError(f"{path}: {message}")
|
| 72 |
|
| 73 |
|
| 74 | def _validate_argv(path: Path, operation: str, value: Any) -> None:
|
| 75 | if value is None:
|
| 76 | return
|
| 77 | if not isinstance(value, list) or not value or not all(isinstance(part, str) for part in value):
|
| 78 | raise _fail(path, f"commands.{operation} must be null or a non-empty list of strings, got {value!r}")
|
| 79 |
|
| 80 |
|
| 81 | def _validate(path: Path, data: dict[str, Any]) -> None:
|
| 82 | if data.get("schema_version") != SCHEMA_VERSION:
|
| 83 | raise _fail(path, f"schema_version must be {SCHEMA_VERSION}, got {data.get('schema_version')!r}")
|
| 84 | kind = data.get("kind")
|
| 85 | if not isinstance(kind, str) or not kind:
|
| 86 | raise _fail(path, "kind must be a non-empty string")
|
| 87 | if kind != path.stem:
|
| 88 | raise _fail(path, f"kind {kind!r} must match filename stem {path.stem!r}")
|
| 89 |
|
| 90 | commands = data.get("commands")
|
| 91 | if not isinstance(commands, dict) or not commands:
|
| 92 | raise _fail(path, "commands must be a non-empty object of operation -> argv template | null")
|
| 93 | for operation, value in commands.items():
|
| 94 | if isinstance(value, dict):
|
| 95 | for track, tracked in value.items():
|
| 96 | _validate_argv(path, f"{operation}.{track}", tracked)
|
| 97 | else:
|
| 98 | _validate_argv(path, operation, value)
|
| 99 |
|
| 100 | snapshot = data.get("snapshot_call_semantics")
|
| 101 | if not isinstance(snapshot, dict):
|
| 102 | raise _fail(path, "snapshot_call_semantics must be an object")
|
| 103 | if not isinstance(snapshot.get("stage_required"), bool):
|
| 104 | raise _fail(path, "snapshot_call_semantics.stage_required must be a boolean")
|
| 105 | if not isinstance(snapshot.get("calls_per_snapshot"), int):
|
| 106 | raise _fail(path, "snapshot_call_semantics.calls_per_snapshot must be an integer")
|
| 107 |
|
| 108 | for key in ("lock_hint_patterns", "server_error_patterns"):
|
| 109 | value = data.get(key)
|
| 110 | if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
| 111 | raise _fail(path, f"{key} must be a list of strings")
|
| 112 |
|
| 113 | integrity = data.get("integrity_command")
|
| 114 | if integrity is not None:
|
| 115 | ok = isinstance(integrity, list) and integrity and all(
|
| 116 | isinstance(cmd, list) and cmd and all(isinstance(part, str) for part in cmd) for cmd in integrity
|
| 117 | )
|
| 118 | if not ok:
|
| 119 | raise _fail(path, "integrity_command must be null or a non-empty list of argv lists")
|
| 120 |
|
| 121 | model = data.get("workspace_model")
|
| 122 | if not isinstance(model, str) or not model:
|
| 123 | raise _fail(path, "workspace_model must be a non-empty string")
|
| 124 |
|
| 125 | remote = data.get("remote_env_var")
|
| 126 | if remote is not None and (not isinstance(remote, str) or not remote):
|
| 127 | raise _fail(path, "remote_env_var must be null or a non-empty string")
|
| 128 |
|
| 129 |
|
| 130 | def available_kinds(root: Path = DEFAULT_DIR) -> list[str]:
|
| 131 | """Sorted kind names with a plugin JSON under config/subject_kinds/."""
|
| 132 | if not root.is_dir():
|
| 133 | return []
|
| 134 | return sorted(path.stem for path in root.glob("*.json"))
|
| 135 |
|
| 136 |
|
| 137 | def load_subject_kind(kind: str, root: Path = DEFAULT_DIR) -> SubjectKind:
|
| 138 | path = root / f"{kind}.json"
|
| 139 | if not path.is_file():
|
| 140 | known = ", ".join(available_kinds(root)) or "none"
|
| 141 | raise ValueError(f"unknown subject kind {kind!r}: no plugin at {path} (available: {known})")
|
| 142 | try:
|
| 143 | data = json.loads(path.read_text())
|
| 144 | except json.JSONDecodeError as exc:
|
| 145 | raise _fail(path, f"invalid JSON: {exc}") from exc
|
| 146 | if not isinstance(data, dict):
|
| 147 | raise _fail(path, "plugin root must be a JSON object")
|
| 148 | _validate(path, data)
|
| 149 | integrity = data["integrity_command"]
|
| 150 | return SubjectKind(
|
| 151 | kind=data["kind"],
|
| 152 | schema_version=data["schema_version"],
|
| 153 | commands=data["commands"],
|
| 154 | snapshot_call_semantics=data["snapshot_call_semantics"],
|
| 155 | lock_hint_patterns=tuple(data["lock_hint_patterns"]),
|
| 156 | integrity_command=tuple(tuple(cmd) for cmd in integrity) if integrity is not None else None,
|
| 157 | workspace_model=data["workspace_model"],
|
| 158 | server_error_patterns=tuple(data["server_error_patterns"]),
|
| 159 | remote_env_var=data["remote_env_var"],
|
| 160 | source_path=path,
|
| 161 | )
|
| 162 |
|
| 163 |
|
| 164 | def command_for(kind: str, operation: str, track: str = DEFAULT_TRACK, root: Path = DEFAULT_DIR) -> list[str] | None:
|
| 165 | """Argv template for kind+operation, or None = capability gap (skip row)."""
|
| 166 | return load_subject_kind(kind, root).command_for(operation, track)
|
| 167 |
|
| 168 |
|
| 169 | def parity_report(
|
| 170 | kind: str,
|
| 171 | command_semantics_path: Path = DEFAULT_COMMAND_SEMANTICS,
|
| 172 | root: Path = DEFAULT_DIR,
|
| 173 | ) -> list[str]:
|
| 174 | """Mismatches between the plugin JSON and command_semantics.json for kind.
|
| 175 |
|
| 176 | Empty list = the plugin layer is measurement-identical to the existing
|
| 177 | source of truth for every operation the harness runs. This is the
|
| 178 | row_parity proof for the config layer: identical commands in, identical
|
| 179 | rows out, so lanes can later read either source without drift.
|
| 180 | """
|
| 181 | plugin = load_subject_kind(kind, root)
|
| 182 | data = json.loads(Path(command_semantics_path).read_text())
|
| 183 | tracks = data.get("harness_tracks", {})
|
| 184 | mismatches: list[str] = []
|
| 185 | if kind not in tracks:
|
| 186 | return [f"kind_not_in_command_semantics:{kind} ({command_semantics_path})"]
|
| 187 | semantics = tracks[kind]
|
| 188 |
|
| 189 | for operation in sorted(semantics):
|
| 190 | expected = semantics[operation]
|
| 191 | if operation not in plugin.commands:
|
| 192 | mismatches.append(f"missing_operation:{kind}:{operation} (in command_semantics.json, not in plugin)")
|
| 193 | continue
|
| 194 | actual = plugin.commands[operation]
|
| 195 | if actual != expected:
|
| 196 | mismatches.append(
|
| 197 | f"command_mismatch:{kind}:{operation} plugin={actual!r} command_semantics={expected!r}"
|
| 198 | )
|
| 199 | for operation in sorted(set(plugin.commands) - set(semantics)):
|
| 200 | mismatches.append(
|
| 201 | f"extra_operation:{kind}:{operation} (in plugin, not in command_semantics.json harness_tracks)"
|
| 202 | )
|
| 203 |
|
| 204 | # Snapshot call-count semantics derive from the stage entry: a non-null
|
| 205 | # stage means the kind pays stage+commit as two recorded calls
|
| 206 | # (oakbench.command_semantics.snapshot_commands).
|
| 207 | stage = semantics.get("stage")
|
| 208 | expected_stage_required = stage is not None
|
| 209 | expected_calls = 2 if stage is not None else 1
|
| 210 | snapshot = plugin.snapshot_call_semantics
|
| 211 | if snapshot.get("stage_required") != expected_stage_required:
|
| 212 | mismatches.append(
|
| 213 | f"snapshot_mismatch:{kind}:stage_required plugin={snapshot.get('stage_required')!r} "
|
| 214 | f"derived_from_command_semantics={expected_stage_required!r}"
|
| 215 | )
|
| 216 | if snapshot.get("calls_per_snapshot") != expected_calls:
|
| 217 | mismatches.append(
|
| 218 | f"snapshot_mismatch:{kind}:calls_per_snapshot plugin={snapshot.get('calls_per_snapshot')!r} "
|
| 219 | f"derived_from_command_semantics={expected_calls!r}"
|
| 220 | )
|
| 221 | return mismatches
|