| 1 | """Information content of VCS output: bytes saved vs information lost.
|
| 2 |
|
| 3 | Compact output is only a win if the agent can still act on it. These oracles
|
| 4 | score a status/diff output against the fixture's ground-truth changed file
|
| 5 | list: information_recall is the fraction of actually-changed paths an agent
|
| 6 | (or a script) could recover from the bytes it was shown. A subject that emits
|
| 7 | half the bytes but names half the files has not saved anything — it has moved
|
| 8 | the cost to a follow-up command.
|
| 9 |
|
| 10 | The second oracle is pipe compatibility: scripts and agents pipe diff output
|
| 11 | into grep/patch/apply tooling that expects unified-diff structure. A diff
|
| 12 | output without recognizable file headers and hunk markers silently breaks that
|
| 13 | ecosystem; the probe records the structure so a format change is a visible
|
| 14 | metric regression, not a downstream surprise.
|
| 15 | """
|
| 16 |
|
| 17 | from __future__ import annotations
|
| 18 |
|
| 19 | import re
|
| 20 | from typing import Any, Iterable
|
| 21 |
|
| 22 | STATUS_LINE = re.compile(r"^(?:(?! )[ MADRCU?!]{2}|[MADRCU?!])\s+(.+)$")
|
| 23 | UNIFIED_HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@", re.MULTILINE)
|
| 24 | UNIFIED_OLD_FILE = re.compile(r"^--- ", re.MULTILINE)
|
| 25 | UNIFIED_NEW_FILE = re.compile(r"^\+\+\+ ", re.MULTILINE)
|
| 26 | GIT_DIFF_HEADER = re.compile(r"^diff --git ", re.MULTILINE)
|
| 27 | BINARY_NOTICE = re.compile(r"^Binary files? .* differ", re.MULTILINE | re.IGNORECASE)
|
| 28 |
|
| 29 |
|
| 30 | def recovered_paths(output_text: str, changed_paths: Iterable[str]) -> list[str]:
|
| 31 | """Ground-truth paths an agent could recover from this output.
|
| 32 |
|
| 33 | A path counts as recovered when its repo-relative form appears in the
|
| 34 | output (including git's a/ b/ prefixes). Substring matching is
|
| 35 | deliberately generous and subject-neutral: it asks "is the information
|
| 36 | present at all?", not "is it in a specific format".
|
| 37 | """
|
| 38 | recovered: list[str] = []
|
| 39 | for path in changed_paths:
|
| 40 | if path in output_text:
|
| 41 | recovered.append(path)
|
| 42 | return recovered
|
| 43 |
|
| 44 |
|
| 45 | def _normalize_reported_path(raw: str) -> str | None:
|
| 46 | path = raw.strip().strip("'\"`:,;()[]{}")
|
| 47 | if not path or path == "/dev/null" or path.startswith(("http://", "https://")):
|
| 48 | return None
|
| 49 | if path.startswith(("a/", "b/")):
|
| 50 | path = path[2:]
|
| 51 | name = path.rsplit("/", 1)[-1]
|
| 52 | root_file_like = "." in name or name in {"Dockerfile", "Makefile", "README", "LICENSE"}
|
| 53 | return path if "/" in path or root_file_like else None
|
| 54 |
|
| 55 |
|
| 56 | def _line_reported_paths(line: str) -> list[str]:
|
| 57 | stripped = line.strip()
|
| 58 | candidates: list[str] = []
|
| 59 | if stripped.startswith("diff --git "):
|
| 60 | candidates.extend(stripped.split()[2:4])
|
| 61 | elif stripped.startswith(("--- ", "+++ ")):
|
| 62 | candidates.append(stripped.split(maxsplit=1)[1].split("\t", 1)[0])
|
| 63 | elif stripped.lower().startswith("binary files "):
|
| 64 | parts = stripped.split()
|
| 65 | if len(parts) >= 5:
|
| 66 | candidates.extend([parts[2], parts[4]])
|
| 67 | elif " | " in stripped:
|
| 68 | candidates.append(stripped.split(" | ", 1)[0])
|
| 69 | else:
|
| 70 | status = STATUS_LINE.match(line)
|
| 71 | if status:
|
| 72 | payload = status.group(1)
|
| 73 | candidates.extend(part.strip() for part in payload.split(" -> "))
|
| 74 | normalized = [_normalize_reported_path(candidate) for candidate in candidates]
|
| 75 | return [path for path in normalized if path is not None]
|
| 76 |
|
| 77 |
|
| 78 | def reported_paths(output_text: str) -> list[str]:
|
| 79 | """Repo-like paths reported by VCS metadata lines.
|
| 80 |
|
| 81 | Diff bodies can contain dotted identifiers and version strings that look
|
| 82 | path-ish. Precision measures named changed files, so it only scans status
|
| 83 | rows, diff headers, file headers, binary notices, and diffstat rows.
|
| 84 | """
|
| 85 | paths: set[str] = set()
|
| 86 | in_unified_hunk = False
|
| 87 | for line in output_text.splitlines():
|
| 88 | stripped = line.strip()
|
| 89 | if stripped.startswith("diff --git "):
|
| 90 | in_unified_hunk = False
|
| 91 | elif stripped.startswith("@@ "):
|
| 92 | in_unified_hunk = True
|
| 93 | continue
|
| 94 | elif in_unified_hunk:
|
| 95 | continue
|
| 96 | paths.update(_line_reported_paths(line))
|
| 97 | return sorted(paths)
|
| 98 |
|
| 99 |
|
| 100 | def information_recall_fields(
|
| 101 | output_text: str,
|
| 102 | changed_paths: list[str],
|
| 103 | output_bytes: int,
|
| 104 | output_truncated: bool,
|
| 105 | ) -> dict[str, Any]:
|
| 106 | """Recall and byte-efficiency fields for one status/diff output.
|
| 107 |
|
| 108 | information_recall is null (unmeasured) when there is no ground truth to
|
| 109 | score against; bytes_per_changed_file_named is null when nothing was
|
| 110 | recovered — never zero (ADR-0002).
|
| 111 | """
|
| 112 | expected_paths = set(changed_paths)
|
| 113 | recovered = recovered_paths(output_text, changed_paths)
|
| 114 | reported = reported_paths(output_text)
|
| 115 | false_positives = [path for path in reported if path not in expected_paths]
|
| 116 | true_reported = [path for path in reported if path in expected_paths]
|
| 117 | expected = len(changed_paths)
|
| 118 | recall = round(len(recovered) / expected, 4) if expected else None
|
| 119 | precision = round(len(true_reported) / len(reported), 4) if expected and reported else None
|
| 120 | bytes_per_named = round(output_bytes / len(recovered), 1) if recovered else None
|
| 121 | return {
|
| 122 | "changed_files_expected": expected,
|
| 123 | "changed_files_recovered": len(recovered),
|
| 124 | "changed_files_reported": len(reported),
|
| 125 | "changed_files_false_positive": len(false_positives),
|
| 126 | "information_recall": recall,
|
| 127 | "information_precision": precision,
|
| 128 | "bytes_per_changed_file_named": bytes_per_named,
|
| 129 | "probe_output_truncated": output_truncated,
|
| 130 | "information_recall_source": "ground_truth_path_substring_match",
|
| 131 | "information_precision_source": "path_token_precision_against_changed_paths",
|
| 132 | }
|
| 133 |
|
| 134 |
|
| 135 | def unified_diff_compat_fields(output_text: str) -> dict[str, Any]:
|
| 136 | """Structure fields for piped-diff compatibility.
|
| 137 |
|
| 138 | pipe_compatible_unified means the output carries both unified file headers
|
| 139 | and hunk markers (or is an explicit binary notice), i.e. standard tooling
|
| 140 | (patch, git apply, diffstat, grep '^+') can consume it.
|
| 141 | """
|
| 142 | hunk_headers = len(UNIFIED_HUNK_HEADER.findall(output_text))
|
| 143 | old_headers = len(UNIFIED_OLD_FILE.findall(output_text))
|
| 144 | new_headers = len(UNIFIED_NEW_FILE.findall(output_text))
|
| 145 | git_headers = len(GIT_DIFF_HEADER.findall(output_text))
|
| 146 | binary_notices = len(BINARY_NOTICE.findall(output_text))
|
| 147 | has_text_structure = hunk_headers > 0 and old_headers > 0 and new_headers > 0
|
| 148 | return {
|
| 149 | "unified_hunk_header_count": hunk_headers,
|
| 150 | "unified_file_header_count": min(old_headers, new_headers),
|
| 151 | "git_diff_header_count": git_headers,
|
| 152 | "binary_notice_count": binary_notices,
|
| 153 | "pipe_compatible_unified": bool(has_text_structure or binary_notices > 0),
|
| 154 | }
|