Viewing files at commit f86d1109f5dfCommit detailsView latest

Browse files
scripts/oakbench/output_semantics.py 154 lines · 6.5 KB · python
1
"""Information content of VCS output: bytes saved vs information lost.
2
3
Compact output is only a win if the agent can still act on it. These oracles
4
score a status/diff output against the fixture's ground-truth changed file
5
list: information_recall is the fraction of actually-changed paths an agent
6
(or a script) could recover from the bytes it was shown. A subject that emits
7
half the bytes but names half the files has not saved anything — it has moved
8
the cost to a follow-up command.
9
10
The second oracle is pipe compatibility: scripts and agents pipe diff output
11
into grep/patch/apply tooling that expects unified-diff structure. A diff
12
output without recognizable file headers and hunk markers silently breaks that
13
ecosystem; the probe records the structure so a format change is a visible
14
metric regression, not a downstream surprise.
15
"""
16
17
from __future__ import annotations
18
19
import re
20
from typing import Any, Iterable
21
22
STATUS_LINE = re.compile(r"^(?:(?!  )[ MADRCU?!]{2}|[MADRCU?!])\s+(.+)$")
23
UNIFIED_HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@", re.MULTILINE)
24
UNIFIED_OLD_FILE = re.compile(r"^--- ", re.MULTILINE)
25
UNIFIED_NEW_FILE = re.compile(r"^\+\+\+ ", re.MULTILINE)
26
GIT_DIFF_HEADER = re.compile(r"^diff --git ", re.MULTILINE)
27
BINARY_NOTICE = re.compile(r"^Binary files? .* differ", re.MULTILINE | re.IGNORECASE)
28
29
30
def recovered_paths(output_text: str, changed_paths: Iterable[str]) -> list[str]:
31
    """Ground-truth paths an agent could recover from this output.
32
33
    A path counts as recovered when its repo-relative form appears in the
34
    output (including git's a/ b/ prefixes). Substring matching is
35
    deliberately generous and subject-neutral: it asks "is the information
36
    present at all?", not "is it in a specific format".
37
    """
38
    recovered: list[str] = []
39
    for path in changed_paths:
40
        if path in output_text:
41
            recovered.append(path)
42
    return recovered
43
44
45
def _normalize_reported_path(raw: str) -> str | None:
46
    path = raw.strip().strip("'\"`:,;()[]{}")
47
    if not path or path == "/dev/null" or path.startswith(("http://", "https://")):
48
        return None
49
    if path.startswith(("a/", "b/")):
50
        path = path[2:]
51
    name = path.rsplit("/", 1)[-1]
52
    root_file_like = "." in name or name in {"Dockerfile", "Makefile", "README", "LICENSE"}
53
    return path if "/" in path or root_file_like else None
54
55
56
def _line_reported_paths(line: str) -> list[str]:
57
    stripped = line.strip()
58
    candidates: list[str] = []
59
    if stripped.startswith("diff --git "):
60
        candidates.extend(stripped.split()[2:4])
61
    elif stripped.startswith(("--- ", "+++ ")):
62
        candidates.append(stripped.split(maxsplit=1)[1].split("\t", 1)[0])
63
    elif stripped.lower().startswith("binary files "):
64
        parts = stripped.split()
65
        if len(parts) >= 5:
66
            candidates.extend([parts[2], parts[4]])
67
    elif " | " in stripped:
68
        candidates.append(stripped.split(" | ", 1)[0])
69
    else:
70
        status = STATUS_LINE.match(line)
71
        if status:
72
            payload = status.group(1)
73
            candidates.extend(part.strip() for part in payload.split(" -> "))
74
    normalized = [_normalize_reported_path(candidate) for candidate in candidates]
75
    return [path for path in normalized if path is not None]
76
77
78
def reported_paths(output_text: str) -> list[str]:
79
    """Repo-like paths reported by VCS metadata lines.
80
81
    Diff bodies can contain dotted identifiers and version strings that look
82
    path-ish. Precision measures named changed files, so it only scans status
83
    rows, diff headers, file headers, binary notices, and diffstat rows.
84
    """
85
    paths: set[str] = set()
86
    in_unified_hunk = False
87
    for line in output_text.splitlines():
88
        stripped = line.strip()
89
        if stripped.startswith("diff --git "):
90
            in_unified_hunk = False
91
        elif stripped.startswith("@@ "):
92
            in_unified_hunk = True
93
            continue
94
        elif in_unified_hunk:
95
            continue
96
        paths.update(_line_reported_paths(line))
97
    return sorted(paths)
98
99
100
def information_recall_fields(
101
    output_text: str,
102
    changed_paths: list[str],
103
    output_bytes: int,
104
    output_truncated: bool,
105
) -> dict[str, Any]:
106
    """Recall and byte-efficiency fields for one status/diff output.
107
108
    information_recall is null (unmeasured) when there is no ground truth to
109
    score against; bytes_per_changed_file_named is null when nothing was
110
    recovered — never zero (ADR-0002).
111
    """
112
    expected_paths = set(changed_paths)
113
    recovered = recovered_paths(output_text, changed_paths)
114
    reported = reported_paths(output_text)
115
    false_positives = [path for path in reported if path not in expected_paths]
116
    true_reported = [path for path in reported if path in expected_paths]
117
    expected = len(changed_paths)
118
    recall = round(len(recovered) / expected, 4) if expected else None
119
    precision = round(len(true_reported) / len(reported), 4) if expected and reported else None
120
    bytes_per_named = round(output_bytes / len(recovered), 1) if recovered else None
121
    return {
122
        "changed_files_expected": expected,
123
        "changed_files_recovered": len(recovered),
124
        "changed_files_reported": len(reported),
125
        "changed_files_false_positive": len(false_positives),
126
        "information_recall": recall,
127
        "information_precision": precision,
128
        "bytes_per_changed_file_named": bytes_per_named,
129
        "probe_output_truncated": output_truncated,
130
        "information_recall_source": "ground_truth_path_substring_match",
131
        "information_precision_source": "path_token_precision_against_changed_paths",
132
    }
133
134
135
def unified_diff_compat_fields(output_text: str) -> dict[str, Any]:
136
    """Structure fields for piped-diff compatibility.
137
138
    pipe_compatible_unified means the output carries both unified file headers
139
    and hunk markers (or is an explicit binary notice), i.e. standard tooling
140
    (patch, git apply, diffstat, grep '^+') can consume it.
141
    """
142
    hunk_headers = len(UNIFIED_HUNK_HEADER.findall(output_text))
143
    old_headers = len(UNIFIED_OLD_FILE.findall(output_text))
144
    new_headers = len(UNIFIED_NEW_FILE.findall(output_text))
145
    git_headers = len(GIT_DIFF_HEADER.findall(output_text))
146
    binary_notices = len(BINARY_NOTICE.findall(output_text))
147
    has_text_structure = hunk_headers > 0 and old_headers > 0 and new_headers > 0
148
    return {
149
        "unified_hunk_header_count": hunk_headers,
150
        "unified_file_header_count": min(old_headers, new_headers),
151
        "git_diff_header_count": git_headers,
152
        "binary_notice_count": binary_notices,
153
        "pipe_compatible_unified": bool(has_text_structure or binary_notices > 0),
154
    }