Viewing files at commit 39bc21f7f67eCommit detailsView latest

Browse files
tests/test_schema_and_oracle.py 193 lines · 9.2 KB · python
1
"""Tests for the agent row schema and the workflow oracles."""
2
3
from __future__ import annotations
4
5
import subprocess
6
import sys
7
import tempfile
8
from pathlib import Path
9
from unittest import mock
10
11
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
12
13
import unittest
14
15
import agent_metrics_schema
16
import agent_workflow
17
import workflow_ab
18
19
20
class SchemaTest(unittest.TestCase):
21
    def test_example_row_validates(self) -> None:
22
        row = agent_metrics_schema.example_row()
23
        self.assertEqual(agent_metrics_schema.validate_row(row), [])
24
25
    def test_missing_group_rejected(self) -> None:
26
        row = agent_metrics_schema.example_row()
27
        del row["turn_metrics"]
28
        errors = agent_metrics_schema.validate_row(row)
29
        self.assertTrue(any("turn_metrics" in error for error in errors))
30
31
    def test_unknown_property_rejected(self) -> None:
32
        row = agent_metrics_schema.example_row()
33
        row["token_metrics"]["surprise"] = 1
34
        errors = agent_metrics_schema.validate_row(row)
35
        self.assertTrue(any("surprise" in error for error in errors))
36
37
    def test_nulls_are_legal_for_unmeasured_signals(self) -> None:
38
        # ADR-0002: unmeasured must be expressible as null, not forced to zero.
39
        row = agent_metrics_schema.example_row()
40
        row["turn_metrics"]["assistant_turns_total"] = None
41
        row["bytes_metrics"]["bytes_hydrated"] = None
42
        row["vcs_metrics"]["dirty_files_at_end"] = None
43
        row["wall_clock_metrics"]["tool_wait_ms"] = None
44
        self.assertEqual(agent_metrics_schema.validate_row(row), [])
45
46
47
class OracleShapeTest(unittest.TestCase):
48
    def workflow(self, name: str) -> workflow_ab.Workflow:
49
        return workflow_ab.WORKFLOWS[name]
50
51
    def test_bugfix_requires_the_one_file(self) -> None:
52
        wf = self.workflow("bugfix_test_loop")
53
        self.assertTrue(agent_workflow.expected_change_ok(wf, ["app/pricing.py"]))
54
        self.assertTrue(
55
            agent_workflow.expected_change_ok(
56
                wf,
57
                [
58
                    "app/pricing.py",
59
                    "app/__pycache__/pricing.cpython-313.pyc",
60
                    "tests/__pycache__/test_pricing.cpython-313.pyc",
61
                ],
62
            )
63
        )
64
        self.assertFalse(agent_workflow.expected_change_ok(wf, []))
65
        self.assertFalse(agent_workflow.expected_change_ok(wf, ["app/pricing.py", "README.md"]))
66
67
    def test_workspace_manifest_ignores_python_bytecode_caches(self) -> None:
68
        with tempfile.TemporaryDirectory() as tmp:
69
            root = Path(tmp)
70
            (root / "app" / "__pycache__").mkdir(parents=True)
71
            (root / "tests" / "__pycache__").mkdir(parents=True)
72
            (root / "app" / "pricing.py").write_text("def f(): pass\n")
73
            (root / "app" / "__pycache__" / "pricing.cpython-313.pyc").write_bytes(b"cache")
74
            (root / "tests" / "__pycache__" / "test_pricing.cpython-313.pyc").write_bytes(b"cache")
75
76
            manifest = agent_workflow.workspace_manifest(root)
77
78
        self.assertEqual(list(manifest), ["app/pricing.py"])
79
80
    def test_history_archaeology_is_read_only(self) -> None:
81
        wf = self.workflow("history_archaeology")
82
        self.assertTrue(agent_workflow.expected_change_ok(wf, []))
83
        self.assertFalse(agent_workflow.expected_change_ok(wf, ["app/pricing.py"]))
84
85
    def test_error_recovery_allows_only_readme(self) -> None:
86
        wf = self.workflow("vcs_error_recovery")
87
        self.assertTrue(agent_workflow.expected_change_ok(wf, ["README.md"]))
88
        self.assertFalse(agent_workflow.expected_change_ok(wf, ["app/pricing.py"]))
89
90
91
class OracleFailClosedTest(unittest.TestCase):
92
    def subject(self, kind):
93
        return workflow_ab.Subject(name=kind, kind=kind, label=kind, bin=Path('/vcs'))
94
95
    def test_failed_status_never_means_clean(self):
96
        for kind in ('git', 'oak'):
97
            for output in ('', 'Nothing to commit, working directory clean'):
98
                with self.subTest(kind=kind, output=output), mock.patch.object(
99
                    agent_workflow, 'run_command',
100
                    return_value=subprocess.CompletedProcess([], 1, output, ''),
101
                ):
102
                    self.assertIsNone(agent_workflow.vcs_status(self.subject(kind), Path('.'))[1])
103
104
    def test_unknown_oak_status_never_means_clean(self):
105
        with mock.patch.object(agent_workflow, 'run_command',
106
                               return_value=subprocess.CompletedProcess([], 0, '', '')):
107
            self.assertIsNone(agent_workflow.vcs_status(self.subject('oak'), Path('.'))[1])
108
109
    def test_oak_structured_status_and_history(self):
110
        for changes, expected in (([], 0), ([{"path": "a", "status": "modified"}], 1)):
111
            with mock.patch.object(agent_workflow, "run_command", return_value=subprocess.CompletedProcess(
112
                [], 0, agent_workflow.json.dumps({"changes": changes}), ""
113
            )) as command:
114
                self.assertEqual(agent_workflow.vcs_status(self.subject("oak"), Path("."))[1], expected)
115
                self.assertEqual(command.call_args[0][0][-2:], ["status", "--json"])
116
        for output, expected in (("[]", 0), ('[{"hash": "' + "a" * 64 + '"}]', 1),
117
                                 ("deadbeef 2026-09-09 main", None), ("{}", None), ('[{}]', None)):
118
            with mock.patch.object(agent_workflow, "command_output", return_value=output) as command:
119
                self.assertEqual(agent_workflow.commit_count(self.subject("oak"), Path(".")), expected)
120
                self.assertEqual(command.call_args[0][0][-2:], ["log", "--json"])
121
122
    def test_manifest_detects_mode_and_symlink_changes(self):
123
        with tempfile.TemporaryDirectory() as tmp:
124
            root = Path(tmp)
125
            a = root / 'a'
126
            b = root / 'b'
127
            a.write_text('same')
128
            b.write_text('same')
129
            link = root / 'link'
130
            link.symlink_to('a')
131
            before = agent_workflow.workspace_manifest(root)
132
            a.chmod(a.stat().st_mode | 0o111)
133
            link.unlink()
134
            link.symlink_to('b')
135
            self.assertEqual(agent_workflow.changed_files(before, agent_workflow.workspace_manifest(root)), ['a', 'link'])
136
            link.unlink()
137
            link.symlink_to('missing')
138
            self.assertIn('link', agent_workflow.workspace_manifest(root))
139
140
    def test_clean_tree_without_new_snapshot_fails_mutating_task(self):
141
        with tempfile.TemporaryDirectory() as tmp:
142
            root = Path(tmp) / 'repo'
143
            root.mkdir()
144
            artifacts = Path(tmp) / 'artifacts'
145
            artifacts.mkdir()
146
            (root / 'README.md').write_text('recovered')
147
            before = agent_workflow.workspace_manifest(root)
148
            (root / 'README.md').write_text('# recovered')
149
            with mock.patch.object(agent_workflow, 'run_command', return_value=subprocess.CompletedProcess([], 0, '', '')), \
150
                 mock.patch.object(agent_workflow, 'vcs_status', return_value=('', 0, 0.0)), \
151
                 mock.patch.object(agent_workflow, 'commit_count', return_value=1):
152
                result = agent_workflow.run_oracle(self.subject('git'), workflow_ab.WORKFLOWS['vcs_error_recovery'], root, before, 1, artifacts)
153
            self.assertFalse(result.checks['snapshot_created'])
154
            self.assertFalse(result.success)
155
156
157
    def test_read_only_task_does_not_require_a_new_snapshot(self):
158
        with tempfile.TemporaryDirectory() as tmp:
159
            root = Path(tmp) / "repo"
160
            root.mkdir()
161
            artifacts = Path(tmp) / "artifacts"
162
            artifacts.mkdir()
163
            before = agent_workflow.workspace_manifest(root)
164
            with mock.patch.object(agent_workflow, "vcs_status", return_value=("", 0, 0.0)), \
165
                 mock.patch.object(agent_workflow, "commit_count", return_value=1):
166
                result = agent_workflow.run_oracle(
167
                    self.subject("git"), workflow_ab.WORKFLOWS["history_archaeology"],
168
                    root, before, 1, artifacts,
169
                )
170
            self.assertTrue(result.checks["snapshot_created"])
171
            self.assertEqual(result.commit_delta, 0)
172
            self.assertFalse(result.success)
173
            self.assertIsNone(result.checks["required_checks_passed"])
174
            self.assertIsNone(result.evidence["workflow_integrity_passed"])
175
            self.assertEqual(result.evidence["workflow_integrity_unmeasured_reason"], "history_answer_oracle_unavailable")
176
177
178
class WorkflowCatalogTest(unittest.TestCase):
179
    def test_snapshot_accounting_in_steps(self) -> None:
180
        # Workflow steps must follow the snapshot convention from the
181
        # command-semantics contract: two git calls, one oak call.
182
        git = workflow_ab.Subject(name="git", kind="git", label="Git", bin=Path("/usr/bin/git"))
183
        oak = workflow_ab.Subject(name="oak", kind="oak", label="Oak", bin=Path("/usr/local/bin/oak"))
184
        git_ops = [s.operation for s in workflow_ab.vcs_workflow_steps(git, "agent-default")]
185
        oak_ops = [s.operation for s in workflow_ab.vcs_workflow_steps(oak, "agent-default")]
186
        self.assertIn("vcs.snapshot.add", git_ops)
187
        self.assertIn("vcs.snapshot.commit", git_ops)
188
        self.assertIn("vcs.snapshot", oak_ops)
189
        self.assertNotIn("vcs.snapshot.add", oak_ops)
190
191
192
if __name__ == "__main__":
193
    unittest.main()