Viewing files at commit 54b9920be8dfCommit detailsView latest

Browse files
tests/test_integration_workflow.py 95 lines · 3.5 KB · python
1
"""One fast end-to-end check: the workflow lane runs a real recipe against git.
2
3
This is the smoke wire from instruments to rows: fixture, command semantics,
4
timed execution, row contract, results store. Skipped when git is unavailable.
5
"""
6
7
from __future__ import annotations
8
9
import sys
10
from pathlib import Path
11
12
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
13
14
import json
15
import shutil
16
import tempfile
17
import unittest
18
from pathlib import Path
19
20
import workflow_ab
21
from oakbench.rows import validate_rows
22
from oakbench.subjects import Subject
23
24
25
@unittest.skipIf(shutil.which("git") is None, "git not installed")
26
class WorkflowLaneIntegrationTest(unittest.TestCase):
27
    def setUp(self) -> None:
28
        self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-test-"))
29
30
    def tearDown(self) -> None:
31
        shutil.rmtree(self.tmp, ignore_errors=True)
32
33
    def test_every_workflow_fixture_prepares(self) -> None:
34
        """Every defined workflow must at least build its fixture.
35
36
        Catches import/codepath breaks in workflows the end-to-end test does
37
        not run (a missing json import once crashed large_asset_manifest only).
38
        """
39
        for name, workflow in workflow_ab.WORKFLOWS.items():
40
            with self.subTest(workflow=name):
41
                fixture = workflow_ab.prepare_fixture(self.tmp / name, workflow)
42
                self.assertTrue(fixture.exists())
43
44
    def test_error_recovery_workflow_end_to_end(self) -> None:
45
        subject = Subject(name="git", kind="git", label="Git", bin=Path(shutil.which("git")))
46
        workflow = workflow_ab.WORKFLOWS["vcs_error_recovery"]
47
        fixture = workflow_ab.prepare_fixture(self.tmp, workflow)
48
        metadata = {
49
            "bench_id": "test",
50
            "profile": "workflow-ab",
51
            "benchmark_track": "agent-default",
52
            "command_semantics_version": workflow_ab.COMMAND_SEMANTICS_VERSION,
53
            "timestamp_utc": "test",
54
        }
55
        rows = workflow_ab.run_subject_workflow(
56
            subject,
57
            workflow,
58
            fixture,
59
            run_index=0,
60
            run_root=self.tmp / "runs",
61
            metadata=metadata,
62
            admitted_output_chars=20_000,
63
            track="agent-default",
64
        )
65
66
        self.assertEqual(validate_rows(rows, "workflow"), [])
67
68
        totals = [row for row in rows if row["operation"] == "workflow.total"]
69
        self.assertEqual(len(totals), 1)
70
        self.assertEqual(totals[0]["returncode"], 0)
71
72
        # The induced failure must be recorded, not skipped: the empty commit
73
        # fails at the process level while staying a tolerated step.
74
        commit_nothing = next(row for row in rows if row["operation"] == "error.commit_nothing")
75
        self.assertEqual(commit_nothing["returncode"], 0)
76
        self.assertNotEqual(commit_nothing["process_returncode"], 0)
77
78
        # Snapshot accounting: git pays two calls.
79
        operations = [row["operation"] for row in rows]
80
        self.assertIn("vcs.snapshot.add", operations)
81
        self.assertIn("vcs.snapshot.commit", operations)
82
83
        # Rows are JSON-serializable as-is (the durable format).
84
        for row in rows:
85
            json.dumps(row)
86
87
88
class WorkflowFailurePolicyTest(unittest.TestCase):
89
    def test_lane_failure_predicate_ignores_skips_but_flags_failures(self) -> None:
90
        self.assertFalse(workflow_ab.has_lane_failures([{"returncode": workflow_ab.SKIP_RETURNCODE, "skipped": True}]))
91
        self.assertTrue(workflow_ab.has_lane_failures([{"returncode": 1}]))
92
93
94
if __name__ == "__main__":
95
    unittest.main()