| 1 | """One fast end-to-end check: the workflow lane runs a real recipe against git.
|
| 2 |
|
| 3 | This is the smoke wire from instruments to rows: fixture, command semantics,
|
| 4 | timed execution, row contract, results store. Skipped when git is unavailable.
|
| 5 | """
|
| 6 |
|
| 7 | from __future__ import annotations
|
| 8 |
|
| 9 | import sys
|
| 10 | from pathlib import Path
|
| 11 |
|
| 12 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 13 |
|
| 14 | import json
|
| 15 | import shutil
|
| 16 | import tempfile
|
| 17 | import unittest
|
| 18 | from pathlib import Path
|
| 19 |
|
| 20 | import workflow_ab
|
| 21 | from oakbench.rows import validate_rows
|
| 22 | from oakbench.subjects import Subject
|
| 23 |
|
| 24 |
|
| 25 | @unittest.skipIf(shutil.which("git") is None, "git not installed")
|
| 26 | class WorkflowLaneIntegrationTest(unittest.TestCase):
|
| 27 | def setUp(self) -> None:
|
| 28 | self.tmp = Path(tempfile.mkdtemp(prefix="oakbench-test-"))
|
| 29 |
|
| 30 | def tearDown(self) -> None:
|
| 31 | shutil.rmtree(self.tmp, ignore_errors=True)
|
| 32 |
|
| 33 | def test_every_workflow_fixture_prepares(self) -> None:
|
| 34 | """Every defined workflow must at least build its fixture.
|
| 35 |
|
| 36 | Catches import/codepath breaks in workflows the end-to-end test does
|
| 37 | not run (a missing json import once crashed large_asset_manifest only).
|
| 38 | """
|
| 39 | for name, workflow in workflow_ab.WORKFLOWS.items():
|
| 40 | with self.subTest(workflow=name):
|
| 41 | fixture = workflow_ab.prepare_fixture(self.tmp / name, workflow)
|
| 42 | self.assertTrue(fixture.exists())
|
| 43 |
|
| 44 | def test_error_recovery_workflow_end_to_end(self) -> None:
|
| 45 | subject = Subject(name="git", kind="git", label="Git", bin=Path(shutil.which("git")))
|
| 46 | workflow = workflow_ab.WORKFLOWS["vcs_error_recovery"]
|
| 47 | fixture = workflow_ab.prepare_fixture(self.tmp, workflow)
|
| 48 | metadata = {
|
| 49 | "bench_id": "test",
|
| 50 | "profile": "workflow-ab",
|
| 51 | "benchmark_track": "agent-default",
|
| 52 | "command_semantics_version": workflow_ab.COMMAND_SEMANTICS_VERSION,
|
| 53 | "timestamp_utc": "test",
|
| 54 | }
|
| 55 | rows = workflow_ab.run_subject_workflow(
|
| 56 | subject,
|
| 57 | workflow,
|
| 58 | fixture,
|
| 59 | run_index=0,
|
| 60 | run_root=self.tmp / "runs",
|
| 61 | metadata=metadata,
|
| 62 | admitted_output_chars=20_000,
|
| 63 | track="agent-default",
|
| 64 | )
|
| 65 |
|
| 66 | self.assertEqual(validate_rows(rows, "workflow"), [])
|
| 67 |
|
| 68 | totals = [row for row in rows if row["operation"] == "workflow.total"]
|
| 69 | self.assertEqual(len(totals), 1)
|
| 70 | self.assertEqual(totals[0]["returncode"], 0)
|
| 71 |
|
| 72 | # The induced failure must be recorded, not skipped: the empty commit
|
| 73 | # fails at the process level while staying a tolerated step.
|
| 74 | commit_nothing = next(row for row in rows if row["operation"] == "error.commit_nothing")
|
| 75 | self.assertEqual(commit_nothing["returncode"], 0)
|
| 76 | self.assertNotEqual(commit_nothing["process_returncode"], 0)
|
| 77 |
|
| 78 | # Snapshot accounting: git pays two calls.
|
| 79 | operations = [row["operation"] for row in rows]
|
| 80 | self.assertIn("vcs.snapshot.add", operations)
|
| 81 | self.assertIn("vcs.snapshot.commit", operations)
|
| 82 |
|
| 83 | # Rows are JSON-serializable as-is (the durable format).
|
| 84 | for row in rows:
|
| 85 | json.dumps(row)
|
| 86 |
|
| 87 |
|
| 88 | class WorkflowFailurePolicyTest(unittest.TestCase):
|
| 89 | def test_lane_failure_predicate_ignores_skips_but_flags_failures(self) -> None:
|
| 90 | self.assertFalse(workflow_ab.has_lane_failures([{"returncode": workflow_ab.SKIP_RETURNCODE, "skipped": True}]))
|
| 91 | self.assertTrue(workflow_ab.has_lane_failures([{"returncode": 1}]))
|
| 92 |
|
| 93 |
|
| 94 | if __name__ == "__main__":
|
| 95 | unittest.main()
|