| 1 | """Mount probe correctness checks that do not require a real Oak mount."""
|
| 2 |
|
| 3 | from __future__ import annotations
|
| 4 |
|
| 5 | import shutil
|
| 6 | import sys
|
| 7 | import tempfile
|
| 8 | import unittest
|
| 9 | from pathlib import Path
|
| 10 | from unittest import mock
|
| 11 |
|
| 12 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
| 13 |
|
| 14 | import mount_probe # noqa: E402
|
| 15 |
|
| 16 |
|
| 17 | META = {"bench_id": "test", "profile": "mount", "subject": "oak_mount"}
|
| 18 |
|
| 19 |
|
| 20 | class MountProbeContentCheckTest(unittest.TestCase):
|
| 21 | def test_contains_check_records_pass(self) -> None:
|
| 22 | with tempfile.TemporaryDirectory() as tmp:
|
| 23 | root = Path(tmp)
|
| 24 | (root / "README.md").write_text("# oak-benchmarks-tmp\n")
|
| 25 | fields = mount_probe.content_check_fields(
|
| 26 | root,
|
| 27 | "README.md",
|
| 28 | (root / "README.md").read_bytes(),
|
| 29 | {"target_read_contains": "# oak-benchmarks-tmp"},
|
| 30 | )
|
| 31 | self.assertTrue(fields["content_verified"])
|
| 32 | self.assertEqual(fields["content_check_method"], "contains")
|
| 33 | self.assertEqual(fields["content_check_status"], "passed")
|
| 34 |
|
| 35 | def test_required_manifest_missing_records_failure(self) -> None:
|
| 36 | with tempfile.TemporaryDirectory() as tmp:
|
| 37 | root = Path(tmp)
|
| 38 | (root / "README.md").write_text("content\n")
|
| 39 | fields = mount_probe.content_check_fields(
|
| 40 | root,
|
| 41 | "README.md",
|
| 42 | (root / "README.md").read_bytes(),
|
| 43 | {"target_read_contains": None, "require_content_manifest": True},
|
| 44 | )
|
| 45 | self.assertFalse(fields["content_verified"])
|
| 46 | self.assertEqual(fields["content_check_method"], "manifest_sha256")
|
| 47 | self.assertEqual(fields["content_check_status"], "failed")
|
| 48 |
|
| 49 |
|
| 50 | class MountProbeStartupReadinessTest(unittest.TestCase):
|
| 51 | def test_readiness_probe_accepts_empty_target_file(self) -> None:
|
| 52 | with tempfile.TemporaryDirectory() as tmp:
|
| 53 | root = Path(tmp)
|
| 54 | (root / "README.md").write_bytes(b"")
|
| 55 | fields = mount_probe.mount_readiness_probe(
|
| 56 | root,
|
| 57 | {"target_read": "README.md", "target_read_bytes": 4096},
|
| 58 | )
|
| 59 |
|
| 60 | self.assertTrue(fields["mount_ready_list_ok"])
|
| 61 | self.assertTrue(fields["mount_ready_read_ok"])
|
| 62 | self.assertEqual(fields["mount_ready_read_bytes"], 0)
|
| 63 | self.assertTrue(fields["checks"]["ready_to_read"])
|
| 64 |
|
| 65 | def test_mount_start_records_command_and_ready_to_read_timings(self) -> None:
|
| 66 | with tempfile.TemporaryDirectory() as tmp:
|
| 67 | run_root = Path(tmp)
|
| 68 |
|
| 69 | def fake_run_process(command, cwd, timeout, retries, capture_stdout=True): # noqa: ANN001
|
| 70 | if command[:3] == ["oak", "mount", "end"]:
|
| 71 | shutil.rmtree(command[3], ignore_errors=True)
|
| 72 | return {"elapsed_ms": 3.0, "returncode": 0, "stdout": "", "stderr": "", "attempts": 1, "retry_count": 0}
|
| 73 | if command[:2] == ["oak", "mount"] and len(command) == 4:
|
| 74 | path = Path(command[3])
|
| 75 | path.mkdir(parents=True)
|
| 76 | (path / "README.md").write_text("# oak-benchmarks-tmp\n")
|
| 77 | return {"elapsed_ms": 12.0, "returncode": 0, "stdout": "", "stderr": "", "attempts": 1, "retry_count": 0}
|
| 78 | raise AssertionError(command)
|
| 79 |
|
| 80 | with (
|
| 81 | mock.patch.object(mount_probe, "run_process", side_effect=fake_run_process),
|
| 82 | mock.patch.object(
|
| 83 | mount_probe.oakbench_cachectl,
|
| 84 | "purge_fs_caches",
|
| 85 | return_value={"attempted": True, "ok": True, "method": "darwin_purge", "error": None},
|
| 86 | ),
|
| 87 | ):
|
| 88 | rows, cleanup_safe = mount_probe.run_operation_sequence(
|
| 89 | META,
|
| 90 | "oak",
|
| 91 | "oak/repo",
|
| 92 | False,
|
| 93 | {"name": "cold_mount_startup", "cache_state": "cold", "operations": ["mount.start"]},
|
| 94 | {"target_read": "README.md", "target_read_bytes": 4096},
|
| 95 | run_root,
|
| 96 | 0,
|
| 97 | )
|
| 98 |
|
| 99 | mount_row = next(row for row in rows if row["operation"] == "mount.start")
|
| 100 | self.assertTrue(cleanup_safe)
|
| 101 | self.assertEqual(mount_row["returncode"], 0)
|
| 102 | self.assertEqual(mount_row["mount_command_returncode"], 0)
|
| 103 | self.assertEqual(mount_row["mount_command_elapsed_ms"], 12.0)
|
| 104 | self.assertTrue(mount_row["mount_ready_read_ok"])
|
| 105 | self.assertIn("time_to_mount_command_return_ms", mount_row)
|
| 106 | self.assertIn("time_to_mount_ready_to_read_ms", mount_row)
|
| 107 | self.assertEqual(mount_row["requested_cache_state"], "cold")
|
| 108 | self.assertEqual(mount_row["cache_state"], "cold")
|
| 109 | self.assertEqual(mount_row["cache_state_reason"], "fs_caches_purged: darwin_purge")
|
| 110 |
|
| 111 | def test_readiness_probe_failure_fails_mount_start_row(self) -> None:
|
| 112 | with tempfile.TemporaryDirectory() as tmp:
|
| 113 | run_root = Path(tmp)
|
| 114 |
|
| 115 | def fake_run_process(command, cwd, timeout, retries, capture_stdout=True): # noqa: ANN001
|
| 116 | if command[:2] == ["oak", "mount"] and len(command) == 4:
|
| 117 | Path(command[3]).mkdir(parents=True)
|
| 118 | return {"elapsed_ms": 12.0, "returncode": 0, "stdout": "", "stderr": "", "attempts": 1, "retry_count": 0}
|
| 119 | raise AssertionError(command)
|
| 120 |
|
| 121 | with (
|
| 122 | mock.patch.object(mount_probe, "run_process", side_effect=fake_run_process),
|
| 123 | mock.patch.object(
|
| 124 | mount_probe.oakbench_cachectl,
|
| 125 | "purge_fs_caches",
|
| 126 | return_value={"attempted": True, "ok": False, "method": "darwin_purge", "error": "exit 1"},
|
| 127 | ),
|
| 128 | ):
|
| 129 | rows, cleanup_safe = mount_probe.run_operation_sequence(
|
| 130 | META,
|
| 131 | "oak",
|
| 132 | "oak/repo",
|
| 133 | False,
|
| 134 | {"name": "cold_mount_startup", "cache_state": "cold", "operations": ["mount.start"]},
|
| 135 | {"target_read": "README.md", "target_read_bytes": 4096},
|
| 136 | run_root,
|
| 137 | 0,
|
| 138 | )
|
| 139 |
|
| 140 | mount_row = next(row for row in rows if row["operation"] == "mount.start")
|
| 141 | self.assertFalse(cleanup_safe)
|
| 142 | self.assertEqual(mount_row["returncode"], 1)
|
| 143 | self.assertFalse(mount_row["checks"]["ready_to_read"])
|
| 144 | self.assertEqual(mount_row["cache_state"], "unknown")
|
| 145 | self.assertEqual(mount_row["cache_state_reason"], "purge_failed: exit 1")
|
| 146 |
|
| 147 | def test_interrupted_recovery_non_empty_destination_is_explicit_skip(self) -> None:
|
| 148 | with tempfile.TemporaryDirectory() as tmp:
|
| 149 | run_root = Path(tmp)
|
| 150 |
|
| 151 | def fake_run_process(command, cwd, timeout, retries, capture_stdout=True): # noqa: ANN001
|
| 152 | self.assertEqual(command[:2], ["oak", "mount"])
|
| 153 | return {
|
| 154 | "elapsed_ms": 5.0,
|
| 155 | "returncode": 6,
|
| 156 | "stdout": "",
|
| 157 | "stderr": "error: mount destination is not empty",
|
| 158 | "attempts": 1,
|
| 159 | "retry_count": 0,
|
| 160 | }
|
| 161 |
|
| 162 | with mock.patch.object(mount_probe, "run_process", side_effect=fake_run_process):
|
| 163 | rows, cleanup_safe = mount_probe.run_recovery_probe(
|
| 164 | META,
|
| 165 | "oak",
|
| 166 | "oak/repo",
|
| 167 | {"name": "interrupted_recovery", "cache_state": "unknown"},
|
| 168 | {},
|
| 169 | run_root,
|
| 170 | 0,
|
| 171 | )
|
| 172 |
|
| 173 | row = rows[0]
|
| 174 | self.assertTrue(cleanup_safe)
|
| 175 | self.assertEqual(row["returncode"], mount_probe.SKIP_RETURNCODE)
|
| 176 | self.assertTrue(row["skipped"])
|
| 177 | self.assertEqual(row["skip_reason"], "mount_partial_destination_recovery_not_supported")
|
| 178 | self.assertEqual(row["process_returncode"], 6)
|
| 179 | self.assertFalse(row["recovery_supported"])
|
| 180 |
|
| 181 | def test_oak_finish_uses_desc_file(self) -> None:
|
| 182 | with tempfile.TemporaryDirectory() as tmp:
|
| 183 | run_root = Path(tmp)
|
| 184 | seen_commands: list[list[str]] = []
|
| 185 |
|
| 186 | def fake_run_process(command, cwd, timeout, retries, capture_stdout=True): # noqa: ANN001
|
| 187 | seen_commands.append(command)
|
| 188 | if command[:2] == ["oak", "mount"] and len(command) == 4:
|
| 189 | path = Path(command[3])
|
| 190 | path.mkdir(parents=True)
|
| 191 | (path / "README.md").write_text("# oak-benchmarks-tmp\n")
|
| 192 | return {"elapsed_ms": 12.0, "returncode": 0, "stdout": "", "stderr": "", "attempts": 1, "retry_count": 0}
|
| 193 | if command[:2] == ["oak", "finish"]:
|
| 194 | self.assertIn("--desc-file", command)
|
| 195 | desc_file = Path(command[command.index("--desc-file") + 1])
|
| 196 | self.assertTrue(desc_file.is_absolute())
|
| 197 | self.assertEqual(desc_file.read_text(), "finish message\n")
|
| 198 | shutil.rmtree(cwd, ignore_errors=True)
|
| 199 | return {"elapsed_ms": 7.0, "returncode": 0, "stdout": "", "stderr": "", "attempts": 1, "retry_count": 0}
|
| 200 | raise AssertionError(command)
|
| 201 |
|
| 202 | with (
|
| 203 | mock.patch.object(mount_probe, "run_process", side_effect=fake_run_process),
|
| 204 | mock.patch.object(mount_probe, "oak_supports", return_value=True),
|
| 205 | ):
|
| 206 | rows, cleanup_safe = mount_probe.run_operation_sequence(
|
| 207 | META,
|
| 208 | "oak",
|
| 209 | "oak/repo",
|
| 210 | False,
|
| 211 | {"name": "task_lifecycle_loop", "operations": ["mount.start", "oak.finish"]},
|
| 212 | {
|
| 213 | "target_read": "README.md",
|
| 214 | "target_read_bytes": 4096,
|
| 215 | "desc_message": "finish message",
|
| 216 | },
|
| 217 | run_root,
|
| 218 | 0,
|
| 219 | )
|
| 220 |
|
| 221 | finish_row = next(row for row in rows if row["operation"] == "oak.finish")
|
| 222 | self.assertTrue(cleanup_safe)
|
| 223 | self.assertEqual(finish_row["returncode"], 0)
|
| 224 | self.assertTrue(finish_row["checks"]["path_released"])
|
| 225 | self.assertEqual(
|
| 226 | seen_commands[-1],
|
| 227 | ["oak", "finish", "--desc-file", finish_row["desc_file"]],
|
| 228 | )
|
| 229 |
|
| 230 | def test_cold_scenario_cache_fields_record_purge_result(self) -> None:
|
| 231 | fields = mount_probe.scenario_cache_fields(
|
| 232 | {"name": "cold_mount_startup", "cache_state": "cold"},
|
| 233 | {"attempted": True, "ok": True, "method": "darwin_purge", "error": None},
|
| 234 | )
|
| 235 |
|
| 236 | self.assertEqual(fields["requested_cache_state"], "cold")
|
| 237 | self.assertEqual(fields["cache_state"], "cold")
|
| 238 | self.assertEqual(fields["cache_state_reason"], "fs_caches_purged: darwin_purge")
|
| 239 |
|
| 240 |
|
| 241 | class MountProbeCacheStateTest(unittest.TestCase):
|
| 242 | def test_missing_cache_state_gets_concrete_unknown_unset_fields(self) -> None:
|
| 243 | self.assertEqual(
|
| 244 | mount_probe.scenario_cache_fields({"name": "default_mount", "operations": ["mount.start"]}),
|
| 245 | {
|
| 246 | "requested_cache_state": mount_probe.UNSET_CACHE_STATE,
|
| 247 | "cache_state": mount_probe.UNKNOWN_CACHE_STATE,
|
| 248 | "cache_state_reason": mount_probe.CACHE_STATE_UNSET,
|
| 249 | },
|
| 250 | )
|
| 251 |
|
| 252 | def test_missing_cache_state_is_applied_to_rows_without_none_values(self) -> None:
|
| 253 | rows = [{"operation": "mount.start"}]
|
| 254 | mount_probe.apply_scenario_cache_state(rows, {"name": "default_mount"})
|
| 255 |
|
| 256 | self.assertEqual(rows[0]["requested_cache_state"], mount_probe.UNSET_CACHE_STATE)
|
| 257 | self.assertEqual(rows[0]["cache_state"], mount_probe.UNKNOWN_CACHE_STATE)
|
| 258 | self.assertEqual(rows[0]["cache_state_reason"], mount_probe.CACHE_STATE_UNSET)
|
| 259 | self.assertNotIn(None, rows[0].values())
|
| 260 |
|
| 261 | def test_missing_cache_state_preserves_explicit_reason(self) -> None:
|
| 262 | fields = mount_probe.scenario_cache_fields(
|
| 263 | {"name": "default_mount", "cache_state_reason": "configured_by_fixture"},
|
| 264 | )
|
| 265 |
|
| 266 | self.assertEqual(fields["requested_cache_state"], mount_probe.UNSET_CACHE_STATE)
|
| 267 | self.assertEqual(fields["cache_state"], mount_probe.UNKNOWN_CACHE_STATE)
|
| 268 | self.assertEqual(fields["cache_state_reason"], "configured_by_fixture")
|
| 269 |
|
| 270 | def test_explicit_cache_state_uses_same_row_keys(self) -> None:
|
| 271 | fields = mount_probe.scenario_cache_fields({"name": "warm_mount", "cache_state": "warm"})
|
| 272 |
|
| 273 | self.assertEqual(fields["requested_cache_state"], "warm")
|
| 274 | self.assertEqual(fields["cache_state"], "warm")
|
| 275 | self.assertEqual(fields["cache_state_reason"], mount_probe.EXPLICIT_CACHE_STATE)
|
| 276 |
|
| 277 |
|
| 278 | if __name__ == "__main__":
|
| 279 | unittest.main()
|