| 1 | """Syscall-boundary crash points (ALICE/CrashMonkey method, process-crash only).
|
| 2 |
|
| 3 | A tiny interposer library counts durability boundaries (fsync, F_FULLFSYNC /
|
| 4 | F_BARRIERFSYNC, rename) in the subject process and SIGKILLs it immediately
|
| 5 | before the N-th one. Enumerating N = 1..count gives a crash state at every
|
| 6 | durability boundary the operation crosses, instead of guessing time offsets.
|
| 7 |
|
| 8 | Scope and honesty:
|
| 9 | - This is process death, not power loss: the page cache survives, so
|
| 10 | un-fsynced writes are NOT dropped. Power-loss states need LazyFS (Linux,
|
| 11 | external tool); rows say ``crash_model: process_kill``.
|
| 12 | - macOS: DYLD_INSERT_LIBRARIES is ignored for hardened-runtime binaries, so
|
| 13 | the harness runs an ad-hoc re-signed copy of the SAME Mach-O (identical code;
|
| 14 | both sha256 recorded). Git needs the copy named ``git`` (builtin dispatch)
|
| 15 | and GIT_EXEC_PATH pointing at the original libexec. Children exec'd from the
|
| 16 | original signed libexec do not load the library (they are not crash targets).
|
| 17 | - Linux: LD_PRELOAD with the same source (symbol wrapping via dlsym) is the
|
| 18 | intended port; unsupported rows are emitted until it is exercised.
|
| 19 | """
|
| 20 |
|
| 21 | from __future__ import annotations
|
| 22 |
|
| 23 | import hashlib
|
| 24 | import os
|
| 25 | import platform
|
| 26 | import shutil
|
| 27 | import subprocess
|
| 28 | from dataclasses import dataclass
|
| 29 | from pathlib import Path
|
| 30 | from typing import Optional
|
| 31 |
|
| 32 | INTERPOSER_SOURCE = r"""
|
| 33 | #include <stdio.h>
|
| 34 | #include <stdlib.h>
|
| 35 | #include <unistd.h>
|
| 36 | #include <signal.h>
|
| 37 | #include <fcntl.h>
|
| 38 | #include <stdarg.h>
|
| 39 | static long counter = 0, target = -1; static int inited = 0; static const char *logp = 0;
|
| 40 | static void init(void) { if (inited) return; inited = 1;
|
| 41 | const char *t = getenv("P4_CRASH_AT"); if (t) target = atol(t); logp = getenv("P4_CRASH_LOG"); }
|
| 42 | static void tick(const char *what) { init(); counter++;
|
| 43 | if (logp) { int fd = open(logp, O_WRONLY | O_APPEND | O_CREAT, 0644);
|
| 44 | if (fd >= 0) { char b[96]; int n = snprintf(b, sizeof b, "%d %ld %s\n", (int)getpid(), counter, what); write(fd, b, n); close(fd); } }
|
| 45 | if (target > 0 && counter == target) kill(getpid(), SIGKILL); }
|
| 46 | static int p4_fsync(int fd) { tick("fsync"); return fsync(fd); }
|
| 47 | static int p4_rename(const char *a, const char *b) { tick("rename"); return rename(a, b); }
|
| 48 | static int p4_fcntl(int fd, int cmd, ...) { va_list ap; va_start(ap, cmd); void *arg = va_arg(ap, void *); va_end(ap);
|
| 49 | if (cmd == F_FULLFSYNC || cmd == F_BARRIERFSYNC) tick("fullfsync"); return fcntl(fd, cmd, arg); }
|
| 50 | #define INTERPOSE(r, o) __attribute__((used)) static struct { const void *replacement; const void *replacee; } \
|
| 51 | _interpose_##o __attribute__((section("__DATA,__interpose"))) = { (const void *)(unsigned long)&r, (const void *)(unsigned long)&o };
|
| 52 | INTERPOSE(p4_fsync, fsync)
|
| 53 | INTERPOSE(p4_rename, rename)
|
| 54 | INTERPOSE(p4_fcntl, fcntl)
|
| 55 | """
|
| 56 |
|
| 57 |
|
| 58 | NO_BOUNDARY_REASON = {
|
| 59 | "oak": "no durability boundary (fsync / F_FULLFSYNC / rename) observed in the oak client process during the "
|
| 60 | "clean instrumented run",
|
| 61 | "git": "no durability boundary observed in the top-level git process (git pull delegates to libexec children "
|
| 62 | "that the interposer cannot load into)",
|
| 63 | }
|
| 64 |
|
| 65 |
|
| 66 | @dataclass
|
| 67 | class CrashInstrument:
|
| 68 | dylib: Path
|
| 69 | oak_bin: Optional[Path]
|
| 70 | git_bin: Optional[Path]
|
| 71 | git_exec_path: Optional[str]
|
| 72 | provenance: dict
|
| 73 | skipped: dict = None # kind -> reason the instrumented copy is unavailable (e.g. codesign failed)
|
| 74 |
|
| 75 | def env(self, target: int, log: Optional[Path] = None, kind: str = "oak") -> dict:
|
| 76 | env = {"DYLD_INSERT_LIBRARIES": str(self.dylib), "P4_CRASH_AT": str(target)}
|
| 77 | if log is not None:
|
| 78 | env["P4_CRASH_LOG"] = str(log)
|
| 79 | if kind == "git" and self.git_exec_path:
|
| 80 | env["GIT_EXEC_PATH"] = self.git_exec_path
|
| 81 | return env
|
| 82 |
|
| 83 |
|
| 84 | def _sha(path: Path) -> str:
|
| 85 | return hashlib.sha256(path.read_bytes()).hexdigest()
|
| 86 |
|
| 87 |
|
| 88 | def build(workdir: Path, oak_bin: Optional[str], git_bin: Optional[str]) -> tuple[Optional[CrashInstrument], str]:
|
| 89 | """Build the interposer + re-signed subject copies. Returns (instrument, reason)."""
|
| 90 | if platform.system() != "Darwin":
|
| 91 | return None, "syscall crash points implemented for macOS DYLD interposition only (Linux LD_PRELOAD port pending)"
|
| 92 | clang = shutil.which("clang")
|
| 93 | codesign = shutil.which("codesign")
|
| 94 | if not clang or not codesign:
|
| 95 | return None, "clang and codesign required to build the crash interposer"
|
| 96 | workdir.mkdir(parents=True, exist_ok=True)
|
| 97 | source = workdir / "p4crash.c"
|
| 98 | source.write_text(INTERPOSER_SOURCE)
|
| 99 | dylib = workdir / "p4crash.dylib"
|
| 100 | proc = subprocess.run([clang, "-dynamiclib", "-O1", "-o", str(dylib), str(source)], capture_output=True, text=True)
|
| 101 | if proc.returncode != 0:
|
| 102 | return None, "interposer build failed: " + proc.stderr[-300:]
|
| 103 | provenance = {"interposer_sha256": _sha(dylib), "crash_model": "process_kill"}
|
| 104 | oak_copy = git_copy = None
|
| 105 | git_exec = None
|
| 106 | skipped: dict = {}
|
| 107 | if oak_bin:
|
| 108 | oak_copy = workdir / "oak-bin" / "oak"
|
| 109 | oak_copy.parent.mkdir(exist_ok=True)
|
| 110 | shutil.copy2(oak_bin, oak_copy)
|
| 111 | why = _resign(codesign, oak_copy)
|
| 112 | if why:
|
| 113 | skipped["oak"], oak_copy = why, None
|
| 114 | else:
|
| 115 | provenance.update(oak_original_sha256=_sha(Path(oak_bin)), oak_adhoc_sha256=_sha(oak_copy))
|
| 116 | if git_bin:
|
| 117 | git_copy = workdir / "git-bin" / "git"
|
| 118 | git_copy.parent.mkdir(exist_ok=True)
|
| 119 | shutil.copy2(git_bin, git_copy)
|
| 120 | why = _resign(codesign, git_copy)
|
| 121 | if why:
|
| 122 | skipped["git"], git_copy = why, None
|
| 123 | else:
|
| 124 | exec_path = subprocess.run([git_bin, "--exec-path"], capture_output=True, text=True).stdout.strip()
|
| 125 | git_exec = exec_path or None
|
| 126 | provenance.update(git_original_sha256=_sha(Path(git_bin)), git_adhoc_sha256=_sha(git_copy),
|
| 127 | git_exec_path=git_exec)
|
| 128 | return CrashInstrument(dylib, oak_copy, git_copy, git_exec, provenance, skipped), ""
|
| 129 |
|
| 130 |
|
| 131 | def _resign(codesign: str, binary: Path) -> str:
|
| 132 | """Ad-hoc re-sign a copy so DYLD_INSERT_LIBRARIES loads; '' on success, else the reason."""
|
| 133 | proc = subprocess.run([codesign, "-f", "-s", "-", str(binary)], capture_output=True, text=True)
|
| 134 | if proc.returncode != 0:
|
| 135 | return f"codesign -f -s - failed (rc={proc.returncode}): {(proc.stderr or proc.stdout).strip()[-200:]}"
|
| 136 | check = subprocess.run([codesign, "--verify", str(binary)], capture_output=True, text=True)
|
| 137 | if check.returncode != 0:
|
| 138 | return f"re-signed copy does not verify: {(check.stderr or check.stdout).strip()[-200:]}"
|
| 139 | return ""
|
| 140 |
|
| 141 |
|
| 142 | def unavailable_reason(instrument: Optional[CrashInstrument], build_reason: str, kind: str) -> str:
|
| 143 | """Why syscall crash points cannot run for this subject ('' when they can)."""
|
| 144 | if instrument is None:
|
| 145 | return build_reason or "crash interposer unavailable"
|
| 146 | if (instrument.skipped or {}).get(kind):
|
| 147 | return instrument.skipped[kind]
|
| 148 | if (instrument.oak_bin if kind == "oak" else instrument.git_bin) is None:
|
| 149 | return f"no instrumented {kind} copy was built"
|
| 150 | return ""
|
| 151 |
|
| 152 |
|
| 153 | def count_boundaries(log: Path, pid: Optional[int] = None) -> int:
|
| 154 | """Durability boundaries crossed by the first (top-level) process in the log."""
|
| 155 | if not log.exists():
|
| 156 | return 0
|
| 157 | counts: dict[str, int] = {}
|
| 158 | first: Optional[str] = None
|
| 159 | for line in log.read_text().splitlines():
|
| 160 | parts = line.split()
|
| 161 | if len(parts) < 3:
|
| 162 | continue
|
| 163 | first = first or parts[0]
|
| 164 | counts[parts[0]] = max(counts.get(parts[0], 0), int(parts[1]))
|
| 165 | if pid is not None:
|
| 166 | return counts.get(str(pid), 0)
|
| 167 | return counts.get(first, 0) if first else 0
|
| 168 |
|
| 169 |
|
| 170 | def boundary_kinds(log: Path) -> list[str]:
|
| 171 | if not log.exists():
|
| 172 | return []
|
| 173 | lines = [line.split() for line in log.read_text().splitlines() if line.strip()]
|
| 174 | if not lines:
|
| 175 | return []
|
| 176 | first = lines[0][0]
|
| 177 | return [parts[2] for parts in lines if parts[0] == first]
|