Viewing files at commit 1a441edafc71Commit detailsView latest

suites/robustness/robustness/crashpoints.py 177 lines · 7.8 KB · python
1
"""Syscall-boundary crash points (ALICE/CrashMonkey method, process-crash only).
2
3
A tiny interposer library counts durability boundaries (fsync, F_FULLFSYNC /
4
F_BARRIERFSYNC, rename) in the subject process and SIGKILLs it immediately
5
before the N-th one. Enumerating N = 1..count gives a crash state at every
6
durability boundary the operation crosses, instead of guessing time offsets.
7
8
Scope and honesty:
9
- This is process death, not power loss: the page cache survives, so
10
  un-fsynced writes are NOT dropped. Power-loss states need LazyFS (Linux,
11
  external tool); rows say ``crash_model: process_kill``.
12
- macOS: DYLD_INSERT_LIBRARIES is ignored for hardened-runtime binaries, so
13
  the harness runs an ad-hoc re-signed copy of the SAME Mach-O (identical code;
14
  both sha256 recorded). Git needs the copy named ``git`` (builtin dispatch)
15
  and GIT_EXEC_PATH pointing at the original libexec. Children exec'd from the
16
  original signed libexec do not load the library (they are not crash targets).
17
- Linux: LD_PRELOAD with the same source (symbol wrapping via dlsym) is the
18
  intended port; unsupported rows are emitted until it is exercised.
19
"""
20
21
from __future__ import annotations
22
23
import hashlib
24
import os
25
import platform
26
import shutil
27
import subprocess
28
from dataclasses import dataclass
29
from pathlib import Path
30
from typing import Optional
31
32
INTERPOSER_SOURCE = r"""
33
#include <stdio.h>
34
#include <stdlib.h>
35
#include <unistd.h>
36
#include <signal.h>
37
#include <fcntl.h>
38
#include <stdarg.h>
39
static long counter = 0, target = -1; static int inited = 0; static const char *logp = 0;
40
static void init(void) { if (inited) return; inited = 1;
41
  const char *t = getenv("P4_CRASH_AT"); if (t) target = atol(t); logp = getenv("P4_CRASH_LOG"); }
42
static void tick(const char *what) { init(); counter++;
43
  if (logp) { int fd = open(logp, O_WRONLY | O_APPEND | O_CREAT, 0644);
44
    if (fd >= 0) { char b[96]; int n = snprintf(b, sizeof b, "%d %ld %s\n", (int)getpid(), counter, what); write(fd, b, n); close(fd); } }
45
  if (target > 0 && counter == target) kill(getpid(), SIGKILL); }
46
static int p4_fsync(int fd) { tick("fsync"); return fsync(fd); }
47
static int p4_rename(const char *a, const char *b) { tick("rename"); return rename(a, b); }
48
static int p4_fcntl(int fd, int cmd, ...) { va_list ap; va_start(ap, cmd); void *arg = va_arg(ap, void *); va_end(ap);
49
  if (cmd == F_FULLFSYNC || cmd == F_BARRIERFSYNC) tick("fullfsync"); return fcntl(fd, cmd, arg); }
50
#define INTERPOSE(r, o) __attribute__((used)) static struct { const void *replacement; const void *replacee; } \
51
  _interpose_##o __attribute__((section("__DATA,__interpose"))) = { (const void *)(unsigned long)&r, (const void *)(unsigned long)&o };
52
INTERPOSE(p4_fsync, fsync)
53
INTERPOSE(p4_rename, rename)
54
INTERPOSE(p4_fcntl, fcntl)
55
"""
56
57
58
NO_BOUNDARY_REASON = {
59
    "oak": "no durability boundary (fsync / F_FULLFSYNC / rename) observed in the oak client process during the "
60
           "clean instrumented run",
61
    "git": "no durability boundary observed in the top-level git process (git pull delegates to libexec children "
62
           "that the interposer cannot load into)",
63
}
64
65
66
@dataclass
67
class CrashInstrument:
68
    dylib: Path
69
    oak_bin: Optional[Path]
70
    git_bin: Optional[Path]
71
    git_exec_path: Optional[str]
72
    provenance: dict
73
    skipped: dict = None  # kind -> reason the instrumented copy is unavailable (e.g. codesign failed)
74
75
    def env(self, target: int, log: Optional[Path] = None, kind: str = "oak") -> dict:
76
        env = {"DYLD_INSERT_LIBRARIES": str(self.dylib), "P4_CRASH_AT": str(target)}
77
        if log is not None:
78
            env["P4_CRASH_LOG"] = str(log)
79
        if kind == "git" and self.git_exec_path:
80
            env["GIT_EXEC_PATH"] = self.git_exec_path
81
        return env
82
83
84
def _sha(path: Path) -> str:
85
    return hashlib.sha256(path.read_bytes()).hexdigest()
86
87
88
def build(workdir: Path, oak_bin: Optional[str], git_bin: Optional[str]) -> tuple[Optional[CrashInstrument], str]:
89
    """Build the interposer + re-signed subject copies. Returns (instrument, reason)."""
90
    if platform.system() != "Darwin":
91
        return None, "syscall crash points implemented for macOS DYLD interposition only (Linux LD_PRELOAD port pending)"
92
    clang = shutil.which("clang")
93
    codesign = shutil.which("codesign")
94
    if not clang or not codesign:
95
        return None, "clang and codesign required to build the crash interposer"
96
    workdir.mkdir(parents=True, exist_ok=True)
97
    source = workdir / "p4crash.c"
98
    source.write_text(INTERPOSER_SOURCE)
99
    dylib = workdir / "p4crash.dylib"
100
    proc = subprocess.run([clang, "-dynamiclib", "-O1", "-o", str(dylib), str(source)], capture_output=True, text=True)
101
    if proc.returncode != 0:
102
        return None, "interposer build failed: " + proc.stderr[-300:]
103
    provenance = {"interposer_sha256": _sha(dylib), "crash_model": "process_kill"}
104
    oak_copy = git_copy = None
105
    git_exec = None
106
    skipped: dict = {}
107
    if oak_bin:
108
        oak_copy = workdir / "oak-bin" / "oak"
109
        oak_copy.parent.mkdir(exist_ok=True)
110
        shutil.copy2(oak_bin, oak_copy)
111
        why = _resign(codesign, oak_copy)
112
        if why:
113
            skipped["oak"], oak_copy = why, None
114
        else:
115
            provenance.update(oak_original_sha256=_sha(Path(oak_bin)), oak_adhoc_sha256=_sha(oak_copy))
116
    if git_bin:
117
        git_copy = workdir / "git-bin" / "git"
118
        git_copy.parent.mkdir(exist_ok=True)
119
        shutil.copy2(git_bin, git_copy)
120
        why = _resign(codesign, git_copy)
121
        if why:
122
            skipped["git"], git_copy = why, None
123
        else:
124
            exec_path = subprocess.run([git_bin, "--exec-path"], capture_output=True, text=True).stdout.strip()
125
            git_exec = exec_path or None
126
            provenance.update(git_original_sha256=_sha(Path(git_bin)), git_adhoc_sha256=_sha(git_copy),
127
                              git_exec_path=git_exec)
128
    return CrashInstrument(dylib, oak_copy, git_copy, git_exec, provenance, skipped), ""
129
130
131
def _resign(codesign: str, binary: Path) -> str:
132
    """Ad-hoc re-sign a copy so DYLD_INSERT_LIBRARIES loads; '' on success, else the reason."""
133
    proc = subprocess.run([codesign, "-f", "-s", "-", str(binary)], capture_output=True, text=True)
134
    if proc.returncode != 0:
135
        return f"codesign -f -s - failed (rc={proc.returncode}): {(proc.stderr or proc.stdout).strip()[-200:]}"
136
    check = subprocess.run([codesign, "--verify", str(binary)], capture_output=True, text=True)
137
    if check.returncode != 0:
138
        return f"re-signed copy does not verify: {(check.stderr or check.stdout).strip()[-200:]}"
139
    return ""
140
141
142
def unavailable_reason(instrument: Optional[CrashInstrument], build_reason: str, kind: str) -> str:
143
    """Why syscall crash points cannot run for this subject ('' when they can)."""
144
    if instrument is None:
145
        return build_reason or "crash interposer unavailable"
146
    if (instrument.skipped or {}).get(kind):
147
        return instrument.skipped[kind]
148
    if (instrument.oak_bin if kind == "oak" else instrument.git_bin) is None:
149
        return f"no instrumented {kind} copy was built"
150
    return ""
151
152
153
def count_boundaries(log: Path, pid: Optional[int] = None) -> int:
154
    """Durability boundaries crossed by the first (top-level) process in the log."""
155
    if not log.exists():
156
        return 0
157
    counts: dict[str, int] = {}
158
    first: Optional[str] = None
159
    for line in log.read_text().splitlines():
160
        parts = line.split()
161
        if len(parts) < 3:
162
            continue
163
        first = first or parts[0]
164
        counts[parts[0]] = max(counts.get(parts[0], 0), int(parts[1]))
165
    if pid is not None:
166
        return counts.get(str(pid), 0)
167
    return counts.get(first, 0) if first else 0
168
169
170
def boundary_kinds(log: Path) -> list[str]:
171
    if not log.exists():
172
        return []
173
    lines = [line.split() for line in log.read_text().splitlines() if line.strip()]
174
    if not lines:
175
        return []
176
    first = lines[0][0]
177
    return [parts[2] for parts in lines if parts[0] == first]