775 lines · 27.0 KB · Rust
RawDownload
use crash_handler::{CrashEventResult, CrashHandler};use log::info;use minidumper::{LoopAction, MinidumpBinary, Server, SocketName};use parking_lot::Mutex;use serde::{Deserialize, Serialize};use std::{panic::Location, pin::Pin}; use system_specs::GpuSpecs; use std::{    env,    fs::{self, File},    io, panic,    path::{Path, PathBuf},    process::{self},    sync::{        Arc,        atomic::{AtomicBool, Ordering},    },    thread,    time::Duration,}; pub use minidumper::Client; const CRASH_HANDLER_PING_TIMEOUT: Duration = Duration::from_secs(60);const CRASH_HANDLER_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); /// Force a backtrace to be printed on panic.pub fn force_backtrace() {    let old_hook = panic::take_hook();    panic::set_hook(Box::new(move |info| {        unsafe { env::set_var("RUST_BACKTRACE", "1") };        old_hook(info);        // prevent the macOS crash dialog from popping up        if cfg!(target_os = "macos") {            std::process::exit(1);        }    }));} /// Install crash signal handlers and spawn the crash-handler subprocess.////// All work happens lazily in the returned future, so it runs on whichever/// executor polls it. The keepalive task is passed to `spawn` so the caller/// decides which executor to schedule it on.pub fn init<F, S, C, P>(    crash_init: InitCrashHandler,    spawn: S,    socket_path: P,    wait_timer: C,) -> impl Future<Output = Arc<Client>> + use<F, C, S, P>where    F: Future<Output = ()> + Send + Sync + 'static,    C: (Fn(Duration) -> F) + Send + Sync + 'static,    S: FnOnce(Pin<Box<dyn Future<Output = ()> + Send + 'static>>),    P: FnOnce(u32) -> PathBuf,{    connect_and_keepalive(crash_init, socket_path, wait_timer, spawn)} /// Spawn the crash-handler subprocess, connect the IPC client, and run the/// keepalive ping loop. This is the future returned by [`init`], so it runs on/// whichever executor the caller polls it with.async fn connect_and_keepalive<F, C, S, P>(    crash_init: InitCrashHandler,    socket_path: P,    wait_timer: C,    spawn: S,) -> Arc<Client>where    F: Future<Output = ()> + Send + Sync + 'static,    C: (Fn(Duration) -> F) + Send + Sync + 'static,    S: FnOnce(Pin<Box<dyn Future<Output = ()> + Send + 'static>>),    P: FnOnce(u32) -> PathBuf,{    let exe = env::current_exe().expect("unable to find ourselves");    let socket_path = socket_path(process::id());    let mut _crash_handler = spawn_crash_handler(&exe, &socket_path);    info!("spawning crash handler process");    let mut elapsed = Duration::ZERO;    let retry_frequency = Duration::from_millis(100);    let client = loop {        if let Ok(client) = Client::with_name(SocketName::Path(&socket_path)) {            info!("connected to crash handler process after {elapsed:?}");            break client;        }        elapsed += retry_frequency;        wait_timer(retry_frequency).await;    };    let client = Arc::new(client);     panic::set_hook({        let client = client.clone();        Box::new(move |payload| {            panic_hook(                client.clone(),                payload.payload_as_str().unwrap_or("Box<Any>"),                payload.location(),            )        })    });    info!("panic handler registered");    let handler = CrashHandler::attach(unsafe {        let client = client.clone();        let handler = move |crash_context: &crash_handler::CrashContext| {            // set when the first minidump request is made to avoid generating duplicate crash reports            static REQUESTED_MINIDUMP: AtomicBool = AtomicBool::new(false);             // only request a minidump once            let res = if REQUESTED_MINIDUMP                .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)                .is_ok()            {                #[cfg(target_os = "macos")]                macos::suspend_all_other_threads();                 // on macos this "ping" is needed to ensure that all our                // `client.send_message` calls have been processed before we trigger the                // minidump request.                client.ping().ok();                let r = client.request_dump(crash_context);                if let Err(e) = &r {                    eprintln!("failed to request dump: {:?}", e);                }                #[cfg(target_os = "macos")]                macos::resume_all_other_threads();                r.is_ok()            } else {                true            };            CrashEventResult::Handled(res)        };        crash_handler::make_crash_event(handler)    })    .expect("failed to attach signal handler");     info!("crash signal handlers installed");    send_crash_server_message(&client, CrashServerMessage::Init(crash_init));     #[cfg(all(target_os = "linux", target_env = "gnu"))]    if let Some(address) = abort_message_address() {        send_crash_server_message(            &client,            CrashServerMessage::AbortMessageLocation(AbortMessageLocation {                pid: process::id(),                address,            }),        );    }     #[cfg(target_os = "linux")]    handler.set_ptracer(Some(_crash_handler.id()));     info!("crash handler registered");    spawn(Box::pin({        let client = client.clone();        async move {            let _handler = { handler };            loop {                if let Err(e) = client.ping() {                    #[cfg(not(target_os = "windows"))]                    log::error!(                        "ping failed: {:?}, process exit status: {:?}",                        e,                        _crash_handler.try_status()                    );                    #[cfg(target_os = "windows")]                    log::error!("ping failed: {:?}", e,);                    break;                };                wait_timer(Duration::from_secs(10)).await;            }        }    }));    client} pub struct CrashServer {    initialization_params: Mutex<Option<InitCrashHandler>>,    panic_info: Mutex<Option<CrashPanic>>,    active_gpu: Mutex<Option<system_specs::GpuSpecs>>,    user_info: Mutex<Option<UserInfo>>,    abort_message_location: Mutex<Option<AbortMessageLocation>>,    has_connection: Arc<AtomicBool>,    logs_dir: PathBuf,} #[derive(Debug, Deserialize, Serialize, Clone)]pub struct CrashInfo {    pub init: InitCrashHandler,    pub panic: Option<CrashPanic>,    pub minidump_error: Option<String>,    /// The diagnostic the C runtime recorded before aborting the process, e.g.    /// glibc's "free(): invalid pointer". Only present when the crash was a    /// runtime-initiated abort rather than a signal like SIGSEGV or a panic.    #[serde(default)]    pub abort_message: Option<String>,    pub gpus: Vec<system_specs::GpuInfo>,    pub active_gpu: Option<system_specs::GpuSpecs>,    pub user_info: Option<UserInfo>,} /// Where to find the C runtime's abort diagnostic in the crashed process's/// memory. Sent by the client at startup so that after a crash the server can/// recover the message with `process_vm_readv`; the crashed process itself/// can't safely do this work, since its heap may be corrupt and its allocator/// locks may be held by the crashed thread.#[derive(Debug, Deserialize, Serialize, Clone, Copy)]pub struct AbortMessageLocation {    pub pid: u32,    pub address: u64,} #[derive(Debug, Deserialize, Serialize, Clone)]pub struct InitCrashHandler {    pub session_id: String,    pub zed_version: String,    pub binary: String,    pub release_channel: String,    pub commit_sha: String,} #[derive(Deserialize, Serialize, Debug, Clone)]pub struct CrashPanic {    pub message: String,    pub span: String,} #[derive(Deserialize, Serialize, Debug, Clone)]pub struct UserInfo {    pub metrics_id: Option<String>,    pub is_staff: Option<bool>,} fn send_crash_server_message(crash_client: &Arc<Client>, message: CrashServerMessage) {    let data = match serde_json::to_vec(&message) {        Ok(data) => data,        Err(err) => {            log::warn!("Failed to serialize crash server message: {:?}", err);            return;        }    };     if let Err(err) = crash_client.send_message(0, data) {        log::warn!("Failed to send data to crash server {:?}", err);    }} pub fn set_gpu_info(crash_client: &Arc<Client>, specs: GpuSpecs) {    send_crash_server_message(crash_client, CrashServerMessage::GPUInfo(specs));} pub fn set_user_info(crash_client: &Arc<Client>, info: UserInfo) {    send_crash_server_message(crash_client, CrashServerMessage::UserInfo(info));} #[derive(Serialize, Deserialize, Debug)]enum CrashServerMessage {    Init(InitCrashHandler),    Panic(CrashPanic),    GPUInfo(GpuSpecs),    UserInfo(UserInfo),    AbortMessageLocation(AbortMessageLocation),} /// glibc records the diagnostic it prints just before aborting (malloc integrity/// failures like "free(): invalid pointer", assertion failures, stack-smashing/// reports) in the private global `__abort_msg`, specifically so it can be/// recovered post-mortem. Resolve its address here, in a safe context at startup./// The symbol is only exported at the GLIBC_PRIVATE version, which plain `dlsym`/// won't resolve, and it has no stability guarantee, so a null result (e.g. musl,/// or a future glibc removing it) just disables this diagnostic.#[cfg(all(target_os = "linux", target_env = "gnu"))]fn abort_message_address() -> Option<u64> {    let ptr = unsafe {        libc::dlvsym(            libc::RTLD_DEFAULT,            c"__abort_msg".as_ptr(),            c"GLIBC_PRIVATE".as_ptr(),        )    };    std::ptr::NonNull::new(ptr).map(|ptr| ptr.as_ptr() as u64)} /// Read the crashed process's abort diagnostic. `__abort_msg` points to a/// `struct abort_msg_s { unsigned int size; char msg[]; }` that glibc allocates/// with mmap so that it stays intact even when the heap is corrupt. `size` is/// the total byte size of that mapping (header included, rounded up to whole/// pages), not the message length; the message itself is NUL-terminated.#[cfg(target_os = "linux")]fn read_abort_message(location: AbortMessageLocation) -> Option<String> {    let pointer_bytes = read_process_memory(location.pid, location.address, size_of::<usize>())?;    let message_address = usize::from_ne_bytes(pointer_bytes.try_into().ok()?) as u64;    if message_address == 0 {        return None;    }    let size_bytes = read_process_memory(location.pid, message_address, size_of::<u32>())?;    let size = u32::from_ne_bytes(size_bytes.try_into().ok()?);    let message_bytes = read_process_memory(        location.pid,        message_address + size_of::<u32>() as u64,        abort_message_read_len(size)?,    )?;    parse_abort_message(&message_bytes)} /// How many message bytes to read given the `size` field of glibc's/// `abort_msg_s`. `size` holds the total size of the mmap'd allocation, so a/// value that isn't a whole number of pages means the layout has changed and/// we shouldn't trust it. Reading is capped at (one page minus the header),/// which both bounds the work and ensures the read never extends past the end/// of the mapping.#[cfg(any(target_os = "linux", test))]fn abort_message_read_len(size: u32) -> Option<usize> {    // Every Linux page size (4 KiB, 16 KiB, 64 KiB, ...) is a multiple of 4 KiB.    const PAGE_MULTIPLE: usize = 4096;    const MAX_READ: usize = 4096;     let size = size as usize;    if size == 0 || !size.is_multiple_of(PAGE_MULTIPLE) {        log::warn!("__abort_msg size field {size} is not page-rounded; layout may have changed");        return None;    }    Some(size.min(MAX_READ) - size_of::<u32>())} /// The message is NUL-terminated inside a zero-filled mapping, so truncate at/// the first NUL; `trim` alone would keep the padding, since NUL is not/// whitespace.#[cfg(any(target_os = "linux", test))]fn parse_abort_message(bytes: &[u8]) -> Option<String> {    let len = bytes        .iter()        .position(|&byte| byte == 0)        .unwrap_or(bytes.len());    let message = String::from_utf8_lossy(&bytes[..len]).trim().to_string();    (!message.is_empty()).then_some(message)} #[cfg(target_os = "linux")]fn read_process_memory(pid: u32, address: u64, len: usize) -> Option<Vec<u8>> {    let mut buffer = vec![0u8; len];    let local = libc::iovec {        iov_base: buffer.as_mut_ptr().cast(),        iov_len: len,    };    let remote = libc::iovec {        iov_base: address as *mut libc::c_void,        iov_len: len,    };    let bytes_read =        unsafe { libc::process_vm_readv(pid as libc::pid_t, &local, 1, &remote, 1, 0) };    if bytes_read < 0 {        log::warn!(            "process_vm_readv of {len} bytes at {address:#x} in pid {pid} failed: {}",            io::Error::last_os_error()        );        return None;    }    if bytes_read as usize != len {        log::warn!(            "process_vm_readv short read at {address:#x} in pid {pid}: {bytes_read} of {len} bytes"        );        return None;    }    Some(buffer)} impl minidumper::ServerHandler for CrashServer {    fn create_minidump_file(&self) -> Result<(File, PathBuf), io::Error> {        let dump_path = self            .logs_dir            .join(                &self                    .initialization_params                    .lock()                    .as_ref()                    .expect("Missing initialization data")                    .session_id,            )            .with_extension("dmp");        let file = File::create(&dump_path)?;        Ok((file, dump_path))    }     fn on_minidump_created(&self, result: Result<MinidumpBinary, minidumper::Error>) -> LoopAction {        let minidump_error = match result {            Ok(MinidumpBinary { mut file, path, .. }) => {                use io::Write;                file.flush().ok();                // TODO: clean this up once https://github.com/EmbarkStudios/crash-handling/issues/101 is addressed                drop(file);                let original_file = File::open(&path).unwrap();                let compressed_path = path.with_extension("zstd");                let compressed_file = File::create(&compressed_path).unwrap();                zstd::stream::copy_encode(original_file, compressed_file, 0).ok();                fs::rename(&compressed_path, path).unwrap();                None            }            Err(e) => Some(format!("{e:?}")),        };         #[cfg(not(any(target_os = "linux", target_os = "freebsd")))]        let gpus = vec![];         #[cfg(any(target_os = "linux", target_os = "freebsd"))]        let gpus = match system_specs::read_gpu_info_from_sys_class_drm() {            Ok(gpus) => gpus,            Err(err) => {                log::warn!("Failed to collect GPU information for crash report: {err}");                vec![]            }        };         // The crashed process is still alive at this point: it stays parked in        // its signal handler until the server acknowledges the dump request,        // which happens after this callback returns.        #[cfg(target_os = "linux")]        let abort_message = (*self.abort_message_location.lock()).and_then(read_abort_message);        #[cfg(not(target_os = "linux"))]        let abort_message = None;         let crash_info = CrashInfo {            init: self                .initialization_params                .lock()                .clone()                .expect("not initialized"),            panic: self.panic_info.lock().clone(),            minidump_error,            abort_message,            active_gpu: self.active_gpu.lock().clone(),            gpus,            user_info: self.user_info.lock().clone(),        };         let crash_data_path = self            .logs_dir            .join(&crash_info.init.session_id)            .with_extension("json");         fs::write(crash_data_path, serde_json::to_vec(&crash_info).unwrap()).ok();         LoopAction::Exit    }     fn on_message(&self, _: u32, buffer: Vec<u8>) {        let message: CrashServerMessage =            serde_json::from_slice(&buffer).expect("invalid init data");        match message {            CrashServerMessage::Init(init_data) => {                self.initialization_params.lock().replace(init_data);            }            CrashServerMessage::Panic(crash_panic) => {                self.panic_info.lock().replace(crash_panic);            }            CrashServerMessage::GPUInfo(gpu_specs) => {                self.active_gpu.lock().replace(gpu_specs);            }            CrashServerMessage::UserInfo(user_info) => {                self.user_info.lock().replace(user_info);            }            CrashServerMessage::AbortMessageLocation(location) => {                self.abort_message_location.lock().replace(location);            }        }    }     fn on_client_disconnected(&self, _clients: usize) -> LoopAction {        LoopAction::Exit    }     fn on_client_connected(&self, _clients: usize) -> LoopAction {        self.has_connection.store(true, Ordering::SeqCst);        LoopAction::Continue    }} /// Rust's string-slicing panics embed the user's string content in the message,/// e.g. "byte index 4 is out of bounds of `a`". Strip that suffix so we/// don't upload arbitrary user text in crash reports.fn strip_user_string_from_panic(message: &str) -> String {    const STRING_PANIC_PREFIXES: &[&str] = &[        // Older rustc (pre-1.95):        "byte index ",        "begin <= end (",        // Newer rustc (1.95+):        // https://github.com/rust-lang/rust/pull/145024        "start byte index ",        "end byte index ",        "begin > end (",    ];     if (message.ends_with('`') || message.ends_with("`[...]"))        && STRING_PANIC_PREFIXES            .iter()            .any(|prefix| message.starts_with(prefix))        && let Some(open) = message.find('`')    {

Lines 1–500 of 775.