tenant.openagents/omega

1use crash_handler::{CrashEventResult, CrashHandler};
2use log::info;
3use minidumper::{LoopAction, MinidumpBinary, Server, SocketName};
4use parking_lot::Mutex;
5use serde::{Deserialize, Serialize};
6use std::{panic::Location, pin::Pin};
7
8use system_specs::GpuSpecs;
9
10use std::{
11    env,
12    fs::{self, File},
13    io, panic,
14    path::{Path, PathBuf},
15    process::{self},
16    sync::{
17        Arc,
18        atomic::{AtomicBool, Ordering},
19    },
20    thread,
21    time::Duration,
22};
23
24pub use minidumper::Client;
25
26const CRASH_HANDLER_PING_TIMEOUT: Duration = Duration::from_secs(60);
27const CRASH_HANDLER_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
28
29/// Force a backtrace to be printed on panic.
30pub fn force_backtrace() {
31    let old_hook = panic::take_hook();
32    panic::set_hook(Box::new(move |info| {
33        unsafe { env::set_var("RUST_BACKTRACE", "1") };
34        old_hook(info);
35        // prevent the macOS crash dialog from popping up
36        if cfg!(target_os = "macos") {
37            std::process::exit(1);
38        }
39    }));
40}
41
42/// Install crash signal handlers and spawn the crash-handler subprocess.
43///
44/// All work happens lazily in the returned future, so it runs on whichever
45/// executor polls it. The keepalive task is passed to `spawn` so the caller
46/// decides which executor to schedule it on.
47pub fn init<F, S, C, P>(
48    crash_init: InitCrashHandler,
49    spawn: S,
50    socket_path: P,
51    wait_timer: C,
52) -> impl Future<Output = Arc<Client>> + use<F, C, S, P>
53where
54    F: Future<Output = ()> + Send + Sync + 'static,
55    C: (Fn(Duration) -> F) + Send + Sync + 'static,
56    S: FnOnce(Pin<Box<dyn Future<Output = ()> + Send + 'static>>),
57    P: FnOnce(u32) -> PathBuf,
58{
59    connect_and_keepalive(crash_init, socket_path, wait_timer, spawn)
60}
61
62/// Spawn the crash-handler subprocess, connect the IPC client, and run the
63/// keepalive ping loop. This is the future returned by [`init`], so it runs on
64/// whichever executor the caller polls it with.
65async fn connect_and_keepalive<F, C, S, P>(
66    crash_init: InitCrashHandler,
67    socket_path: P,
68    wait_timer: C,
69    spawn: S,
70) -> Arc<Client>
71where
72    F: Future<Output = ()> + Send + Sync + 'static,
73    C: (Fn(Duration) -> F) + Send + Sync + 'static,
74    S: FnOnce(Pin<Box<dyn Future<Output = ()> + Send + 'static>>),
75    P: FnOnce(u32) -> PathBuf,
76{
77    let exe = env::current_exe().expect("unable to find ourselves");
78    let socket_path = socket_path(process::id());
79    let mut _crash_handler = spawn_crash_handler(&exe, &socket_path);
80    info!("spawning crash handler process");
81    let mut elapsed = Duration::ZERO;
82    let retry_frequency = Duration::from_millis(100);
83    let client = loop {
84        if let Ok(client) = Client::with_name(SocketName::Path(&socket_path)) {
85            info!("connected to crash handler process after {elapsed:?}");
86            break client;
87        }
88        elapsed += retry_frequency;
89        wait_timer(retry_frequency).await;
90    };
91    let client = Arc::new(client);
92
93    panic::set_hook({
94        let client = client.clone();
95        Box::new(move |payload| {
96            panic_hook(
97                client.clone(),
98                payload.payload_as_str().unwrap_or("Box<Any>"),
99                payload.location(),
100            )
101        })
102    });
103    info!("panic handler registered");
104    let handler = CrashHandler::attach(unsafe {
105        let client = client.clone();
106        let handler = move |crash_context: &crash_handler::CrashContext| {
107            // set when the first minidump request is made to avoid generating duplicate crash reports
108            static REQUESTED_MINIDUMP: AtomicBool = AtomicBool::new(false);
109
110            // only request a minidump once
111            let res = if REQUESTED_MINIDUMP
112                .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
113                .is_ok()
114            {
115                #[cfg(target_os = "macos")]
116                macos::suspend_all_other_threads();
117
118                // on macos this "ping" is needed to ensure that all our
119                // `client.send_message` calls have been processed before we trigger the
120                // minidump request.
121                client.ping().ok();
122                let r = client.request_dump(crash_context);
123                if let Err(e) = &r {
124                    eprintln!("failed to request dump: {:?}", e);
125                }
126                #[cfg(target_os = "macos")]
127                macos::resume_all_other_threads();
128                r.is_ok()
129            } else {
130                true
131            };
132            CrashEventResult::Handled(res)
133        };
134        crash_handler::make_crash_event(handler)
135    })
136    .expect("failed to attach signal handler");
137
138    info!("crash signal handlers installed");
139    send_crash_server_message(&client, CrashServerMessage::Init(crash_init));
140
141    #[cfg(all(target_os = "linux", target_env = "gnu"))]
142    if let Some(address) = abort_message_address() {
143        send_crash_server_message(
144            &client,
145            CrashServerMessage::AbortMessageLocation(AbortMessageLocation {
146                pid: process::id(),
147                address,
148            }),
149        );
150    }
151
152    #[cfg(target_os = "linux")]
153    handler.set_ptracer(Some(_crash_handler.id()));
154
155    info!("crash handler registered");
156    spawn(Box::pin({
157        let client = client.clone();
158        async move {
159            let _handler = { handler };
160            loop {
161                if let Err(e) = client.ping() {
162                    #[cfg(not(target_os = "windows"))]
163                    log::error!(
164                        "ping failed: {:?}, process exit status: {:?}",
165                        e,
166                        _crash_handler.try_status()
167                    );
168                    #[cfg(target_os = "windows")]
169                    log::error!("ping failed: {:?}", e,);
170                    break;
171                };
172                wait_timer(Duration::from_secs(10)).await;
173            }
174        }
175    }));
176    client
177}
178
179pub struct CrashServer {
180    initialization_params: Mutex<Option<InitCrashHandler>>,
181    panic_info: Mutex<Option<CrashPanic>>,
182    active_gpu: Mutex<Option<system_specs::GpuSpecs>>,
183    user_info: Mutex<Option<UserInfo>>,
184    abort_message_location: Mutex<Option<AbortMessageLocation>>,
185    has_connection: Arc<AtomicBool>,
186    logs_dir: PathBuf,
187}
188
189#[derive(Debug, Deserialize, Serialize, Clone)]
190pub struct CrashInfo {
191    pub init: InitCrashHandler,
192    pub panic: Option<CrashPanic>,
193    pub minidump_error: Option<String>,
194    /// The diagnostic the C runtime recorded before aborting the process, e.g.
195    /// glibc's "free(): invalid pointer". Only present when the crash was a
196    /// runtime-initiated abort rather than a signal like SIGSEGV or a panic.
197    #[serde(default)]
198    pub abort_message: Option<String>,
199    pub gpus: Vec<system_specs::GpuInfo>,
200    pub active_gpu: Option<system_specs::GpuSpecs>,
201    pub user_info: Option<UserInfo>,
202}
203
204/// Where to find the C runtime's abort diagnostic in the crashed process's
205/// memory. Sent by the client at startup so that after a crash the server can
206/// recover the message with `process_vm_readv`; the crashed process itself
207/// can't safely do this work, since its heap may be corrupt and its allocator
208/// locks may be held by the crashed thread.
209#[derive(Debug, Deserialize, Serialize, Clone, Copy)]
210pub struct AbortMessageLocation {
211    pub pid: u32,
212    pub address: u64,
213}
214
215#[derive(Debug, Deserialize, Serialize, Clone)]
216pub struct InitCrashHandler {
217    pub session_id: String,
218    pub zed_version: String,
219    pub binary: String,
220    pub release_channel: String,
221    pub commit_sha: String,
222}
223
224#[derive(Deserialize, Serialize, Debug, Clone)]
225pub struct CrashPanic {
226    pub message: String,
227    pub span: String,
228}
229
230#[derive(Deserialize, Serialize, Debug, Clone)]
231pub struct UserInfo {
232    pub metrics_id: Option<String>,
233    pub is_staff: Option<bool>,
234}
235
236fn send_crash_server_message(crash_client: &Arc<Client>, message: CrashServerMessage) {
237    let data = match serde_json::to_vec(&message) {
238        Ok(data) => data,
239        Err(err) => {
240            log::warn!("Failed to serialize crash server message: {:?}", err);
241            return;
242        }
243    };
244
245    if let Err(err) = crash_client.send_message(0, data) {
246        log::warn!("Failed to send data to crash server {:?}", err);
247    }
248}
249
250pub fn set_gpu_info(crash_client: &Arc<Client>, specs: GpuSpecs) {
251    send_crash_server_message(crash_client, CrashServerMessage::GPUInfo(specs));
252}
253
254pub fn set_user_info(crash_client: &Arc<Client>, info: UserInfo) {
255    send_crash_server_message(crash_client, CrashServerMessage::UserInfo(info));
256}
257
258#[derive(Serialize, Deserialize, Debug)]
259enum CrashServerMessage {
260    Init(InitCrashHandler),
261    Panic(CrashPanic),
262    GPUInfo(GpuSpecs),
263    UserInfo(UserInfo),
264    AbortMessageLocation(AbortMessageLocation),
265}
266
267/// glibc records the diagnostic it prints just before aborting (malloc integrity
268/// failures like "free(): invalid pointer", assertion failures, stack-smashing
269/// reports) in the private global `__abort_msg`, specifically so it can be
270/// recovered post-mortem. Resolve its address here, in a safe context at startup.
271/// The symbol is only exported at the GLIBC_PRIVATE version, which plain `dlsym`
272/// won't resolve, and it has no stability guarantee, so a null result (e.g. musl,
273/// or a future glibc removing it) just disables this diagnostic.
274#[cfg(all(target_os = "linux", target_env = "gnu"))]
275fn abort_message_address() -> Option<u64> {
276    let ptr = unsafe {
277        libc::dlvsym(
278            libc::RTLD_DEFAULT,
279            c"__abort_msg".as_ptr(),
280            c"GLIBC_PRIVATE".as_ptr(),
281        )
282    };
283    std::ptr::NonNull::new(ptr).map(|ptr| ptr.as_ptr() as u64)
284}
285
286/// Read the crashed process's abort diagnostic. `__abort_msg` points to a
287/// `struct abort_msg_s { unsigned int size; char msg[]; }` that glibc allocates
288/// with mmap so that it stays intact even when the heap is corrupt. `size` is
289/// the total byte size of that mapping (header included, rounded up to whole
290/// pages), not the message length; the message itself is NUL-terminated.
291#[cfg(target_os = "linux")]
292fn read_abort_message(location: AbortMessageLocation) -> Option<String> {
293    let pointer_bytes = read_process_memory(location.pid, location.address, size_of::<usize>())?;
294    let message_address = usize::from_ne_bytes(pointer_bytes.try_into().ok()?) as u64;
295    if message_address == 0 {
296        return None;
297    }
298    let size_bytes = read_process_memory(location.pid, message_address, size_of::<u32>())?;
299    let size = u32::from_ne_bytes(size_bytes.try_into().ok()?);
300    let message_bytes = read_process_memory(
301        location.pid,
302        message_address + size_of::<u32>() as u64,
303        abort_message_read_len(size)?,
304    )?;
305    parse_abort_message(&message_bytes)
306}
307
308/// How many message bytes to read given the `size` field of glibc's
309/// `abort_msg_s`. `size` holds the total size of the mmap'd allocation, so a
310/// value that isn't a whole number of pages means the layout has changed and
311/// we shouldn't trust it. Reading is capped at (one page minus the header),
312/// which both bounds the work and ensures the read never extends past the end
313/// of the mapping.
314#[cfg(any(target_os = "linux", test))]
315fn abort_message_read_len(size: u32) -> Option<usize> {
316    // Every Linux page size (4 KiB, 16 KiB, 64 KiB, ...) is a multiple of 4 KiB.
317    const PAGE_MULTIPLE: usize = 4096;
318    const MAX_READ: usize = 4096;
319
320    let size = size as usize;
321    if size == 0 || !size.is_multiple_of(PAGE_MULTIPLE) {
322        log::warn!("__abort_msg size field {size} is not page-rounded; layout may have changed");
323        return None;
324    }
325    Some(size.min(MAX_READ) - size_of::<u32>())
326}
327
328/// The message is NUL-terminated inside a zero-filled mapping, so truncate at
329/// the first NUL; `trim` alone would keep the padding, since NUL is not
330/// whitespace.
331#[cfg(any(target_os = "linux", test))]
332fn parse_abort_message(bytes: &[u8]) -> Option<String> {
333    let len = bytes
334        .iter()
335        .position(|&byte| byte == 0)
336        .unwrap_or(bytes.len());
337    let message = String::from_utf8_lossy(&bytes[..len]).trim().to_string();
338    (!message.is_empty()).then_some(message)
339}
340
341#[cfg(target_os = "linux")]
342fn read_process_memory(pid: u32, address: u64, len: usize) -> Option<Vec<u8>> {
343    let mut buffer = vec![0u8; len];
344    let local = libc::iovec {
345        iov_base: buffer.as_mut_ptr().cast(),
346        iov_len: len,
347    };
348    let remote = libc::iovec {
349        iov_base: address as *mut libc::c_void,
350        iov_len: len,
351    };
352    let bytes_read =
353        unsafe { libc::process_vm_readv(pid as libc::pid_t, &local, 1, &remote, 1, 0) };
354    if bytes_read < 0 {
355        log::warn!(
356            "process_vm_readv of {len} bytes at {address:#x} in pid {pid} failed: {}",
357            io::Error::last_os_error()
358        );
359        return None;
360    }
361    if bytes_read as usize != len {
362        log::warn!(
363            "process_vm_readv short read at {address:#x} in pid {pid}: {bytes_read} of {len} bytes"
364        );
365        return None;
366    }
367    Some(buffer)
368}
369
370impl minidumper::ServerHandler for CrashServer {
371    fn create_minidump_file(&self) -> Result<(File, PathBuf), io::Error> {
372        let dump_path = self
373            .logs_dir
374            .join(
375                &self
376                    .initialization_params
377                    .lock()
378                    .as_ref()
379                    .expect("Missing initialization data")
380                    .session_id,
381            )
382            .with_extension("dmp");
383        let file = File::create(&dump_path)?;
384        Ok((file, dump_path))
385    }
386
387    fn on_minidump_created(&self, result: Result<MinidumpBinary, minidumper::Error>) -> LoopAction {
388        let minidump_error = match result {
389            Ok(MinidumpBinary { mut file, path, .. }) => {
390                use io::Write;
391                file.flush().ok();
392                // TODO: clean this up once https://github.com/EmbarkStudios/crash-handling/issues/101 is addressed
393                drop(file);
394                let original_file = File::open(&path).unwrap();
395                let compressed_path = path.with_extension("zstd");
396                let compressed_file = File::create(&compressed_path).unwrap();
397                zstd::stream::copy_encode(original_file, compressed_file, 0).ok();
398                fs::rename(&compressed_path, path).unwrap();
399                None
400            }
401            Err(e) => Some(format!("{e:?}")),
402        };
403
404        #[cfg(not(any(target_os = "linux", target_os = "freebsd")))]
405        let gpus = vec![];
406
407        #[cfg(any(target_os = "linux", target_os = "freebsd"))]
408        let gpus = match system_specs::read_gpu_info_from_sys_class_drm() {
409            Ok(gpus) => gpus,
410            Err(err) => {
411                log::warn!("Failed to collect GPU information for crash report: {err}");
412                vec![]
413            }
414        };
415
416        // The crashed process is still alive at this point: it stays parked in
417        // its signal handler until the server acknowledges the dump request,
418        // which happens after this callback returns.
419        #[cfg(target_os = "linux")]
420        let abort_message = (*self.abort_message_location.lock()).and_then(read_abort_message);
421        #[cfg(not(target_os = "linux"))]
422        let abort_message = None;
423
424        let crash_info = CrashInfo {
425            init: self
426                .initialization_params
427                .lock()
428                .clone()
429                .expect("not initialized"),
430            panic: self.panic_info.lock().clone(),
431            minidump_error,
432            abort_message,
433            active_gpu: self.active_gpu.lock().clone(),
434            gpus,
435            user_info: self.user_info.lock().clone(),
436        };
437
438        let crash_data_path = self
439            .logs_dir
440            .join(&crash_info.init.session_id)
441            .with_extension("json");
442
443        fs::write(crash_data_path, serde_json::to_vec(&crash_info).unwrap()).ok();
444
445        LoopAction::Exit
446    }
447
448    fn on_message(&self, _: u32, buffer: Vec<u8>) {
449        let message: CrashServerMessage =
450            serde_json::from_slice(&buffer).expect("invalid init data");
451        match message {
452            CrashServerMessage::Init(init_data) => {
453                self.initialization_params.lock().replace(init_data);
454            }
455            CrashServerMessage::Panic(crash_panic) => {
456                self.panic_info.lock().replace(crash_panic);
457            }
458            CrashServerMessage::GPUInfo(gpu_specs) => {
459                self.active_gpu.lock().replace(gpu_specs);
460            }
461            CrashServerMessage::UserInfo(user_info) => {
462                self.user_info.lock().replace(user_info);
463            }
464            CrashServerMessage::AbortMessageLocation(location) => {
465                self.abort_message_location.lock().replace(location);
466            }
467        }
468    }
469
470    fn on_client_disconnected(&self, _clients: usize) -> LoopAction {
471        LoopAction::Exit
472    }
473
474    fn on_client_connected(&self, _clients: usize) -> LoopAction {
475        self.has_connection.store(true, Ordering::SeqCst);
476        LoopAction::Continue
477    }
478}
479
480/// Rust's string-slicing panics embed the user's string content in the message,
481/// e.g. "byte index 4 is out of bounds of `a`". Strip that suffix so we
482/// don't upload arbitrary user text in crash reports.
483fn strip_user_string_from_panic(message: &str) -> String {
484    const STRING_PANIC_PREFIXES: &[&str] = &[
485        // Older rustc (pre-1.95):
486        "byte index ",
487        "begin <= end (",
488        // Newer rustc (1.95+):
489        // https://github.com/rust-lang/rust/pull/145024
490        "start byte index ",
491        "end byte index ",
492        "begin > end (",
493    ];
494
495    if (message.ends_with('`') || message.ends_with("`[...]"))
496        && STRING_PANIC_PREFIXES
497            .iter()
498            .any(|prefix| message.starts_with(prefix))
499        && let Some(open) = message.find('`')
500    {
501        return format!("{} `<redacted>`", &message[..open]);
502    }
503    message.to_owned()
504}
505
506pub fn panic_hook(crash_client: Arc<Client>, message: &str, location: Option<&Location>) {
507    let message = strip_user_string_from_panic(message);
508
509    let span = location
510        .map(|loc| format!("{}:{}", loc.file(), loc.line()))
511        .unwrap_or_default();
512
513    let current_thread = std::thread::current();
514    let thread_name = current_thread.name().unwrap_or("<unnamed>");
515
516    let location = location.map_or_else(|| "<unknown>".to_owned(), |location| location.to_string());
517    log::error!("thread '{thread_name}' panicked at {location}:\n{message}...");
518
519    send_crash_server_message(
520        &crash_client,
521        CrashServerMessage::Panic(CrashPanic { message, span }),
522    );
523    log::error!("triggering a crash to generate a minidump...");
524
525    #[cfg(target_os = "macos")]
526    macos::set_panic_thread_id();
527    #[cfg(target_os = "windows")]
528    {
529        // https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
530        CrashHandler.simulate_exception(Some(234)); // (MORE_DATA_AVAILABLE)
531    }
532    #[cfg(not(target_os = "windows"))]
533    {
534        std::process::abort();
535    }
536}
537
538#[cfg(target_os = "macos")]
539mod macos {
540    static PANIC_THREAD_ID: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
541
542    pub(super) fn set_panic_thread_id() {
543        PANIC_THREAD_ID.store(
544            unsafe { mach2::mach_init::mach_thread_self() },
545            std::sync::atomic::Ordering::Release,
546        );
547    }
548
549    pub(super) unsafe fn suspend_all_other_threads() {
550        let task = unsafe { mach2::traps::current_task() };
551        let mut threads: mach2::mach_types::thread_act_array_t = std::ptr::null_mut();
552        let mut count = 0;
553        unsafe {
554            mach2::task::task_threads(task, &raw mut threads, &raw mut count);
555        }
556        let current = unsafe { mach2::mach_init::mach_thread_self() };
557        for i in 0..count {
558            let t = unsafe { *threads.add(i as usize) };
559            if t != current {
560                unsafe { mach2::thread_act::thread_suspend(t) };
561            }
562        }
563    }
564
565    pub(super) unsafe fn resume_all_other_threads() {
566        let task = unsafe { mach2::traps::current_task() };
567        let mut threads: mach2::mach_types::thread_act_array_t = std::ptr::null_mut();
568        let mut count = 0;
569        unsafe {
570            mach2::task::task_threads(task, &raw mut threads, &raw mut count);
571        }
572        let current = unsafe { mach2::mach_init::mach_thread_self() };
573        for i in 0..count {
574            let t = unsafe { *threads.add(i as usize) };
575            if t != current {
576                unsafe { mach2::thread_act::thread_resume(t) };
577            }
578        }
579    }
580}
581#[cfg(not(target_os = "windows"))]
582fn spawn_crash_handler(exe: &Path, socket_name: &Path) -> async_process::Child {
583    async_process::Command::new(exe)
584        .arg("--crash-handler")
585        .arg(&socket_name)
586        .spawn()
587        .expect("unable to spawn server process")
588}
589
590#[cfg(target_os = "windows")]
591fn spawn_crash_handler(exe: &Path, socket_name: &Path) {
592    use std::ffi::OsStr;
593    use std::iter::once;
594    use std::os::windows::ffi::OsStrExt;
595    use windows::Win32::System::Threading::{
596        CreateProcessW, PROCESS_CREATION_FLAGS, PROCESS_INFORMATION, STARTF_FORCEOFFFEEDBACK,
597        STARTUPINFOW,
598    };
599    use windows::core::PWSTR;
600
601    let mut command_line: Vec<u16> = OsStr::new(&format!(
602        "\"{}\" --crash-handler \"{}\"",
603        exe.display(),
604        socket_name.display()
605    ))
606    .encode_wide()
607    .chain(once(0))
608    .collect();
609
610    let mut startup_info = STARTUPINFOW::default();
611    startup_info.cb = std::mem::size_of::<STARTUPINFOW>() as u32;
612
613    // By default, Windows enables a "busy" cursor when a GUI application is launched.
614    // This cursor is disabled once the application starts processing window messages.
615    // Since the crash handler process doesn't process messages, this "busy" cursor stays enabled for a long time.
616    // Disable the cursor feedback to prevent this from happening.
617    startup_info.dwFlags = STARTF_FORCEOFFFEEDBACK;
618
619    let mut process_info = PROCESS_INFORMATION::default();
620
621    unsafe {
622        CreateProcessW(
623            None,
624            Some(PWSTR(command_line.as_mut_ptr())),
625            None,
626            None,
627            false,
628            PROCESS_CREATION_FLAGS(0),
629            None,
630            None,
631            &startup_info,
632            &mut process_info,
633        )
634        .expect("unable to spawn server process");
635
636        windows::Win32::Foundation::CloseHandle(process_info.hProcess).ok();
637        windows::Win32::Foundation::CloseHandle(process_info.hThread).ok();
638    }
639}
640
641pub fn crash_server(socket: &Path, logs_dir: PathBuf) {
642    let Ok(mut server) = Server::with_name(SocketName::Path(socket)) else {
643        log::info!("Couldn't create socket, there may already be a running crash server");
644        return;
645    };
646
647    let shutdown = Arc::new(AtomicBool::new(false));
648    let has_connection = Arc::new(AtomicBool::new(false));
649
650    thread::Builder::new()
651        .name("CrashServerTimeout".to_owned())
652        .spawn({
653            let shutdown = shutdown.clone();
654            let has_connection = has_connection.clone();
655            move || {
656                std::thread::sleep(CRASH_HANDLER_CONNECT_TIMEOUT);
657                if !has_connection.load(Ordering::SeqCst) {
658                    shutdown.store(true, Ordering::SeqCst);
659                }
660            }
661        })
662        .unwrap();
663
664    server
665        .run(
666            Box::new(CrashServer {
667                initialization_params: Mutex::default(),
668                panic_info: Mutex::default(),
669                user_info: Mutex::default(),
670                abort_message_location: Mutex::default(),
671                has_connection,
672                active_gpu: Mutex::default(),
673                logs_dir,
674            }),
675            &shutdown,
676            Some(CRASH_HANDLER_PING_TIMEOUT),
677        )
678        .expect("failed to run server");
679}
680
681#[cfg(test)]
682mod tests {
683    use super::*;
684
685    #[test]
686    fn abort_message_read_len_requires_page_rounded_total() {
687        assert_eq!(abort_message_read_len(0), None);
688        // A message length rather than a mapping total means the glibc layout
689        // has changed out from under us.
690        assert_eq!(abort_message_read_len(23), None);
691        assert_eq!(abort_message_read_len(4097), None);
692        // The read must stay within the mapping: one page minus the header.
693        assert_eq!(abort_message_read_len(4096), Some(4092));
694        // Larger totals (long messages, larger page sizes) are clamped.
695        assert_eq!(abort_message_read_len(8192), Some(4092));
696        assert_eq!(abort_message_read_len(65536), Some(4092));
697    }
698
699    #[test]
700    fn parse_abort_message_truncates_at_nul() {
701        let mut buffer = b"free(): invalid pointer\n\0".to_vec();
702        buffer.resize(4092, 0);
703        assert_eq!(
704            parse_abort_message(&buffer),
705            Some("free(): invalid pointer".to_string())
706        );
707    }
708
709    #[test]
710    fn parse_abort_message_handles_missing_nul() {
711        assert_eq!(
712            parse_abort_message(b"double free or corruption (out)"),
713            Some("double free or corruption (out)".to_string())
714        );
715    }
716
717    #[test]
718    fn parse_abort_message_rejects_empty() {
719        assert_eq!(parse_abort_message(&[]), None);
720        assert_eq!(parse_abort_message(&[0; 16]), None);
721        assert_eq!(parse_abort_message(b"\n \0garbage after nul"), None);
722    }
723
724    /// End-to-end check of `read_abort_message` against a synthetic
725    /// `abort_msg_s` in this very process (`process_vm_readv` may always read
726    /// one's own memory). The message page is followed by a `PROT_NONE` guard
727    /// page so the test fails if the read ever extends past the mapping glibc
728    /// would have allocated.
729    #[cfg(target_os = "linux")]
730    #[test]
731    fn read_abort_message_reads_glibc_layout_from_a_live_process() {
732        let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;
733        unsafe {
734            let mapping = libc::mmap(
735                std::ptr::null_mut(),
736                2 * page_size,
737                libc::PROT_READ | libc::PROT_WRITE,
738                libc::MAP_ANON | libc::MAP_PRIVATE,
739                -1,
740                0,
741            );
742            assert_ne!(mapping, libc::MAP_FAILED);
743            assert_eq!(
744                libc::mprotect(
745                    mapping.cast::<u8>().add(page_size).cast(),
746                    page_size,
747                    libc::PROT_NONE
748                ),
749                0
750            );
751
752            mapping.cast::<u32>().write(page_size as u32);
753            let message = b"free(): invalid pointer\n\0";
754            std::ptr::copy_nonoverlapping(
755                message.as_ptr(),
756                mapping.cast::<u8>().add(size_of::<u32>()),
757                message.len(),
758            );
759
760            // Stands in for the `__abort_msg` global: a pointer variable whose
761            // address we hand to the reader.
762            let abort_msg: *mut libc::c_void = mapping;
763            let location = AbortMessageLocation {
764                pid: process::id(),
765                address: (&raw const abort_msg) as u64,
766            };
767            assert_eq!(
768                read_abort_message(location),
769                Some("free(): invalid pointer".to_string())
770            );
771
772            libc::munmap(mapping, 2 * page_size);
773        }
774    }
775}
776