Search code examples
c++rustboostdllpanic

Panic from Rust cdylib is causing a fatal runtime error


I'm currently developing a plugin loading system that utilizes C++ Boost to load Rust cdylib modules. Everything is functioning correctly, except when I invoke panic!() within a cdylib. The panic originates from a separate thread, leading to the termination of the entire application. I'm actively seeking solutions to prevent the main application from being affected. Does anyone have suggestions on how to accomplish this?

Plugin lib.rs

struct FirstPlugin;

impl Plugin for FirstPlugin {
    fn load(&mut self) {
    println!("Load");
    panic!("Test");
    }

    fn unload(&mut self) {
        println!("Unload")
    }
}

impl Default for FirstPlugin {
    fn default() -> Self {
        Self {  }
    }
}

#[no_mangle]
#[link_section = ".vtable"]
pub unsafe extern "C" fn __FIRST_PLUGIN_VTABLE() -> ffi::PluginVTable {
    ffi::PluginVTable::new::<FirstPlugin>()
}

FFI Vtable

#[repr(C)]
#[derive(Clone, Copy)]
pub struct PluginVTable {
    
    // Construct and destruct
    pub construct: extern fn() -> *mut(),
    pub destruct: extern fn(*mut()),

    // Member methods
    pub load: extern fn(*mut ()),
    pub unload: extern fn(*mut ())

}

impl PluginVTable {
    
    pub unsafe fn new<T: Plugin>() -> PluginVTable {
    PluginVTable { 
            construct: std::mem::transmute(PluginVTable::construct::<T> as *mut()), 
        destruct: std::mem::transmute(PluginVTable::destruct::<T> as *mut()), 
        load: std::mem::transmute(T::load as *mut()), 
            unload: std::mem::transmute(T::unload as *mut()) 
    }
    }

    fn construct<T: Plugin>() -> *mut T {
    let plugin = Box::new(T::default());
    Box::into_raw(plugin)
    }

    unsafe fn destruct<T: Plugin>(plugin: *mut T) {
    let _ = Box::from_raw(plugin);
    }
}

Creating thread and call method

pub struct PluginInstance {
    instance: Option<std::thread::JoinHandle<PluginData>>
}

impl PluginInstance {

    fn new(vtable: PluginVTable) -> Self {

        let instance = std::thread::spawn(move||{
            PluginInstance::instance_thread(vtable)
        });

    Self { instance: Some(instance) }
    }

    fn instance_thread(vtable: PluginVTable) -> PluginData {
        
    let ptr = (vtable.construct)();
    (vtable.load)(ptr);

    //std::thread::park();
    //(vtable.unload)(ptr);
        //(vtable.destruct)(ptr);

    PluginData
    }
}

impl From<PluginVTable> for PluginInstance {
    fn from(vtable: PluginVTable) -> Self {
    PluginInstance::new(vtable)
    }
}

impl Drop for PluginInstance {
    fn drop(&mut self) {
    println!("Dropping instance")
        //if let Some(instance) = self.instance.take() {
    // instance.thread().unpark();
    // match instance.join() {
    //  Ok(data) => println!("Plugin ended succesfully {:?}", data),
    //  Err(e) => println!("Plugin ended with an error {:?}", e),
    //}
    //}
    }
}

Output with backtrace

    Finished dev [unoptimized + debuginfo] target(s) in 0.09s
     Running `target\debug\plug.exe`
Load
thread '<unnamed>' panicked at 'Test', src\lib.rs:8:9
stack backtrace:
   0:     0x7ffe7135871c - std::sys_common::backtrace::_print::impl$0::fmt
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\sys_common\backtrace.rs:44
   1:     0x7ffe7136712b - core::fmt::rt::Argument::fmt
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\core\src\fmt\rt.rs:138
   2:     0x7ffe7136712b - core::fmt::write
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\core\src\fmt\mod.rs:1094
   3:     0x7ffe713569ef - std::io::Write::write_fmt<std::sys::windows::stdio::Stderr>
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\io\mod.rs:1714
   4:     0x7ffe713584cb - std::sys_common::backtrace::_print
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\sys_common\backtrace.rs:47
   5:     0x7ffe713584cb - std::sys_common::backtrace::print
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\sys_common\backtrace.rs:34
   6:     0x7ffe7135a19a - std::panicking::default_hook::closure$1
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\panicking.rs:269
   7:     0x7ffe71359def - std::panicking::default_hook
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\panicking.rs:288
   8:     0x7ffe7135a84e - std::panicking::rust_panic_with_hook
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\panicking.rs:705
   9:     0x7ffe7135a6fa - std::panicking::begin_panic_handler::closure$0
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\panicking.rs:595
  10:     0x7ffe71359099 - std::sys_common::backtrace::__rust_end_short_backtrace<std::panicking::begin_panic_handler::closure_env$0,never$>
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\sys_common\backtrace.rs:151
  11:     0x7ffe7135a440 - std::panicking::begin_panic_handler
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\panicking.rs:593
  12:     0x7ffe7136c1d5 - core::panicking::panic_fmt
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\core\src\panicking.rs:67
  13:     0x7ffe71351886 - first::impl$0::load
                               at C:\Users\Johnny\Desktop\plug-rs\plugins\first\src\lib.rs:8
  14:     0x7ff79719425b - plug_loader::PluginInstance::instance_thread
                               at C:\Users\Johnny\Desktop\plug-rs\loader\src\lib.rs:29
  15:     0x7ff797198001 - plug_loader::impl$0::new::closure$0
                               at C:\Users\Johnny\Desktop\plug-rs\loader\src\lib.rs:19
  16:     0x7ff797199339 - std::sys_common::backtrace::__rust_begin_short_backtrace<plug_loader::impl$0::new::closure_env$0,plug_loader::PluginData>
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be\library\std\src\sys_common\backtrace.rs:135
  17:     0x7ff797199339 - std::sys_common::backtrace::__rust_begin_short_backtrace<plug_loader::impl$0::new::closure_env$0,plug_loader::PluginData>
  19:     0x7ff797199371 - core::panic::unwind_safe::impl$23::call_once<plug_loader::PluginData,std::thread::impl$0::spawn_unchecked_::closure$1::closure_env$0<plug_loader::impl$0::new::closure_env$0,plug_loader::PluginData> >
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be\library\core\src\panic\unwind_safe.rs:271
  20:     0x7ff79719651a - std::panicking::try::do_call<core::panic::unwind_safe::AssertUnwindSafe<std::thread::impl$0::spawn_unchecked_::closure$1::closure_env$0<plug_loader::impl$0::new::closure_env$0,plug_loader::PluginData> >,plug_loader::PluginData>
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be\library\std\src\panicking.rs:500
  21:     0x7ff7971966d3 - std::panicking::try::do_catch<core::panic::unwind_safe::AssertUnwindSafe<std::thread::impl$7::drop::closure_env$0<plug_loader::PluginData> >,tuple$<> >
  22:     0x7ff7971963f4 - std::panicking::try<plug_loader::PluginData,core::panic::unwind_safe::AssertUnwindSafe<std::thread::impl$0::spawn_unchecked_::closure$1::closure_env$0<plug_loader::impl$0::new::closure_env$0,plug_loader::PluginData> > >
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be\library\std\src\panicking.rs:464
  23:     0x7ff797195289 - std::panic::catch_unwind
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be\library\std\src\panic.rs:142
  24:     0x7ff797195289 - std::thread::impl$0::spawn_unchecked_::closure$1<plug_loader::impl$0::new::closure_env$0,plug_loader::PluginData>
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be\library\std\src\thread\mod.rs:528
  25:     0x7ff79719228e - core::ops::function::FnOnce::call_once<std::thread::impl$0::spawn_unchecked_::closure_env$1<plug_loader::impl$0::new::closure_env$0,plug_loader::PluginData>,tuple$<> >
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be\library\core\src\ops\function.rs:250
  26:     0x7ff7971a837c - std::sys::windows::thread::impl$0::new::thread_start
                               at /rustc/5680fa18feaa87f3ff04063800aec256c3d4b4be/library\std\src\sys\windows\thread.rs:57
  27:     0x7ffeb600257d - BaseThreadInitThunk
  28:     0x7ffeb758aa58 - RtlUserThreadStart
fatal runtime error: Rust cannot catch foreign exceptions
error: process didn't exit successfully: `target\debug\plug.exe` (exit code: 0xc0000409, STATUS_STACK_BUFFER_OVERRUN)

I tried using both panic=abort and panic=unwind. Also tried wrapping the (vtable.load)(ptr) with std::panic::catch_unwind.


Solution

  • Solved by wrapping function pointers with catch_unwind before adding them to the vtable.

    pub unsafe fn new<T: Plugin>() -> PluginVTable {
        PluginVTable { 
            load: std::mem::transmute(PluginVTable::load::<T> as *mut())
        }
    }
    
    unsafe fn load<T: Plugin>(ptr: *mut T) -> *mut() {
        match catch_unwind(||{ T::load(&mut *ptr) }) {
            Ok(_) => std::ptr::null_mut(),
            Err(e) => Box::into_raw(e) as *mut(),
        }
    }
    

    The panic can then be resumed by the caller using resume_unwind.

    let result = (vtable.load)(ptr) as *mut (dyn Any + Send);
    
    if !result.is_null() {
        std::panic::resume_unwind(Box::from_raw(result))
    }