//! Utilities for testing and fuzzing out-of-memory handling. //! //! Inspired by SpiderMonkey's `oomTest()` helper: //! https://firefox-source-docs.mozilla.org/js/hacking_tips.html#how-to-debug-oomtest-failures use backtrace::Backtrace; use std::{alloc::GlobalAlloc, cell::Cell, mem, ptr, time}; use wasmtime_core::error::{Error, OutOfMemory, Result, bail}; /// An allocator for use with `OomTest`. #[non_exhaustive] pub struct OomTestAllocator; impl OomTestAllocator { /// Create a new OOM test allocator. pub const fn new() -> Self { OomTestAllocator } } #[derive(Clone, Debug, Default, PartialEq, Eq)] enum OomState { /// We are in code that is not part of an OOM test. #[default] OutsideOomTest, /// We are inside an OOM test and should inject an OOM when the counter /// reaches zero. OomOnAlloc { counter: u32, allow_alloc_after: bool, }, /// We are inside an OOM test and we already injected an OOM. DidOom { allow_alloc: bool }, } thread_local! { static OOM_STATE: Cell = const { Cell::new(OomState::OutsideOomTest) }; } /// Set the new OOM state, returning the old state. fn set_oom_state(state: OomState) -> OomState { OOM_STATE.with(|s| s.replace(state)) } /// RAII helper to set the OOM state within a block of code and reset it upon /// exiting that block (even if exiting via panic unwinding). struct ScopedOomState { prev_state: OomState, } impl ScopedOomState { fn new(state: OomState) -> Self { ScopedOomState { prev_state: set_oom_state(state), } } /// Finish this OOM state scope early, resetting the OOM state to what it /// was before this scope was created, and returning the previous state that /// was just overwritten by the reset. fn finish(&self) -> OomState { set_oom_state(self.prev_state.clone()) } } impl Drop for ScopedOomState { fn drop(&mut self) { set_oom_state(mem::take(&mut self.prev_state)); } } unsafe impl GlobalAlloc for OomTestAllocator { unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 { let old_state = set_oom_state(OomState::OutsideOomTest); let new_state; let ptr; 'outside_oom_test: { // NB: It's okay to log/backtrace/etc... in this block because the // current state is `OutsideOomTest`, so any re-entrant allocations // will be passed through to the system allocator. // Don't panic on allocation-after-OOM attempts if we // are already in the middle of panicking. That will // cause an abort and we won't get as good of an error // message for the original panic, which is most likely // some kind of test failure. if old_state == OomState::OutsideOomTest || std::thread::panicking() { new_state = old_state; ptr = unsafe { std::alloc::System.alloc(layout) }; break 'outside_oom_test; } let bt = Backtrace::new(); let bt = format!("{bt:?}"); // XXX: `env_logger` internally buffers writes in a `Vec` which // means our OOM tests might sporadically fail when you enable // logging to debug stuff, so simply let the allocation through if // `env_logger` is on the stack. if bt.contains("env_logger") { new_state = old_state; ptr = unsafe { std::alloc::System.alloc(layout) }; break 'outside_oom_test; } match old_state { OomState::OutsideOomTest => unreachable!("handled above"), OomState::OomOnAlloc { counter: 0, allow_alloc_after, } => { log::trace!( "injecting OOM for allocation: {layout:?}\nAllocation backtrace:\n{bt}" ); new_state = OomState::DidOom { allow_alloc: allow_alloc_after, }; ptr = ptr::null_mut(); } OomState::OomOnAlloc { counter: c, allow_alloc_after, } => { new_state = OomState::OomOnAlloc { counter: c - 1, allow_alloc_after, }; ptr = unsafe { std::alloc::System.alloc(layout) }; } OomState::DidOom { allow_alloc } => { log::trace!("Attempt to allocate {layout:?} after OOM:\n{bt}"); if allow_alloc { new_state = OomState::DidOom { allow_alloc: true }; ptr = ptr::null_mut(); } else { panic!( "OOM test attempted to allocate after OOM: {layout:?}\n\ \n\ Hint: if this is acceptable, configure the OOM test to allow allocation \ after OOM with `OomTest::allow_alloc_after_oom`" ) } } } } set_oom_state(new_state); ptr } unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) { unsafe { std::alloc::System.dealloc(ptr, layout); } } } /// A test helper that checks that some code handles OOM correctly. /// /// `OomTest` will only work correctly when `OomTestAllocator` is configured as /// the global allocator. /// /// `OomTest` does not support reentrancy, so you cannot run an `OomTest` within /// an `OomTest`. /// /// # Example /// /// ```no_run /// use std::time::Duration; /// use wasmtime::Result; /// use wasmtime_fuzzing::oom::{OomTest, OomTestAllocator}; /// /// #[global_allocator] /// static GLOBAL_ALOCATOR: OomTestAllocator = OomTestAllocator::new(); /// /// #[test] /// fn my_oom_test() -> Result<()> { /// OomTest::new() /// .max_iters(1_000_000) /// .max_duration(Duration::from_secs(5)) /// .allow_alloc_after_oom(true) /// .test(|| { /// todo!("insert code here that should handle OOM here...") /// }) /// } /// ``` pub struct OomTest { max_iters: Option, max_duration: Option, allow_alloc_after_oom: bool, } impl OomTest { /// Create a new OOM test. /// /// By default there is no iteration or time limit, tests will be executed /// until the pass (or fail). pub fn new() -> Self { let _ = env_logger::try_init(); // NB: `std::backtrace::Backtrace` doesn't have ways to handle // OOM. Ideally we would just disable the `"backtrace"` cargo feature, // but workspace feature resolution doesn't play nice with that. wasmtime_core::error::disable_backtrace(); OomTest { max_iters: None, max_duration: None, allow_alloc_after_oom: false, } } /// Configure the maximum number of times to run an OOM test. pub fn max_iters(&mut self, max_iters: u32) -> &mut Self { self.max_iters = Some(max_iters); self } /// Configure the maximum duration of time to run an OOM text. pub fn max_duration(&mut self, max_duration: time::Duration) -> &mut Self { self.max_duration = Some(max_duration); self } /// Configure whether to allow allocation attempts after an OOM has already /// been injected. /// /// The default is `false`. pub fn allow_alloc_after_oom(&mut self, allow: bool) -> &mut Self { self.allow_alloc_after_oom = allow; self } /// Repeatedly run the given test function, injecting OOMs at different /// times and checking that it correctly handles them. /// /// The test function should not use threads, or else allocations may not be /// tracked correctly and OOM injection may be incorrect. /// /// The test function should return an `Err(_)` if and only if it encounters /// an OOM. /// /// Returns early once the test function returns `Ok(())` before an OOM has /// been injected. pub fn test(&self, test_func: impl Fn() -> Result<()>) -> Result<()> { let start = time::Instant::now(); for i in 0.. { if self.max_iters.is_some_and(|n| i >= n) || self.max_duration.is_some_and(|d| start.elapsed() >= d) { break; } log::trace!("=== Injecting OOM after {i} allocations ==="); let (result, old_state) = { let guard = ScopedOomState::new(OomState::OomOnAlloc { counter: i, allow_alloc_after: self.allow_alloc_after_oom, }); assert_eq!(guard.prev_state, OomState::OutsideOomTest); let result = test_func(); (result, guard.finish()) }; match (result, old_state) { (_, OomState::OutsideOomTest) => unreachable!(), // The test function completed successfully before we ran out of // allocation fuel, so we're done. (Ok(()), OomState::OomOnAlloc { .. }) => break, // We injected an OOM and the test function handled it // correctly; continue to the next iteration. (Err(e), OomState::DidOom { .. }) if self.is_oom_error(&e) => {} // Missed OOMs. (Ok(()), OomState::DidOom { .. }) => { bail!("OOM test function missed an OOM: returned Ok(())"); } (Err(e), OomState::DidOom { .. }) => { return Err( e.context("OOM test function missed an OOM: returned non-OOM error") ); } // Unexpected error. (Err(e), OomState::OomOnAlloc { .. }) => { return Err( e.context("OOM test function returned an error when there was no OOM") ); } } } Ok(()) } fn is_oom_error(&self, e: &Error) -> bool { e.is::() } }