1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Allocator support. 4 //! 5 //! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" 6 //! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the 7 //! typical application of the different kernel allocators. 8 //! 9 //! Reference: <https://docs.kernel.org/core-api/memory-allocation.html> 10 11 use super::{flags::*, Flags}; 12 use core::alloc::{GlobalAlloc, Layout}; 13 use core::ptr; 14 use core::ptr::NonNull; 15 16 use crate::alloc::{AllocError, Allocator}; 17 use crate::bindings; 18 use crate::pr_warn; 19 20 /// The contiguous kernel allocator. 21 /// 22 /// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also 23 /// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. 24 /// 25 /// For more details see [self]. 26 pub struct Kmalloc; 27 28 /// The virtually contiguous kernel allocator. 29 /// 30 /// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel 31 /// virtual space. It is typically used for large allocations. The memory allocated with this 32 /// allocator is not physically contiguous. 33 /// 34 /// For more details see [self]. 35 pub struct Vmalloc; 36 37 /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. 38 fn aligned_size(new_layout: Layout) -> usize { 39 // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. 40 let layout = new_layout.pad_to_align(); 41 42 // Note that `layout.size()` (after padding) is guaranteed to be a multiple of `layout.align()` 43 // which together with the slab guarantees means the `krealloc` will return a properly aligned 44 // object (see comments in `kmalloc()` for more information). 45 layout.size() 46 } 47 48 /// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. 49 /// 50 /// # Safety 51 /// 52 /// - `ptr` can be either null or a pointer which has been allocated by this allocator. 53 /// - `new_layout` must have a non-zero size. 54 pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 { 55 let size = aligned_size(new_layout); 56 57 // SAFETY: 58 // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the 59 // function safety requirement. 60 // - `size` is greater than 0 since it's from `layout.size()` (which cannot be zero according 61 // to the function safety requirement) 62 unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } 63 } 64 65 /// # Invariants 66 /// 67 /// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. 68 struct ReallocFunc( 69 unsafe extern "C" fn(*const core::ffi::c_void, usize, u32) -> *mut core::ffi::c_void, 70 ); 71 72 impl ReallocFunc { 73 // INVARIANT: `krealloc` satisfies the type invariants. 74 const KREALLOC: Self = Self(bindings::krealloc); 75 76 // INVARIANT: `vrealloc` satisfies the type invariants. 77 const VREALLOC: Self = Self(bindings::vrealloc); 78 79 /// # Safety 80 /// 81 /// This method has the same safety requirements as [`Allocator::realloc`]. 82 /// 83 /// # Guarantees 84 /// 85 /// This method has the same guarantees as `Allocator::realloc`. Additionally 86 /// - it accepts any pointer to a valid memory allocation allocated by this function. 87 /// - memory allocated by this function remains valid until it is passed to this function. 88 unsafe fn call( 89 &self, 90 ptr: Option<NonNull<u8>>, 91 layout: Layout, 92 old_layout: Layout, 93 flags: Flags, 94 ) -> Result<NonNull<[u8]>, AllocError> { 95 let size = aligned_size(layout); 96 let ptr = match ptr { 97 Some(ptr) => { 98 if old_layout.size() == 0 { 99 ptr::null() 100 } else { 101 ptr.as_ptr() 102 } 103 } 104 None => ptr::null(), 105 }; 106 107 // SAFETY: 108 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that 109 // `ptr` is NULL or valid. 110 // - `ptr` is either NULL or valid by the safety requirements of this function. 111 // 112 // GUARANTEE: 113 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. 114 // - Those functions provide the guarantees of this function. 115 let raw_ptr = unsafe { 116 // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. 117 self.0(ptr.cast(), size, flags.0).cast() 118 }; 119 120 let ptr = if size == 0 { 121 crate::alloc::dangling_from_layout(layout) 122 } else { 123 NonNull::new(raw_ptr).ok_or(AllocError)? 124 }; 125 126 Ok(NonNull::slice_from_raw_parts(ptr, size)) 127 } 128 } 129 130 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 131 // - memory remains valid until it is explicitly freed, 132 // - passing a pointer to a valid memory allocation is OK, 133 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 134 unsafe impl Allocator for Kmalloc { 135 #[inline] 136 unsafe fn realloc( 137 ptr: Option<NonNull<u8>>, 138 layout: Layout, 139 old_layout: Layout, 140 flags: Flags, 141 ) -> Result<NonNull<[u8]>, AllocError> { 142 // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. 143 unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } 144 } 145 } 146 147 // SAFETY: TODO. 148 unsafe impl GlobalAlloc for Kmalloc { 149 unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 150 // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety 151 // requirement. 152 unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) } 153 } 154 155 unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { 156 // SAFETY: TODO. 157 unsafe { 158 bindings::kfree(ptr as *const core::ffi::c_void); 159 } 160 } 161 162 unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 163 // SAFETY: 164 // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not 165 // overflow `isize` by the function safety requirement. 166 // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two). 167 let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; 168 169 // SAFETY: 170 // - `ptr` is either null or a pointer allocated by this allocator by the function safety 171 // requirement. 172 // - the size of `layout` is not zero because `new_size` is not zero by the function safety 173 // requirement. 174 unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) } 175 } 176 177 unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { 178 // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety 179 // requirement. 180 unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) } 181 } 182 } 183 184 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 185 // - memory remains valid until it is explicitly freed, 186 // - passing a pointer to a valid memory allocation is OK, 187 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 188 unsafe impl Allocator for Vmalloc { 189 #[inline] 190 unsafe fn realloc( 191 ptr: Option<NonNull<u8>>, 192 layout: Layout, 193 old_layout: Layout, 194 flags: Flags, 195 ) -> Result<NonNull<[u8]>, AllocError> { 196 // TODO: Support alignments larger than PAGE_SIZE. 197 if layout.align() > bindings::PAGE_SIZE { 198 pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); 199 return Err(AllocError); 200 } 201 202 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 203 // allocated with this `Allocator`. 204 unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } 205 } 206 } 207 208 #[global_allocator] 209 static ALLOCATOR: Kmalloc = Kmalloc; 210 211 // See <https://github.com/rust-lang/rust/pull/86844>. 212 #[no_mangle] 213 static __rust_no_alloc_shim_is_unstable: u8 = 0; 214