1 //! Memory management for executable code. 2 3 use anyhow::{anyhow, bail, Context, Result}; 4 use object::read::{File, Object, ObjectSection}; 5 use object::ObjectSymbol; 6 use std::mem::ManuallyDrop; 7 use std::ops::Range; 8 use wasmtime_environ::obj; 9 use wasmtime_jit_icache_coherence as icache_coherence; 10 use wasmtime_runtime::{libcalls, MmapVec, UnwindRegistration}; 11 12 /// Management of executable memory within a `MmapVec` 13 /// 14 /// This type consumes ownership of a region of memory and will manage the 15 /// executable permissions of the contained JIT code as necessary. 16 pub struct CodeMemory { 17 // NB: these are `ManuallyDrop` because `unwind_registration` must be 18 // dropped first since it refers to memory owned by `mmap`. 19 mmap: ManuallyDrop<MmapVec>, 20 unwind_registration: ManuallyDrop<Option<UnwindRegistration>>, 21 published: bool, 22 enable_branch_protection: bool, 23 24 relocations: Vec<(usize, obj::LibCall)>, 25 26 // Ranges within `self.mmap` of where the particular sections lie. 27 text: Range<usize>, 28 unwind: Range<usize>, 29 trap_data: Range<usize>, 30 wasm_data: Range<usize>, 31 address_map_data: Range<usize>, 32 func_name_data: Range<usize>, 33 info_data: Range<usize>, 34 dwarf: Range<usize>, 35 } 36 37 impl Drop for CodeMemory { 38 fn drop(&mut self) { 39 // Drop `unwind_registration` before `self.mmap` 40 unsafe { 41 ManuallyDrop::drop(&mut self.unwind_registration); 42 ManuallyDrop::drop(&mut self.mmap); 43 } 44 } 45 } 46 47 fn _assert() { 48 fn _assert_send_sync<T: Send + Sync>() {} 49 _assert_send_sync::<CodeMemory>(); 50 } 51 52 impl CodeMemory { 53 /// Creates a new `CodeMemory` by taking ownership of the provided 54 /// `MmapVec`. 55 /// 56 /// The returned `CodeMemory` manages the internal `MmapVec` and the 57 /// `publish` method is used to actually make the memory executable. 58 pub fn new(mmap: MmapVec) -> Result<Self> { 59 let obj = File::parse(&mmap[..]) 60 .with_context(|| "failed to parse internal compilation artifact")?; 61 62 let mut relocations = Vec::new(); 63 let mut text = 0..0; 64 let mut unwind = 0..0; 65 let mut enable_branch_protection = None; 66 let mut trap_data = 0..0; 67 let mut wasm_data = 0..0; 68 let mut address_map_data = 0..0; 69 let mut func_name_data = 0..0; 70 let mut info_data = 0..0; 71 let mut dwarf = 0..0; 72 for section in obj.sections() { 73 let data = section.data()?; 74 let name = section.name()?; 75 let range = subslice_range(data, &mmap); 76 77 // Double-check that sections are all aligned properly. 78 if section.align() != 0 && data.len() != 0 { 79 if (data.as_ptr() as u64 - mmap.as_ptr() as u64) % section.align() != 0 { 80 bail!( 81 "section `{}` isn't aligned to {:#x}", 82 section.name().unwrap_or("ERROR"), 83 section.align() 84 ); 85 } 86 } 87 88 match name { 89 obj::ELF_WASM_BTI => match data.len() { 90 1 => enable_branch_protection = Some(data[0] != 0), 91 _ => bail!("invalid `{name}` section"), 92 }, 93 ".text" => { 94 text = range; 95 96 // The text section might have relocations for things like 97 // libcalls which need to be applied, so handle those here. 98 // 99 // Note that only a small subset of possible relocations are 100 // handled. Only those required by the compiler side of 101 // things are processed. 102 for (offset, reloc) in section.relocations() { 103 assert_eq!(reloc.kind(), object::RelocationKind::Absolute); 104 assert_eq!(reloc.encoding(), object::RelocationEncoding::Generic); 105 assert_eq!(usize::from(reloc.size()), std::mem::size_of::<usize>()); 106 assert_eq!(reloc.addend(), 0); 107 let sym = match reloc.target() { 108 object::RelocationTarget::Symbol(id) => id, 109 other => panic!("unknown relocation target {other:?}"), 110 }; 111 let sym = obj.symbol_by_index(sym).unwrap().name().unwrap(); 112 let libcall = obj::LibCall::from_str(sym) 113 .unwrap_or_else(|| panic!("unknown symbol relocation: {sym}")); 114 115 let offset = usize::try_from(offset).unwrap(); 116 relocations.push((offset, libcall)); 117 } 118 } 119 UnwindRegistration::SECTION_NAME => unwind = range, 120 obj::ELF_WASM_DATA => wasm_data = range, 121 obj::ELF_WASMTIME_ADDRMAP => address_map_data = range, 122 obj::ELF_WASMTIME_TRAPS => trap_data = range, 123 obj::ELF_NAME_DATA => func_name_data = range, 124 obj::ELF_WASMTIME_INFO => info_data = range, 125 obj::ELF_WASMTIME_DWARF => dwarf = range, 126 127 _ => log::debug!("ignoring section {name}"), 128 } 129 } 130 Ok(Self { 131 mmap: ManuallyDrop::new(mmap), 132 unwind_registration: ManuallyDrop::new(None), 133 published: false, 134 enable_branch_protection: enable_branch_protection 135 .ok_or_else(|| anyhow!("missing `{}` section", obj::ELF_WASM_BTI))?, 136 text, 137 unwind, 138 trap_data, 139 address_map_data, 140 func_name_data, 141 dwarf, 142 info_data, 143 wasm_data, 144 relocations, 145 }) 146 } 147 148 /// Returns a reference to the underlying `MmapVec` this memory owns. 149 #[inline] 150 pub fn mmap(&self) -> &MmapVec { 151 &self.mmap 152 } 153 154 /// Returns the contents of the text section of the ELF executable this 155 /// represents. 156 #[inline] 157 pub fn text(&self) -> &[u8] { 158 &self.mmap[self.text.clone()] 159 } 160 161 /// Returns the contents of the `ELF_WASMTIME_DWARF` section. 162 #[inline] 163 pub fn dwarf(&self) -> &[u8] { 164 &self.mmap[self.dwarf.clone()] 165 } 166 167 /// Returns the data in the `ELF_NAME_DATA` section. 168 #[inline] 169 pub fn func_name_data(&self) -> &[u8] { 170 &self.mmap[self.func_name_data.clone()] 171 } 172 173 /// Returns the concatenated list of all data associated with this wasm 174 /// module. 175 /// 176 /// This is used for initialization of memories and all data ranges stored 177 /// in a `Module` are relative to the slice returned here. 178 #[inline] 179 pub fn wasm_data(&self) -> &[u8] { 180 &self.mmap[self.wasm_data.clone()] 181 } 182 183 /// Returns the encoded address map section used to pass to 184 /// `wasmtime_environ::lookup_file_pos`. 185 #[inline] 186 pub fn address_map_data(&self) -> &[u8] { 187 &self.mmap[self.address_map_data.clone()] 188 } 189 190 /// Returns the contents of the `ELF_WASMTIME_INFO` section, or an empty 191 /// slice if it wasn't found. 192 #[inline] 193 pub fn wasmtime_info(&self) -> &[u8] { 194 &self.mmap[self.info_data.clone()] 195 } 196 197 /// Returns the contents of the `ELF_WASMTIME_TRAPS` section, or an empty 198 /// slice if it wasn't found. 199 #[inline] 200 pub fn trap_data(&self) -> &[u8] { 201 &self.mmap[self.trap_data.clone()] 202 } 203 204 /// Publishes the internal ELF image to be ready for execution. 205 /// 206 /// This method can only be called once and will panic if called twice. This 207 /// will parse the ELF image from the original `MmapVec` and do everything 208 /// necessary to get it ready for execution, including: 209 /// 210 /// * Change page protections from read/write to read/execute. 211 /// * Register unwinding information with the OS 212 /// 213 /// After this function executes all JIT code should be ready to execute. 214 pub fn publish(&mut self) -> Result<()> { 215 assert!(!self.published); 216 self.published = true; 217 218 if self.text().is_empty() { 219 return Ok(()); 220 } 221 222 // The unsafety here comes from a few things: 223 // 224 // * We're actually updating some page protections to executable memory. 225 // 226 // * We're registering unwinding information which relies on the 227 // correctness of the information in the first place. This applies to 228 // both the actual unwinding tables as well as the validity of the 229 // pointers we pass in itself. 230 unsafe { 231 // First, if necessary, apply relocations. This can happen for 232 // things like libcalls which happen late in the lowering process 233 // that don't go through the Wasm-based libcalls layer that's 234 // indirected through the `VMContext`. Note that most modules won't 235 // have relocations, so this typically doesn't do anything. 236 self.apply_relocations()?; 237 238 // Next freeze the contents of this image by making all of the 239 // memory readonly. Nothing after this point should ever be modified 240 // so commit everything. For a compiled-in-memory image this will 241 // mean IPIs to evict writable mappings from other cores. For 242 // loaded-from-disk images this shouldn't result in IPIs so long as 243 // there weren't any relocations because nothing should have 244 // otherwise written to the image at any point either. 245 self.mmap.make_readonly(0..self.mmap.len())?; 246 247 let text = self.text(); 248 249 // Clear the newly allocated code from cache if the processor requires it 250 // 251 // Do this before marking the memory as R+X, technically we should be able to do it after 252 // but there are some CPU's that have had errata about doing this with read only memory. 253 icache_coherence::clear_cache(text.as_ptr().cast(), text.len()) 254 .expect("Failed cache clear"); 255 256 // Switch the executable portion from readonly to read/execute. 257 self.mmap 258 .make_executable(self.text.clone(), self.enable_branch_protection) 259 .context("unable to make memory executable")?; 260 261 // Flush any in-flight instructions from the pipeline 262 icache_coherence::pipeline_flush_mt().expect("Failed pipeline flush"); 263 264 // With all our memory set up use the platform-specific 265 // `UnwindRegistration` implementation to inform the general 266 // runtime that there's unwinding information available for all 267 // our just-published JIT functions. 268 self.register_unwind_info()?; 269 } 270 271 Ok(()) 272 } 273 274 unsafe fn apply_relocations(&mut self) -> Result<()> { 275 if self.relocations.is_empty() { 276 return Ok(()); 277 } 278 279 for (offset, libcall) in self.relocations.iter() { 280 let offset = self.text.start + offset; 281 let libcall = match libcall { 282 obj::LibCall::FloorF32 => libcalls::relocs::floorf32 as usize, 283 obj::LibCall::FloorF64 => libcalls::relocs::floorf64 as usize, 284 obj::LibCall::NearestF32 => libcalls::relocs::nearestf32 as usize, 285 obj::LibCall::NearestF64 => libcalls::relocs::nearestf64 as usize, 286 obj::LibCall::CeilF32 => libcalls::relocs::ceilf32 as usize, 287 obj::LibCall::CeilF64 => libcalls::relocs::ceilf64 as usize, 288 obj::LibCall::TruncF32 => libcalls::relocs::truncf32 as usize, 289 obj::LibCall::TruncF64 => libcalls::relocs::truncf64 as usize, 290 obj::LibCall::FmaF32 => libcalls::relocs::fmaf32 as usize, 291 obj::LibCall::FmaF64 => libcalls::relocs::fmaf64 as usize, 292 #[cfg(target_arch = "x86_64")] 293 obj::LibCall::X86Pshufb => libcalls::relocs::x86_pshufb as usize, 294 #[cfg(not(target_arch = "x86_64"))] 295 obj::LibCall::X86Pshufb => unreachable!(), 296 }; 297 self.mmap 298 .as_mut_ptr() 299 .add(offset) 300 .cast::<usize>() 301 .write_unaligned(libcall); 302 } 303 Ok(()) 304 } 305 306 unsafe fn register_unwind_info(&mut self) -> Result<()> { 307 if self.unwind.len() == 0 { 308 return Ok(()); 309 } 310 let text = self.text(); 311 let unwind_info = &self.mmap[self.unwind.clone()]; 312 let registration = 313 UnwindRegistration::new(text.as_ptr(), unwind_info.as_ptr(), unwind_info.len()) 314 .context("failed to create unwind info registration")?; 315 *self.unwind_registration = Some(registration); 316 Ok(()) 317 } 318 } 319 320 /// Returns the range of `inner` within `outer`, such that `outer[range]` is the 321 /// same as `inner`. 322 /// 323 /// This method requires that `inner` is a sub-slice of `outer`, and if that 324 /// isn't true then this method will panic. 325 fn subslice_range(inner: &[u8], outer: &[u8]) -> Range<usize> { 326 if inner.len() == 0 { 327 return 0..0; 328 } 329 330 assert!(outer.as_ptr() <= inner.as_ptr()); 331 assert!((&inner[inner.len() - 1] as *const _) <= (&outer[outer.len() - 1] as *const _)); 332 333 let start = inner.as_ptr() as usize - outer.as_ptr() as usize; 334 start..start + inner.len() 335 } 336