1 //! An `Instance` contains all the runtime state used by execution of a 2 //! wasm module (except its callstack and register state). An 3 //! `InstanceHandle` is a reference-counting handle for an `Instance`. 4 5 use crate::runtime::vm::const_expr::{ConstEvalContext, ConstExprEvaluator}; 6 use crate::runtime::vm::export::Export; 7 use crate::runtime::vm::memory::{Memory, RuntimeMemoryCreator}; 8 use crate::runtime::vm::table::{Table, TableElement, TableElementType}; 9 use crate::runtime::vm::vmcontext::{ 10 VMBuiltinFunctionsArray, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition, 11 VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMStoreContext, 12 VMTableDefinition, VMTableImport, VMTagDefinition, VMTagImport, 13 }; 14 use crate::runtime::vm::{ 15 ExportFunction, ExportGlobal, ExportGlobalKind, ExportMemory, ExportTable, ExportTag, GcStore, 16 Imports, ModuleRuntimeInfo, SendSyncPtr, VMFunctionBody, VMGcRef, VMStore, VMStoreRawPtr, 17 VmPtr, VmSafe, WasmFault, 18 }; 19 use crate::store::{InstanceId, StoreInner, StoreOpaque}; 20 use crate::{StoreContextMut, prelude::*}; 21 use alloc::sync::Arc; 22 use core::alloc::Layout; 23 use core::ops::Range; 24 use core::ptr::NonNull; 25 #[cfg(target_has_atomic = "64")] 26 use core::sync::atomic::AtomicU64; 27 use core::{mem, ptr}; 28 #[cfg(feature = "gc")] 29 use wasmtime_environ::ModuleInternedTypeIndex; 30 use wasmtime_environ::{ 31 DataIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, 32 ElemIndex, EntityIndex, EntityRef, EntitySet, FuncIndex, GlobalIndex, HostPtr, MemoryIndex, 33 Module, PrimaryMap, PtrSize, TableIndex, TableInitialValue, TableSegmentElements, TagIndex, 34 Trap, VMCONTEXT_MAGIC, VMOffsets, VMSharedTypeIndex, WasmHeapTopType, 35 packed_option::ReservedValue, 36 }; 37 #[cfg(feature = "wmemcheck")] 38 use wasmtime_wmemcheck::Wmemcheck; 39 40 mod allocator; 41 pub use allocator::*; 42 43 /// The pair of an instance and a raw pointer its associated store. 44 /// 45 /// ### Safety 46 /// 47 /// Getting a borrow of a vmctx's store is one of the fundamental bits of unsafe 48 /// code in Wasmtime. No matter how we architect the runtime, some kind of 49 /// unsafe conversion from a raw vmctx pointer that Wasm is using into a Rust 50 /// struct must happen. 51 /// 52 /// It is our responsibility to ensure that multiple (exclusive) borrows of the 53 /// vmctx's store never exist at the same time. The distinction between the 54 /// `Instance` type (which doesn't expose its underlying vmctx pointer or a way 55 /// to get a borrow of its associated store) and this type (which does) is 56 /// designed to help with that. 57 /// 58 /// Going from a `*mut VMContext` to a `&mut StoreInner<T>` is naturally unsafe 59 /// due to the raw pointer usage, but additionally the `T` type parameter needs 60 /// to be the same `T` that was used to define the `dyn VMStore` trait object 61 /// that was stuffed into the vmctx. 62 /// 63 /// ### Usage 64 /// 65 /// Usage generally looks like: 66 /// 67 /// 1. You get a raw `*mut VMContext` from Wasm 68 /// 69 /// 2. You call `InstanceAndStore::from_vmctx` on that raw pointer 70 /// 71 /// 3. You then call `InstanceAndStore::unpack_mut` (or another helper) to get 72 /// the underlying `&mut Instance` and `&mut dyn VMStore` (or `&mut 73 /// StoreInner<T>`). 74 /// 75 /// 4. You then use whatever `Instance` methods you need to, each of which take 76 /// a store argument as necessary. 77 /// 78 /// In step (4) you no longer need to worry about double exclusive borrows of 79 /// the store, so long as you don't do (1-2) again. Note also that the borrow 80 /// checker prevents repeating step (3) if you never repeat (1-2). In general, 81 /// steps (1-3) should be done in a single, common, internally-unsafe, 82 /// plumbing-code bottleneck and the raw pointer should never be exposed to Rust 83 /// code that does (4) after the `InstanceAndStore` is created. Follow this 84 /// pattern, and everything using the resulting `Instance` and `Store` can be 85 /// safe code (at least, with regards to accessing the store itself). 86 /// 87 /// As an illustrative example, the common plumbing code for our various 88 /// libcalls performs steps (1-3) before calling into each actual libcall 89 /// implementation function that does (4). The plumbing code hides the raw vmctx 90 /// pointer and never gives out access to it to the libcall implementation 91 /// functions, nor does an `Instance` expose its internal vmctx pointer, which 92 /// would allow unsafely repeating steps (1-2). 93 #[repr(transparent)] 94 pub struct InstanceAndStore { 95 instance: Instance, 96 } 97 98 impl InstanceAndStore { 99 /// Converts the provided `*mut VMContext` to an `InstanceAndStore` 100 /// reference and calls the provided closure with it. 101 /// 102 /// This method will move the `vmctx` pointer backwards to point to the 103 /// original `Instance` that precedes it. The closure is provided a 104 /// temporary reference to the `InstanceAndStore` with a constrained 105 /// lifetime to ensure that it doesn't accidentally escape. 106 /// 107 /// # Safety 108 /// 109 /// Callers must validate that the `vmctx` pointer is a valid allocation and 110 /// that it's valid to acquire `&mut InstanceAndStore` at this time. For 111 /// example this can't be called twice on the same `VMContext` to get two 112 /// active mutable borrows to the same `InstanceAndStore`. 113 /// 114 /// See also the safety discussion in this type's documentation. 115 #[inline] 116 pub(crate) unsafe fn from_vmctx<R>( 117 vmctx: NonNull<VMContext>, 118 f: impl for<'a> FnOnce(&'a mut Self) -> R, 119 ) -> R { 120 const _: () = assert!(mem::size_of::<InstanceAndStore>() == mem::size_of::<Instance>()); 121 let mut ptr = vmctx 122 .byte_sub(mem::size_of::<Instance>()) 123 .cast::<InstanceAndStore>(); 124 125 f(ptr.as_mut()) 126 } 127 128 /// Unpacks this `InstanceAndStore` into its underlying `Instance` and `dyn 129 /// VMStore`. 130 #[inline] 131 pub(crate) fn unpack_mut(&mut self) -> (&mut Instance, &mut dyn VMStore) { 132 unsafe { 133 let store = &mut *self.store_ptr(); 134 (&mut self.instance, store) 135 } 136 } 137 138 /// Unpacks this `InstanceAndStore` into its underlying `Instance` and 139 /// `StoreInner<T>`. 140 /// 141 /// # Safety 142 /// 143 /// The `T` must be the same `T` that was used to define this store's 144 /// instance. 145 #[inline] 146 pub(crate) unsafe fn unpack_context_mut<T>( 147 &mut self, 148 ) -> (&mut Instance, StoreContextMut<'_, T>) { 149 let store_ptr = self.store_ptr().cast::<StoreInner<T>>(); 150 (&mut self.instance, StoreContextMut(&mut *store_ptr)) 151 } 152 153 /// Gets a pointer to this instance's `Store` which was originally 154 /// configured on creation. 155 /// 156 /// # Panics 157 /// 158 /// May panic if the originally configured store was `None`. That can happen 159 /// for host functions so host functions can't be queried what their 160 /// original `Store` was since it's just retained as null (since host 161 /// functions are shared amongst threads and don't all share the same 162 /// store). 163 #[inline] 164 fn store_ptr(&self) -> *mut dyn VMStore { 165 self.instance.store.unwrap().0.as_ptr() 166 } 167 } 168 169 /// A type that roughly corresponds to a WebAssembly instance, but is also used 170 /// for host-defined objects. 171 /// 172 /// This structure is never allocated directly but is instead managed through 173 /// an `InstanceHandle`. This structure ends with a `VMContext` which has a 174 /// dynamic size corresponding to the `module` configured within. Memory 175 /// management of this structure is always externalized. 176 /// 177 /// Instances here can correspond to actual instantiated modules, but it's also 178 /// used ubiquitously for host-defined objects. For example creating a 179 /// host-defined memory will have a `module` that looks like it exports a single 180 /// memory (and similar for other constructs). 181 /// 182 /// This `Instance` type is used as a ubiquitous representation for WebAssembly 183 /// values, whether or not they were created on the host or through a module. 184 #[repr(C)] // ensure that the vmctx field is last. 185 pub struct Instance { 186 /// The index, within a `Store` that this instance lives at 187 id: InstanceId, 188 189 /// The runtime info (corresponding to the "compiled module" 190 /// abstraction in higher layers) that is retained and needed for 191 /// lazy initialization. This provides access to the underlying 192 /// Wasm module entities, the compiled JIT code, metadata about 193 /// functions, lazy initialization state, etc. 194 runtime_info: ModuleRuntimeInfo, 195 196 /// WebAssembly linear memory data. 197 /// 198 /// This is where all runtime information about defined linear memories in 199 /// this module lives. 200 /// 201 /// The `MemoryAllocationIndex` was given from our `InstanceAllocator` and 202 /// must be given back to the instance allocator when deallocating each 203 /// memory. 204 memories: PrimaryMap<DefinedMemoryIndex, (MemoryAllocationIndex, Memory)>, 205 206 /// WebAssembly table data. 207 /// 208 /// Like memories, this is only for defined tables in the module and 209 /// contains all of their runtime state. 210 /// 211 /// The `TableAllocationIndex` was given from our `InstanceAllocator` and 212 /// must be given back to the instance allocator when deallocating each 213 /// table. 214 tables: PrimaryMap<DefinedTableIndex, (TableAllocationIndex, Table)>, 215 216 /// Stores the dropped passive element segments in this instantiation by index. 217 /// If the index is present in the set, the segment has been dropped. 218 dropped_elements: EntitySet<ElemIndex>, 219 220 /// Stores the dropped passive data segments in this instantiation by index. 221 /// If the index is present in the set, the segment has been dropped. 222 dropped_data: EntitySet<DataIndex>, 223 224 /// A pointer to the `vmctx` field at the end of the `Instance`. 225 /// 226 /// If you're looking at this a reasonable question would be "why do we need 227 /// a pointer to ourselves?" because after all the pointer's value is 228 /// trivially derivable from any `&Instance` pointer. The rationale for this 229 /// field's existence is subtle, but it's required for correctness. The 230 /// short version is "this makes miri happy". 231 /// 232 /// The long version of why this field exists is that the rules that MIRI 233 /// uses to ensure pointers are used correctly have various conditions on 234 /// them depend on how pointers are used. More specifically if `*mut T` is 235 /// derived from `&mut T`, then that invalidates all prior pointers drived 236 /// from the `&mut T`. This means that while we liberally want to re-acquire 237 /// a `*mut VMContext` throughout the implementation of `Instance` the 238 /// trivial way, a function `fn vmctx(&mut Instance) -> *mut VMContext` 239 /// would effectively invalidate all prior `*mut VMContext` pointers 240 /// acquired. The purpose of this field is to serve as a sort of 241 /// source-of-truth for where `*mut VMContext` pointers come from. 242 /// 243 /// This field is initialized when the `Instance` is created with the 244 /// original allocation's pointer. That means that the provenance of this 245 /// pointer contains the entire allocation (both instance and `VMContext`). 246 /// This provenance bit is then "carried through" where `fn vmctx` will base 247 /// all returned pointers on this pointer itself. This provides the means of 248 /// never invalidating this pointer throughout MIRI and additionally being 249 /// able to still temporarily have `&mut Instance` methods and such. 250 /// 251 /// It's important to note, though, that this is not here purely for MIRI. 252 /// The careful construction of the `fn vmctx` method has ramifications on 253 /// the LLVM IR generated, for example. A historical CVE on Wasmtime, 254 /// GHSA-ch89-5g45-qwc7, was caused due to relying on undefined behavior. By 255 /// deriving VMContext pointers from this pointer it specifically hints to 256 /// LLVM that trickery is afoot and it properly informs `noalias` and such 257 /// annotations and analysis. More-or-less this pointer is actually loaded 258 /// in LLVM IR which helps defeat otherwise present aliasing optimizations, 259 /// which we want, since writes to this should basically never be optimized 260 /// out. 261 /// 262 /// As a final note it's worth pointing out that the machine code generated 263 /// for accessing `fn vmctx` is still as one would expect. This member isn't 264 /// actually ever loaded at runtime (or at least shouldn't be). Perhaps in 265 /// the future if the memory consumption of this field is a problem we could 266 /// shrink it slightly, but for now one extra pointer per wasm instance 267 /// seems not too bad. 268 vmctx_self_reference: SendSyncPtr<VMContext>, 269 270 // TODO: add support for multiple memories; `wmemcheck_state` corresponds to 271 // memory 0. 272 #[cfg(feature = "wmemcheck")] 273 pub(crate) wmemcheck_state: Option<Wmemcheck>, 274 275 /// Self-pointer back to `Store<T>` and its functions. Not present for 276 /// the brief time that `Store<T>` is itself being created. Also not 277 /// present for some niche uses that are disconnected from stores (e.g. 278 /// cross-thread stuff used in `InstancePre`) 279 store: Option<VMStoreRawPtr>, 280 281 /// Additional context used by compiled wasm code. This field is last, and 282 /// represents a dynamically-sized array that extends beyond the nominal 283 /// end of the struct (similar to a flexible array member). 284 vmctx: VMContext, 285 } 286 287 impl Instance { 288 /// Create an instance at the given memory address. 289 /// 290 /// It is assumed the memory was properly aligned and the 291 /// allocation was `alloc_size` in bytes. 292 unsafe fn new( 293 req: InstanceAllocationRequest, 294 memories: PrimaryMap<DefinedMemoryIndex, (MemoryAllocationIndex, Memory)>, 295 tables: PrimaryMap<DefinedTableIndex, (TableAllocationIndex, Table)>, 296 memory_tys: &PrimaryMap<MemoryIndex, wasmtime_environ::Memory>, 297 ) -> InstanceHandle { 298 // The allocation must be *at least* the size required of `Instance`. 299 let layout = Self::alloc_layout(req.runtime_info.offsets()); 300 let ptr = alloc::alloc::alloc(layout); 301 if ptr.is_null() { 302 alloc::alloc::handle_alloc_error(layout); 303 } 304 let ptr = ptr.cast::<Instance>(); 305 306 let module = req.runtime_info.env_module(); 307 let dropped_elements = EntitySet::with_capacity(module.passive_elements.len()); 308 let dropped_data = EntitySet::with_capacity(module.passive_data_map.len()); 309 310 #[cfg(not(feature = "wmemcheck"))] 311 let _ = memory_tys; 312 313 ptr::write( 314 ptr, 315 Instance { 316 id: req.id, 317 runtime_info: req.runtime_info.clone(), 318 memories, 319 tables, 320 dropped_elements, 321 dropped_data, 322 vmctx_self_reference: SendSyncPtr::new(NonNull::new(ptr.add(1).cast()).unwrap()), 323 vmctx: VMContext { 324 _marker: core::marker::PhantomPinned, 325 }, 326 #[cfg(feature = "wmemcheck")] 327 wmemcheck_state: { 328 if req.wmemcheck { 329 let size = memory_tys 330 .iter() 331 .next() 332 .map(|memory| memory.1.limits.min) 333 .unwrap_or(0) 334 * 64 335 * 1024; 336 Some(Wmemcheck::new(size as usize)) 337 } else { 338 None 339 } 340 }, 341 store: None, 342 }, 343 ); 344 345 (*ptr).initialize_vmctx(module, req.runtime_info.offsets(), req.store, req.imports); 346 InstanceHandle { 347 instance: Some(SendSyncPtr::new(NonNull::new(ptr).unwrap())), 348 } 349 } 350 351 /// Converts the provided `*mut VMContext` to an `Instance` pointer and runs 352 /// the provided closure with the instance. 353 /// 354 /// This method will move the `vmctx` pointer backwards to point to the 355 /// original `Instance` that precedes it. The closure is provided a 356 /// temporary version of the `Instance` pointer with a constrained lifetime 357 /// to the closure to ensure it doesn't accidentally escape. 358 /// 359 /// # Unsafety 360 /// 361 /// Callers must validate that the `vmctx` pointer is a valid allocation 362 /// and that it's valid to acquire `&mut Instance` at this time. For example 363 /// this can't be called twice on the same `VMContext` to get two active 364 /// pointers to the same `Instance`. 365 #[inline] 366 pub unsafe fn from_vmctx<R>( 367 vmctx: NonNull<VMContext>, 368 f: impl FnOnce(&mut Instance) -> R, 369 ) -> R { 370 let mut ptr = vmctx 371 .byte_sub(mem::size_of::<Instance>()) 372 .cast::<Instance>(); 373 f(ptr.as_mut()) 374 } 375 376 /// Helper function to access various locations offset from our `*mut 377 /// VMContext` object. 378 /// 379 /// Note that this method takes `&self` as an argument but returns 380 /// `NonNull<T>` which is frequently used to mutate said memory. This is an 381 /// intentional design decision where the safety of the modification of 382 /// memory is placed as a burden onto the caller. The implementation of this 383 /// method explicitly does not require `&mut self` to acquire mutable 384 /// provenance to update the `VMContext` region. Instead all pointers into 385 /// the `VMContext` area have provenance/permissions to write. 386 /// 387 /// Also note though that care must be taken to ensure that reads/writes of 388 /// memory must only happen where appropriate, for example a non-atomic 389 /// write (as most are) should never happen concurrently with another read 390 /// or write. It's generally on the burden of the caller to adhere to this. 391 /// 392 /// Also of note is that most of the time the usage of this method falls 393 /// into one of: 394 /// 395 /// * Something in the VMContext is being read or written. In that case use 396 /// `vmctx_plus_offset` or `vmctx_plus_offset_mut` if possible due to 397 /// that having a safer lifetime. 398 /// 399 /// * A pointer is being created to pass to other VM* data structures. In 400 /// that situation the lifetime of all VM data structures are typically 401 /// tied to the `Store<T>` which is what provides the guarantees around 402 /// concurrency/etc. 403 /// 404 /// There's quite a lot of unsafety riding on this method, especially 405 /// related to the ascription `T` of the byte `offset`. It's hoped that in 406 /// the future we're able to settle on an in theory safer design. 407 /// 408 /// # Safety 409 /// 410 /// This method is unsafe because the `offset` must be within bounds of the 411 /// `VMContext` object trailing this instance. Additionally `T` must be a 412 /// valid ascription of the value that resides at that location. 413 unsafe fn vmctx_plus_offset_raw<T: VmSafe>(&self, offset: impl Into<u32>) -> NonNull<T> { 414 // SAFETY: the safety requirements of `byte_add` are forwarded to this 415 // method's caller. 416 unsafe { 417 self.vmctx() 418 .byte_add(usize::try_from(offset.into()).unwrap()) 419 .cast() 420 } 421 } 422 423 /// Helper above `vmctx_plus_offset_raw` which transfers the lifetime of 424 /// `&self` to the returned reference `&T`. 425 /// 426 /// # Safety 427 /// 428 /// See the safety documentation of `vmctx_plus_offset_raw`. 429 unsafe fn vmctx_plus_offset<T: VmSafe>(&self, offset: impl Into<u32>) -> &T { 430 // SAFETY: this method has the same safety requirements as 431 // `vmctx_plus_offset_raw`. 432 unsafe { self.vmctx_plus_offset_raw(offset).as_ref() } 433 } 434 435 /// Helper above `vmctx_plus_offset_raw` which transfers the lifetime of 436 /// `&mut self` to the returned reference `&mut T`. 437 /// 438 /// # Safety 439 /// 440 /// See the safety documentation of `vmctx_plus_offset_raw`. 441 unsafe fn vmctx_plus_offset_mut<T: VmSafe>(&mut self, offset: impl Into<u32>) -> &mut T { 442 // SAFETY: this method has the same safety requirements as 443 // `vmctx_plus_offset_raw`. 444 unsafe { self.vmctx_plus_offset_raw(offset).as_mut() } 445 } 446 447 pub(crate) fn env_module(&self) -> &Arc<wasmtime_environ::Module> { 448 self.runtime_info.env_module() 449 } 450 451 #[cfg(feature = "gc")] 452 pub(crate) fn runtime_module(&self) -> Option<&crate::Module> { 453 match &self.runtime_info { 454 ModuleRuntimeInfo::Module(m) => Some(m), 455 ModuleRuntimeInfo::Bare(_) => None, 456 } 457 } 458 459 /// Translate a module-level interned type index into an engine-level 460 /// interned type index. 461 #[cfg(feature = "gc")] 462 pub fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex { 463 self.runtime_info.engine_type_index(module_index) 464 } 465 466 #[inline] 467 fn offsets(&self) -> &VMOffsets<HostPtr> { 468 self.runtime_info.offsets() 469 } 470 471 /// Return the indexed `VMFunctionImport`. 472 fn imported_function(&self, index: FuncIndex) -> &VMFunctionImport { 473 unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmfunction_import(index)) } 474 } 475 476 /// Return the index `VMTableImport`. 477 fn imported_table(&self, index: TableIndex) -> &VMTableImport { 478 unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtable_import(index)) } 479 } 480 481 /// Return the indexed `VMMemoryImport`. 482 fn imported_memory(&self, index: MemoryIndex) -> &VMMemoryImport { 483 unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmmemory_import(index)) } 484 } 485 486 /// Return the indexed `VMGlobalImport`. 487 fn imported_global(&self, index: GlobalIndex) -> &VMGlobalImport { 488 unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmglobal_import(index)) } 489 } 490 491 /// Return the indexed `VMTagImport`. 492 fn imported_tag(&self, index: TagIndex) -> &VMTagImport { 493 unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtag_import(index)) } 494 } 495 496 /// Return the indexed `VMTagDefinition`. 497 pub fn tag_ptr(&self, index: DefinedTagIndex) -> NonNull<VMTagDefinition> { 498 unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmtag_definition(index)) } 499 } 500 501 /// Return the indexed `VMTableDefinition`. 502 pub fn table(&self, index: DefinedTableIndex) -> VMTableDefinition { 503 unsafe { self.table_ptr(index).read() } 504 } 505 506 /// Updates the value for a defined table to `VMTableDefinition`. 507 fn set_table(&mut self, index: DefinedTableIndex, table: VMTableDefinition) { 508 unsafe { 509 self.table_ptr(index).write(table); 510 } 511 } 512 513 /// Return a pointer to the `index`'th table within this instance, stored 514 /// in vmctx memory. 515 pub fn table_ptr(&self, index: DefinedTableIndex) -> NonNull<VMTableDefinition> { 516 unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmtable_definition(index)) } 517 } 518 519 /// Get a locally defined or imported memory. 520 pub(crate) fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition { 521 if let Some(defined_index) = self.env_module().defined_memory_index(index) { 522 self.memory(defined_index) 523 } else { 524 let import = self.imported_memory(index); 525 unsafe { VMMemoryDefinition::load(import.from.as_ptr()) } 526 } 527 } 528 529 /// Get a locally defined or imported memory. 530 #[cfg(feature = "threads")] 531 pub(crate) fn get_runtime_memory(&mut self, index: MemoryIndex) -> &mut Memory { 532 if let Some(defined_index) = self.env_module().defined_memory_index(index) { 533 unsafe { &mut *self.get_defined_memory(defined_index) } 534 } else { 535 let import = self.imported_memory(index); 536 unsafe { 537 let ptr = Instance::from_vmctx(import.vmctx.as_non_null(), |i| { 538 i.get_defined_memory(import.index) 539 }); 540 &mut *ptr 541 } 542 } 543 } 544 545 /// Return the indexed `VMMemoryDefinition`, loaded from vmctx memory 546 /// already. 547 pub fn memory(&self, index: DefinedMemoryIndex) -> VMMemoryDefinition { 548 unsafe { VMMemoryDefinition::load(self.memory_ptr(index).as_ptr()) } 549 } 550 551 /// Set the indexed memory to `VMMemoryDefinition`. 552 fn set_memory(&self, index: DefinedMemoryIndex, mem: VMMemoryDefinition) { 553 unsafe { 554 self.memory_ptr(index).write(mem); 555 } 556 } 557 558 /// Return the address of the specified memory at `index` within this vmctx. 559 /// 560 /// Note that the returned pointer resides in wasm-code-readable-memory in 561 /// the vmctx. 562 pub fn memory_ptr(&self, index: DefinedMemoryIndex) -> NonNull<VMMemoryDefinition> { 563 unsafe { 564 self.vmctx_plus_offset::<VmPtr<_>>(self.offsets().vmctx_vmmemory_pointer(index)) 565 .as_non_null() 566 } 567 } 568 569 /// Return the indexed `VMGlobalDefinition`. 570 pub fn global_ptr(&self, index: DefinedGlobalIndex) -> NonNull<VMGlobalDefinition> { 571 unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmglobal_definition(index)) } 572 } 573 574 /// Get a raw pointer to the global at the given index regardless whether it 575 /// is defined locally or imported from another module. 576 /// 577 /// Panics if the index is out of bound or is the reserved value. 578 pub(crate) fn defined_or_imported_global_ptr( 579 &mut self, 580 index: GlobalIndex, 581 ) -> NonNull<VMGlobalDefinition> { 582 if let Some(index) = self.env_module().defined_global_index(index) { 583 self.global_ptr(index) 584 } else { 585 self.imported_global(index).from.as_non_null() 586 } 587 } 588 589 /// Get all globals within this instance. 590 /// 591 /// Returns both import and defined globals. 592 /// 593 /// Returns both exported and non-exported globals. 594 /// 595 /// Gives access to the full globals space. 596 pub fn all_globals<'a>( 597 &'a mut self, 598 ) -> impl ExactSizeIterator<Item = (GlobalIndex, ExportGlobal)> + 'a { 599 let module = self.env_module().clone(); 600 module 601 .globals 602 .keys() 603 .map(move |idx| (idx, self.get_exported_global(idx))) 604 } 605 606 /// Get the globals defined in this instance (not imported). 607 pub fn defined_globals<'a>( 608 &'a mut self, 609 ) -> impl ExactSizeIterator<Item = (DefinedGlobalIndex, ExportGlobal)> + 'a { 610 let module = self.env_module().clone(); 611 module 612 .globals 613 .keys() 614 .skip(module.num_imported_globals) 615 .map(move |global_idx| { 616 let def_idx = module.defined_global_index(global_idx).unwrap(); 617 let global = ExportGlobal { 618 definition: self.global_ptr(def_idx), 619 kind: ExportGlobalKind::Instance(self.vmctx(), def_idx), 620 global: self.env_module().globals[global_idx], 621 }; 622 (def_idx, global) 623 }) 624 } 625 626 /// Return a pointer to the interrupts structure 627 #[inline] 628 pub fn vm_store_context(&mut self) -> NonNull<Option<VmPtr<VMStoreContext>>> { 629 unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx_store_context()) } 630 } 631 632 /// Return a pointer to the global epoch counter used by this instance. 633 #[cfg(target_has_atomic = "64")] 634 pub fn epoch_ptr(&mut self) -> &mut Option<VmPtr<AtomicU64>> { 635 unsafe { self.vmctx_plus_offset_mut(self.offsets().ptr.vmctx_epoch_ptr()) } 636 } 637 638 /// Return a pointer to the collector-specific heap data. 639 pub fn gc_heap_data(&mut self) -> &mut Option<VmPtr<u8>> { 640 unsafe { self.vmctx_plus_offset_mut(self.offsets().ptr.vmctx_gc_heap_data()) } 641 } 642 643 pub(crate) unsafe fn set_store(&mut self, store: Option<NonNull<dyn VMStore>>) { 644 self.store = store.map(VMStoreRawPtr); 645 if let Some(mut store) = store { 646 let store = store.as_mut(); 647 self.vm_store_context() 648 .write(Some(store.vm_store_context_ptr().into())); 649 #[cfg(target_has_atomic = "64")] 650 { 651 *self.epoch_ptr() = Some(NonNull::from(store.engine().epoch_counter()).into()); 652 } 653 654 if self.env_module().needs_gc_heap { 655 self.set_gc_heap(Some(store.gc_store().expect( 656 "if we need a GC heap, then `Instance::new_raw` should have already \ 657 allocated it for us", 658 ))); 659 } else { 660 self.set_gc_heap(None); 661 } 662 } else { 663 self.vm_store_context().write(None); 664 #[cfg(target_has_atomic = "64")] 665 { 666 *self.epoch_ptr() = None; 667 } 668 self.set_gc_heap(None); 669 } 670 } 671 672 unsafe fn set_gc_heap(&mut self, gc_store: Option<&GcStore>) { 673 if let Some(gc_store) = gc_store { 674 *self.gc_heap_data() = Some(gc_store.gc_heap.vmctx_gc_heap_data().into()); 675 } else { 676 *self.gc_heap_data() = None; 677 } 678 } 679 680 pub(crate) unsafe fn set_callee(&mut self, callee: Option<NonNull<VMFunctionBody>>) { 681 let callee = callee.map(|p| VmPtr::from(p)); 682 *self.vmctx_plus_offset_mut(self.offsets().ptr.vmctx_callee()) = callee; 683 } 684 685 /// Return a reference to the vmctx used by compiled wasm code. 686 #[inline] 687 pub fn vmctx(&self) -> NonNull<VMContext> { 688 // The definition of this method is subtle but intentional. The goal 689 // here is that effectively this should return `&mut self.vmctx`, but 690 // it's not quite so simple. Some more documentation is available on the 691 // `vmctx_self_reference` field, but the general idea is that we're 692 // creating a pointer to return with proper provenance. Provenance is 693 // still in the works in Rust at the time of this writing but the load 694 // of the `self.vmctx_self_reference` field is important here as it 695 // affects how LLVM thinks about aliasing with respect to the returned 696 // pointer. 697 // 698 // The intention of this method is to codegen to machine code as `&mut 699 // self.vmctx`, however. While it doesn't show up like this in LLVM IR 700 // (there's an actual load of the field) it does look like that by the 701 // time the backend runs. (that's magic to me, the backend removing 702 // loads...) 703 let addr = &raw const self.vmctx; 704 let ret = self.vmctx_self_reference.as_ptr().with_addr(addr.addr()); 705 NonNull::new(ret).unwrap() 706 } 707 708 /// Lookup a function by index. 709 /// 710 /// # Panics 711 /// 712 /// Panics if `index` is out of bounds for this instance. 713 pub fn get_exported_func(&self, index: FuncIndex) -> ExportFunction { 714 let func_ref = self.get_func_ref(index).unwrap(); 715 ExportFunction { func_ref } 716 } 717 718 fn get_exported_table(&self, index: TableIndex) -> ExportTable { 719 let ty = self.env_module().tables[index]; 720 let (definition, vmctx, index) = 721 if let Some(def_index) = self.env_module().defined_table_index(index) { 722 (self.table_ptr(def_index), self.vmctx(), def_index) 723 } else { 724 let import = self.imported_table(index); 725 ( 726 import.from.as_non_null(), 727 import.vmctx.as_non_null(), 728 import.index, 729 ) 730 }; 731 ExportTable { 732 definition, 733 vmctx, 734 table: ty, 735 index, 736 } 737 } 738 739 fn get_exported_memory(&self, index: MemoryIndex) -> ExportMemory { 740 let (definition, vmctx, def_index) = 741 if let Some(def_index) = self.env_module().defined_memory_index(index) { 742 (self.memory_ptr(def_index), self.vmctx(), def_index) 743 } else { 744 let import = self.imported_memory(index); 745 ( 746 import.from.as_non_null(), 747 import.vmctx.as_non_null(), 748 import.index, 749 ) 750 }; 751 ExportMemory { 752 definition, 753 vmctx, 754 memory: self.env_module().memories[index], 755 index: def_index, 756 } 757 } 758 759 fn get_exported_global(&self, index: GlobalIndex) -> ExportGlobal { 760 let global = self.env_module().globals[index]; 761 if let Some(def_index) = self.env_module().defined_global_index(index) { 762 ExportGlobal { 763 definition: self.global_ptr(def_index), 764 kind: ExportGlobalKind::Instance(self.vmctx(), def_index), 765 global, 766 } 767 } else { 768 ExportGlobal::from_vmimport(self.imported_global(index), global) 769 } 770 } 771 772 fn get_exported_tag(&self, index: TagIndex) -> ExportTag { 773 let tag = self.env_module().tags[index]; 774 let (vmctx, definition, index) = 775 if let Some(def_index) = self.env_module().defined_tag_index(index) { 776 (self.vmctx(), self.tag_ptr(def_index), def_index) 777 } else { 778 let import = self.imported_tag(index); 779 ( 780 import.vmctx.as_non_null(), 781 import.from.as_non_null(), 782 import.index, 783 ) 784 }; 785 ExportTag { 786 definition, 787 vmctx, 788 index, 789 tag, 790 } 791 } 792 793 /// Return an iterator over the exports of this instance. 794 /// 795 /// Specifically, it provides access to the key-value pairs, where the keys 796 /// are export names, and the values are export declarations which can be 797 /// resolved `lookup_by_declaration`. 798 pub fn exports(&self) -> wasmparser::collections::index_map::Iter<String, EntityIndex> { 799 self.env_module().exports.iter() 800 } 801 802 /// Return the table index for the given `VMTableDefinition`. 803 pub unsafe fn table_index(&mut self, table: &VMTableDefinition) -> DefinedTableIndex { 804 let index = DefinedTableIndex::new( 805 usize::try_from( 806 (table as *const VMTableDefinition) 807 .offset_from(self.table_ptr(DefinedTableIndex::new(0)).as_ptr()), 808 ) 809 .unwrap(), 810 ); 811 assert!(index.index() < self.tables.len()); 812 index 813 } 814 815 /// Get the given memory's page size, in bytes. 816 pub(crate) fn memory_page_size(&self, index: MemoryIndex) -> usize { 817 usize::try_from(self.env_module().memories[index].page_size()).unwrap() 818 } 819 820 /// Grow memory by the specified amount of pages. 821 /// 822 /// Returns `None` if memory can't be grown by the specified amount 823 /// of pages. Returns `Some` with the old size in bytes if growth was 824 /// successful. 825 pub(crate) fn memory_grow( 826 &mut self, 827 store: &mut dyn VMStore, 828 index: MemoryIndex, 829 delta: u64, 830 ) -> Result<Option<usize>, Error> { 831 match self.env_module().defined_memory_index(index) { 832 Some(idx) => self.defined_memory_grow(store, idx, delta), 833 None => { 834 let import = self.imported_memory(index); 835 unsafe { 836 Instance::from_vmctx(import.vmctx.as_non_null(), |i| { 837 i.defined_memory_grow(store, import.index, delta) 838 }) 839 } 840 } 841 } 842 } 843 844 fn defined_memory_grow( 845 &mut self, 846 store: &mut dyn VMStore, 847 idx: DefinedMemoryIndex, 848 delta: u64, 849 ) -> Result<Option<usize>, Error> { 850 let memory = &mut self.memories[idx].1; 851 852 let result = unsafe { memory.grow(delta, Some(store)) }; 853 854 // Update the state used by a non-shared Wasm memory in case the base 855 // pointer and/or the length changed. 856 if memory.as_shared_memory().is_none() { 857 let vmmemory = memory.vmmemory(); 858 self.set_memory(idx, vmmemory); 859 } 860 861 result 862 } 863 864 pub(crate) fn table_element_type(&mut self, table_index: TableIndex) -> TableElementType { 865 unsafe { (*self.get_table(table_index)).element_type() } 866 } 867 868 /// Grow table by the specified amount of elements, filling them with 869 /// `init_value`. 870 /// 871 /// Returns `None` if table can't be grown by the specified amount of 872 /// elements, or if `init_value` is the wrong type of table element. 873 pub(crate) fn table_grow( 874 &mut self, 875 store: &mut dyn VMStore, 876 table_index: TableIndex, 877 delta: u64, 878 init_value: TableElement, 879 ) -> Result<Option<usize>, Error> { 880 self.with_defined_table_index_and_instance(table_index, |i, instance| { 881 instance.defined_table_grow(store, i, delta, init_value) 882 }) 883 } 884 885 fn defined_table_grow( 886 &mut self, 887 store: &mut dyn VMStore, 888 table_index: DefinedTableIndex, 889 delta: u64, 890 init_value: TableElement, 891 ) -> Result<Option<usize>, Error> { 892 let table = &mut self 893 .tables 894 .get_mut(table_index) 895 .unwrap_or_else(|| panic!("no table for index {}", table_index.index())) 896 .1; 897 898 let result = unsafe { table.grow(delta, init_value, store) }; 899 900 // Keep the `VMContext` pointers used by compiled Wasm code up to 901 // date. 902 let element = self.tables[table_index].1.vmtable(); 903 self.set_table(table_index, element); 904 905 result 906 } 907 908 fn alloc_layout(offsets: &VMOffsets<HostPtr>) -> Layout { 909 let size = mem::size_of::<Self>() 910 .checked_add(usize::try_from(offsets.size_of_vmctx()).unwrap()) 911 .unwrap(); 912 let align = mem::align_of::<Self>(); 913 Layout::from_size_align(size, align).unwrap() 914 } 915 916 fn type_ids_array(&mut self) -> NonNull<VmPtr<VMSharedTypeIndex>> { 917 unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx_type_ids_array()) } 918 } 919 920 /// Construct a new VMFuncRef for the given function 921 /// (imported or defined in this module) and store into the given 922 /// location. Used during lazy initialization. 923 /// 924 /// Note that our current lazy-init scheme actually calls this every 925 /// time the funcref pointer is fetched; this turns out to be better 926 /// than tracking state related to whether it's been initialized 927 /// before, because resetting that state on (re)instantiation is 928 /// very expensive if there are many funcrefs. 929 fn construct_func_ref( 930 &self, 931 index: FuncIndex, 932 type_index: VMSharedTypeIndex, 933 into: *mut VMFuncRef, 934 ) { 935 let func_ref = if let Some(def_index) = self.env_module().defined_func_index(index) { 936 VMFuncRef { 937 array_call: self 938 .runtime_info 939 .array_to_wasm_trampoline(def_index) 940 .expect("should have array-to-Wasm trampoline for escaping function") 941 .into(), 942 wasm_call: Some(self.runtime_info.function(def_index).into()), 943 vmctx: VMOpaqueContext::from_vmcontext(self.vmctx()).into(), 944 type_index, 945 } 946 } else { 947 let import = self.imported_function(index); 948 VMFuncRef { 949 array_call: import.array_call, 950 wasm_call: Some(import.wasm_call), 951 vmctx: import.vmctx, 952 type_index, 953 } 954 }; 955 956 // Safety: we have a `&mut self`, so we have exclusive access 957 // to this Instance. 958 unsafe { 959 ptr::write(into, func_ref); 960 } 961 } 962 963 /// Get a `&VMFuncRef` for the given `FuncIndex`. 964 /// 965 /// Returns `None` if the index is the reserved index value. 966 /// 967 /// The returned reference is a stable reference that won't be moved and can 968 /// be passed into JIT code. 969 pub(crate) fn get_func_ref(&self, index: FuncIndex) -> Option<NonNull<VMFuncRef>> { 970 if index == FuncIndex::reserved_value() { 971 return None; 972 } 973 974 // Safety: we have a `&mut self`, so we have exclusive access 975 // to this Instance. 976 unsafe { 977 // For now, we eagerly initialize an funcref struct in-place 978 // whenever asked for a reference to it. This is mostly 979 // fine, because in practice each funcref is unlikely to be 980 // requested more than a few times: once-ish for funcref 981 // tables used for call_indirect (the usual compilation 982 // strategy places each function in the table at most once), 983 // and once or a few times when fetching exports via API. 984 // Note that for any case driven by table accesses, the lazy 985 // table init behaves like a higher-level cache layer that 986 // protects this initialization from happening multiple 987 // times, via that particular table at least. 988 // 989 // When `ref.func` becomes more commonly used or if we 990 // otherwise see a use-case where this becomes a hotpath, 991 // we can reconsider by using some state to track 992 // "uninitialized" explicitly, for example by zeroing the 993 // funcrefs (perhaps together with other 994 // zeroed-at-instantiate-time state) or using a separate 995 // is-initialized bitmap. 996 // 997 // We arrived at this design because zeroing memory is 998 // expensive, so it's better for instantiation performance 999 // if we don't have to track "is-initialized" state at 1000 // all! 1001 let func = &self.env_module().functions[index]; 1002 let sig = func.signature.unwrap_engine_type_index(); 1003 let func_ref = self 1004 .vmctx_plus_offset_raw::<VMFuncRef>(self.offsets().vmctx_func_ref(func.func_ref)); 1005 self.construct_func_ref(index, sig, func_ref.as_ptr()); 1006 1007 Some(func_ref) 1008 } 1009 } 1010 1011 /// Get the passive elements segment at the given index. 1012 /// 1013 /// Returns an empty segment if the index is out of bounds or if the segment 1014 /// has been dropped. 1015 /// 1016 /// The `storage` parameter should always be `None`; it is a bit of a hack 1017 /// to work around lifetime issues. 1018 pub(crate) fn passive_element_segment<'a>( 1019 &self, 1020 storage: &'a mut Option<(Arc<wasmtime_environ::Module>, TableSegmentElements)>, 1021 elem_index: ElemIndex, 1022 ) -> &'a TableSegmentElements { 1023 debug_assert!(storage.is_none()); 1024 *storage = Some(( 1025 // TODO: this `clone()` shouldn't be necessary but is used for now to 1026 // inform `rustc` that the lifetime of the elements here are 1027 // disconnected from the lifetime of `self`. 1028 self.env_module().clone(), 1029 // NB: fall back to an expressions-based list of elements which 1030 // doesn't have static type information (as opposed to 1031 // `TableSegmentElements::Functions`) since we don't know what type 1032 // is needed in the caller's context. Let the type be inferred by 1033 // how they use the segment. 1034 TableSegmentElements::Expressions(Box::new([])), 1035 )); 1036 let (module, empty) = storage.as_ref().unwrap(); 1037 1038 match module.passive_elements_map.get(&elem_index) { 1039 Some(index) if !self.dropped_elements.contains(elem_index) => { 1040 &module.passive_elements[*index] 1041 } 1042 _ => empty, 1043 } 1044 } 1045 1046 /// The `table.init` operation: initializes a portion of a table with a 1047 /// passive element. 1048 /// 1049 /// # Errors 1050 /// 1051 /// Returns a `Trap` error when the range within the table is out of bounds 1052 /// or the range within the passive element is out of bounds. 1053 pub(crate) fn table_init( 1054 &mut self, 1055 store: &mut StoreOpaque, 1056 table_index: TableIndex, 1057 elem_index: ElemIndex, 1058 dst: u64, 1059 src: u64, 1060 len: u64, 1061 ) -> Result<(), Trap> { 1062 let mut storage = None; 1063 let elements = self.passive_element_segment(&mut storage, elem_index); 1064 let mut const_evaluator = ConstExprEvaluator::default(); 1065 self.table_init_segment( 1066 store, 1067 &mut const_evaluator, 1068 table_index, 1069 elements, 1070 dst, 1071 src, 1072 len, 1073 ) 1074 } 1075 1076 pub(crate) fn table_init_segment( 1077 &mut self, 1078 store: &mut StoreOpaque, 1079 const_evaluator: &mut ConstExprEvaluator, 1080 table_index: TableIndex, 1081 elements: &TableSegmentElements, 1082 dst: u64, 1083 src: u64, 1084 len: u64, 1085 ) -> Result<(), Trap> { 1086 // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init 1087 1088 let table = unsafe { &mut *self.get_table(table_index) }; 1089 let src = usize::try_from(src).map_err(|_| Trap::TableOutOfBounds)?; 1090 let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?; 1091 let module = self.env_module().clone(); 1092 1093 match elements { 1094 TableSegmentElements::Functions(funcs) => { 1095 let elements = funcs 1096 .get(src..) 1097 .and_then(|s| s.get(..len)) 1098 .ok_or(Trap::TableOutOfBounds)?; 1099 table.init_func(dst, elements.iter().map(|idx| self.get_func_ref(*idx)))?; 1100 } 1101 TableSegmentElements::Expressions(exprs) => { 1102 let exprs = exprs 1103 .get(src..) 1104 .and_then(|s| s.get(..len)) 1105 .ok_or(Trap::TableOutOfBounds)?; 1106 let mut context = ConstEvalContext::new(self); 1107 match module.tables[table_index].ref_type.heap_type.top() { 1108 WasmHeapTopType::Extern => table.init_gc_refs( 1109 dst, 1110 exprs.iter().map(|expr| unsafe { 1111 let raw = const_evaluator 1112 .eval(store, &mut context, expr) 1113 .expect("const expr should be valid"); 1114 VMGcRef::from_raw_u32(raw.get_externref()) 1115 }), 1116 )?, 1117 WasmHeapTopType::Any => table.init_gc_refs( 1118 dst, 1119 exprs.iter().map(|expr| unsafe { 1120 let raw = const_evaluator 1121 .eval(store, &mut context, expr) 1122 .expect("const expr should be valid"); 1123 VMGcRef::from_raw_u32(raw.get_anyref()) 1124 }), 1125 )?, 1126 WasmHeapTopType::Func => table.init_func( 1127 dst, 1128 exprs.iter().map(|expr| unsafe { 1129 NonNull::new( 1130 const_evaluator 1131 .eval(store, &mut context, expr) 1132 .expect("const expr should be valid") 1133 .get_funcref() 1134 .cast(), 1135 ) 1136 }), 1137 )?, 1138 WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. 1139 } 1140 } 1141 } 1142 1143 Ok(()) 1144 } 1145 1146 /// Drop an element. 1147 pub(crate) fn elem_drop(&mut self, elem_index: ElemIndex) { 1148 // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop 1149 1150 self.dropped_elements.insert(elem_index); 1151 1152 // Note that we don't check that we actually removed a segment because 1153 // dropping a non-passive segment is a no-op (not a trap). 1154 } 1155 1156 /// Get a locally-defined memory. 1157 pub fn get_defined_memory(&mut self, index: DefinedMemoryIndex) -> *mut Memory { 1158 // SAFETY: the `unsafe` here is projecting from `*mut (A, B)` to 1159 // `*mut A`, which should be a safe operation to do. 1160 unsafe { &raw mut (*self.memories.get_raw_mut(index).unwrap()).1 } 1161 } 1162 1163 /// Do a `memory.copy` 1164 /// 1165 /// # Errors 1166 /// 1167 /// Returns a `Trap` error when the source or destination ranges are out of 1168 /// bounds. 1169 pub(crate) fn memory_copy( 1170 &mut self, 1171 dst_index: MemoryIndex, 1172 dst: u64, 1173 src_index: MemoryIndex, 1174 src: u64, 1175 len: u64, 1176 ) -> Result<(), Trap> { 1177 // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy 1178 1179 let src_mem = self.get_memory(src_index); 1180 let dst_mem = self.get_memory(dst_index); 1181 1182 let src = self.validate_inbounds(src_mem.current_length(), src, len)?; 1183 let dst = self.validate_inbounds(dst_mem.current_length(), dst, len)?; 1184 let len = usize::try_from(len).unwrap(); 1185 1186 // Bounds and casts are checked above, by this point we know that 1187 // everything is safe. 1188 unsafe { 1189 let dst = dst_mem.base.as_ptr().add(dst); 1190 let src = src_mem.base.as_ptr().add(src); 1191 // FIXME audit whether this is safe in the presence of shared memory 1192 // (https://github.com/bytecodealliance/wasmtime/issues/4203). 1193 ptr::copy(src, dst, len); 1194 } 1195 1196 Ok(()) 1197 } 1198 1199 fn validate_inbounds(&self, max: usize, ptr: u64, len: u64) -> Result<usize, Trap> { 1200 let oob = || Trap::MemoryOutOfBounds; 1201 let end = ptr 1202 .checked_add(len) 1203 .and_then(|i| usize::try_from(i).ok()) 1204 .ok_or_else(oob)?; 1205 if end > max { 1206 Err(oob()) 1207 } else { 1208 Ok(ptr.try_into().unwrap()) 1209 } 1210 } 1211 1212 /// Perform the `memory.fill` operation on a locally defined memory. 1213 /// 1214 /// # Errors 1215 /// 1216 /// Returns a `Trap` error if the memory range is out of bounds. 1217 pub(crate) fn memory_fill( 1218 &mut self, 1219 memory_index: MemoryIndex, 1220 dst: u64, 1221 val: u8, 1222 len: u64, 1223 ) -> Result<(), Trap> { 1224 let memory = self.get_memory(memory_index); 1225 let dst = self.validate_inbounds(memory.current_length(), dst, len)?; 1226 let len = usize::try_from(len).unwrap(); 1227 1228 // Bounds and casts are checked above, by this point we know that 1229 // everything is safe. 1230 unsafe { 1231 let dst = memory.base.as_ptr().add(dst); 1232 // FIXME audit whether this is safe in the presence of shared memory 1233 // (https://github.com/bytecodealliance/wasmtime/issues/4203). 1234 ptr::write_bytes(dst, val, len); 1235 } 1236 1237 Ok(()) 1238 } 1239 1240 /// Get the internal storage range of a particular Wasm data segment. 1241 pub(crate) fn wasm_data_range(&self, index: DataIndex) -> Range<u32> { 1242 match self.env_module().passive_data_map.get(&index) { 1243 Some(range) if !self.dropped_data.contains(index) => range.clone(), 1244 _ => 0..0, 1245 } 1246 } 1247 1248 /// Given an internal storage range of a Wasm data segment (or subset of a 1249 /// Wasm data segment), get the data's raw bytes. 1250 pub(crate) fn wasm_data(&self, range: Range<u32>) -> &[u8] { 1251 let start = usize::try_from(range.start).unwrap(); 1252 let end = usize::try_from(range.end).unwrap(); 1253 &self.runtime_info.wasm_data()[start..end] 1254 } 1255 1256 /// Performs the `memory.init` operation. 1257 /// 1258 /// # Errors 1259 /// 1260 /// Returns a `Trap` error if the destination range is out of this module's 1261 /// memory's bounds or if the source range is outside the data segment's 1262 /// bounds. 1263 pub(crate) fn memory_init( 1264 &mut self, 1265 memory_index: MemoryIndex, 1266 data_index: DataIndex, 1267 dst: u64, 1268 src: u32, 1269 len: u32, 1270 ) -> Result<(), Trap> { 1271 let range = self.wasm_data_range(data_index); 1272 self.memory_init_segment(memory_index, range, dst, src, len) 1273 } 1274 1275 pub(crate) fn memory_init_segment( 1276 &mut self, 1277 memory_index: MemoryIndex, 1278 range: Range<u32>, 1279 dst: u64, 1280 src: u32, 1281 len: u32, 1282 ) -> Result<(), Trap> { 1283 // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init 1284 1285 let memory = self.get_memory(memory_index); 1286 let data = self.wasm_data(range); 1287 let dst = self.validate_inbounds(memory.current_length(), dst, len.into())?; 1288 let src = self.validate_inbounds(data.len(), src.into(), len.into())?; 1289 let len = len as usize; 1290 1291 unsafe { 1292 let src_start = data.as_ptr().add(src); 1293 let dst_start = memory.base.as_ptr().add(dst); 1294 // FIXME audit whether this is safe in the presence of shared memory 1295 // (https://github.com/bytecodealliance/wasmtime/issues/4203). 1296 ptr::copy_nonoverlapping(src_start, dst_start, len); 1297 } 1298 1299 Ok(()) 1300 } 1301 1302 /// Drop the given data segment, truncating its length to zero. 1303 pub(crate) fn data_drop(&mut self, data_index: DataIndex) { 1304 self.dropped_data.insert(data_index); 1305 1306 // Note that we don't check that we actually removed a segment because 1307 // dropping a non-passive segment is a no-op (not a trap). 1308 } 1309 1310 /// Get a table by index regardless of whether it is locally-defined 1311 /// or an imported, foreign table. Ensure that the given range of 1312 /// elements in the table is lazily initialized. We define this 1313 /// operation all-in-one for safety, to ensure the lazy-init 1314 /// happens. 1315 /// 1316 /// Takes an `Iterator` for the index-range to lazy-initialize, 1317 /// for flexibility. This can be a range, single item, or empty 1318 /// sequence, for example. The iterator should return indices in 1319 /// increasing order, so that the break-at-out-of-bounds behavior 1320 /// works correctly. 1321 pub(crate) fn get_table_with_lazy_init( 1322 &mut self, 1323 table_index: TableIndex, 1324 range: impl Iterator<Item = u64>, 1325 ) -> *mut Table { 1326 self.with_defined_table_index_and_instance(table_index, |idx, instance| { 1327 instance.get_defined_table_with_lazy_init(idx, range) 1328 }) 1329 } 1330 1331 /// Gets the raw runtime table data structure owned by this instance 1332 /// given the provided `idx`. 1333 /// 1334 /// The `range` specified is eagerly initialized for funcref tables. 1335 pub fn get_defined_table_with_lazy_init( 1336 &mut self, 1337 idx: DefinedTableIndex, 1338 range: impl Iterator<Item = u64>, 1339 ) -> *mut Table { 1340 let elt_ty = self.tables[idx].1.element_type(); 1341 1342 if elt_ty == TableElementType::Func { 1343 for i in range { 1344 let value = match self.tables[idx].1.get(None, i) { 1345 Some(value) => value, 1346 None => { 1347 // Out-of-bounds; caller will handle by likely 1348 // throwing a trap. No work to do to lazy-init 1349 // beyond the end. 1350 break; 1351 } 1352 }; 1353 1354 if !value.is_uninit() { 1355 continue; 1356 } 1357 1358 // The table element `i` is uninitialized and is now being 1359 // initialized. This must imply that a `precompiled` list of 1360 // function indices is available for this table. The precompiled 1361 // list is extracted and then it is consulted with `i` to 1362 // determine the function that is going to be initialized. Note 1363 // that `i` may be outside the limits of the static 1364 // initialization so it's a fallible `get` instead of an index. 1365 let module = self.env_module(); 1366 let precomputed = match &module.table_initialization.initial_values[idx] { 1367 TableInitialValue::Null { precomputed } => precomputed, 1368 TableInitialValue::Expr(_) => unreachable!(), 1369 }; 1370 // Panicking here helps catch bugs rather than silently truncating by accident. 1371 let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned(); 1372 let func_ref = func_index.and_then(|func_index| self.get_func_ref(func_index)); 1373 self.tables[idx] 1374 .1 1375 .set(i, TableElement::FuncRef(func_ref)) 1376 .expect("Table type should match and index should be in-bounds"); 1377 } 1378 } 1379 1380 // SAFETY: the `unsafe` here is projecting from `*mut (A, B)` to 1381 // `*mut A`, which should be a safe operation to do. 1382 unsafe { &raw mut (*self.tables.get_raw_mut(idx).unwrap()).1 } 1383 } 1384 1385 /// Get a table by index regardless of whether it is locally-defined or an 1386 /// imported, foreign table. 1387 pub(crate) fn get_table(&mut self, table_index: TableIndex) -> *mut Table { 1388 self.with_defined_table_index_and_instance(table_index, |idx, instance| unsafe { 1389 // SAFETY: the `unsafe` here is projecting from `*mut (A, B)` to 1390 // `*mut A`, which should be a safe operation to do. 1391 &raw mut (*instance.tables.get_raw_mut(idx).unwrap()).1 1392 }) 1393 } 1394 1395 /// Get a locally-defined table. 1396 pub(crate) fn get_defined_table(&mut self, index: DefinedTableIndex) -> *mut Table { 1397 // SAFETY: the `unsafe` here is projecting from `*mut (A, B)` to 1398 // `*mut A`, which should be a safe operation to do. 1399 unsafe { &raw mut (*self.tables.get_raw_mut(index).unwrap()).1 } 1400 } 1401 1402 pub(crate) fn with_defined_table_index_and_instance<R>( 1403 &mut self, 1404 index: TableIndex, 1405 f: impl FnOnce(DefinedTableIndex, &mut Instance) -> R, 1406 ) -> R { 1407 if let Some(defined_table_index) = self.env_module().defined_table_index(index) { 1408 f(defined_table_index, self) 1409 } else { 1410 let import = self.imported_table(index); 1411 unsafe { 1412 Instance::from_vmctx(import.vmctx.as_non_null(), |foreign_instance| { 1413 let foreign_table_def = import.from.as_ptr(); 1414 let foreign_table_index = foreign_instance.table_index(&*foreign_table_def); 1415 f(foreign_table_index, foreign_instance) 1416 }) 1417 } 1418 } 1419 } 1420 1421 /// Initialize the VMContext data associated with this Instance. 1422 /// 1423 /// The `VMContext` memory is assumed to be uninitialized; any field 1424 /// that we need in a certain state will be explicitly written by this 1425 /// function. 1426 unsafe fn initialize_vmctx( 1427 &mut self, 1428 module: &Module, 1429 offsets: &VMOffsets<HostPtr>, 1430 store: StorePtr, 1431 imports: Imports, 1432 ) { 1433 assert!(ptr::eq(module, self.env_module().as_ref())); 1434 1435 self.vmctx_plus_offset_raw::<u32>(offsets.ptr.vmctx_magic()) 1436 .write(VMCONTEXT_MAGIC); 1437 self.set_callee(None); 1438 self.set_store(store.as_raw()); 1439 1440 // Initialize shared types 1441 let types = NonNull::from(self.runtime_info.type_ids()); 1442 self.type_ids_array().write(types.cast().into()); 1443 1444 // Initialize the built-in functions 1445 static BUILTINS: VMBuiltinFunctionsArray = VMBuiltinFunctionsArray::INIT; 1446 let ptr = BUILTINS.expose_provenance(); 1447 self.vmctx_plus_offset_raw(offsets.ptr.vmctx_builtin_functions()) 1448 .write(VmPtr::from(ptr)); 1449 1450 // Initialize the imports 1451 debug_assert_eq!(imports.functions.len(), module.num_imported_funcs); 1452 ptr::copy_nonoverlapping( 1453 imports.functions.as_ptr(), 1454 self.vmctx_plus_offset_raw(offsets.vmctx_imported_functions_begin()) 1455 .as_ptr(), 1456 imports.functions.len(), 1457 ); 1458 debug_assert_eq!(imports.tables.len(), module.num_imported_tables); 1459 ptr::copy_nonoverlapping( 1460 imports.tables.as_ptr(), 1461 self.vmctx_plus_offset_raw(offsets.vmctx_imported_tables_begin()) 1462 .as_ptr(), 1463 imports.tables.len(), 1464 ); 1465 debug_assert_eq!(imports.memories.len(), module.num_imported_memories); 1466 ptr::copy_nonoverlapping( 1467 imports.memories.as_ptr(), 1468 self.vmctx_plus_offset_raw(offsets.vmctx_imported_memories_begin()) 1469 .as_ptr(), 1470 imports.memories.len(), 1471 ); 1472 debug_assert_eq!(imports.globals.len(), module.num_imported_globals); 1473 ptr::copy_nonoverlapping( 1474 imports.globals.as_ptr(), 1475 self.vmctx_plus_offset_raw(offsets.vmctx_imported_globals_begin()) 1476 .as_ptr(), 1477 imports.globals.len(), 1478 ); 1479 1480 debug_assert_eq!(imports.tags.len(), module.num_imported_tags); 1481 ptr::copy_nonoverlapping( 1482 imports.tags.as_ptr(), 1483 self.vmctx_plus_offset_raw(offsets.vmctx_imported_tags_begin()) 1484 .as_ptr(), 1485 imports.tags.len(), 1486 ); 1487 1488 // N.B.: there is no need to initialize the funcrefs array because we 1489 // eagerly construct each element in it whenever asked for a reference 1490 // to that element. In other words, there is no state needed to track 1491 // the lazy-init, so we don't need to initialize any state now. 1492 1493 // Initialize the defined tables 1494 let mut ptr = self.vmctx_plus_offset_raw(offsets.vmctx_tables_begin()); 1495 for i in 0..module.num_defined_tables() { 1496 ptr.write(self.tables[DefinedTableIndex::new(i)].1.vmtable()); 1497 ptr = ptr.add(1); 1498 } 1499 1500 // Initialize the defined memories. This fills in both the 1501 // `defined_memories` table and the `owned_memories` table at the same 1502 // time. Entries in `defined_memories` hold a pointer to a definition 1503 // (all memories) whereas the `owned_memories` hold the actual 1504 // definitions of memories owned (not shared) in the module. 1505 let mut ptr = self.vmctx_plus_offset_raw(offsets.vmctx_memories_begin()); 1506 let mut owned_ptr = self.vmctx_plus_offset_raw(offsets.vmctx_owned_memories_begin()); 1507 for i in 0..module.num_defined_memories() { 1508 let defined_memory_index = DefinedMemoryIndex::new(i); 1509 let memory_index = module.memory_index(defined_memory_index); 1510 if module.memories[memory_index].shared { 1511 let def_ptr = self.memories[defined_memory_index] 1512 .1 1513 .as_shared_memory() 1514 .unwrap() 1515 .vmmemory_ptr(); 1516 ptr.write(VmPtr::from(def_ptr)); 1517 } else { 1518 owned_ptr.write(self.memories[defined_memory_index].1.vmmemory()); 1519 ptr.write(VmPtr::from(owned_ptr)); 1520 owned_ptr = owned_ptr.add(1); 1521 } 1522 ptr = ptr.add(1); 1523 } 1524 1525 // Zero-initialize the globals so that nothing is uninitialized memory 1526 // after this function returns. The globals are actually initialized 1527 // with their const expression initializers after the instance is fully 1528 // allocated. 1529 for (index, _init) in module.global_initializers.iter() { 1530 self.global_ptr(index).write(VMGlobalDefinition::new()); 1531 } 1532 1533 // Initialize the defined tags 1534 let mut ptr = self.vmctx_plus_offset_raw(offsets.vmctx_tags_begin()); 1535 for i in 0..module.num_defined_tags() { 1536 let defined_index = DefinedTagIndex::new(i); 1537 let tag_index = module.tag_index(defined_index); 1538 let tag = module.tags[tag_index]; 1539 ptr.write(VMTagDefinition::new( 1540 tag.signature.unwrap_engine_type_index(), 1541 )); 1542 ptr = ptr.add(1); 1543 } 1544 } 1545 1546 fn wasm_fault(&self, addr: usize) -> Option<WasmFault> { 1547 let mut fault = None; 1548 for (_, (_, memory)) in self.memories.iter() { 1549 let accessible = memory.wasm_accessible(); 1550 if accessible.start <= addr && addr < accessible.end { 1551 // All linear memories should be disjoint so assert that no 1552 // prior fault has been found. 1553 assert!(fault.is_none()); 1554 fault = Some(WasmFault { 1555 memory_size: memory.byte_size(), 1556 wasm_address: u64::try_from(addr - accessible.start).unwrap(), 1557 }); 1558 } 1559 } 1560 fault 1561 } 1562 1563 /// Returns the id, within this instance's store, that it's assigned. 1564 pub fn id(&self) -> InstanceId { 1565 self.id 1566 } 1567 1568 /// Get all memories within this instance. 1569 /// 1570 /// Returns both import and defined memories. 1571 /// 1572 /// Returns both exported and non-exported memories. 1573 /// 1574 /// Gives access to the full memories space. 1575 pub fn all_memories<'a>( 1576 &'a self, 1577 ) -> impl ExactSizeIterator<Item = (MemoryIndex, ExportMemory)> + 'a { 1578 let indices = (0..self.env_module().memories.len()) 1579 .map(|i| MemoryIndex::new(i)) 1580 .collect::<Vec<_>>(); 1581 indices 1582 .into_iter() 1583 .map(|i| (i, self.get_exported_memory(i))) 1584 } 1585 1586 /// Return the memories defined in this instance (not imported). 1587 pub fn defined_memories<'a>(&'a self) -> impl ExactSizeIterator<Item = ExportMemory> + 'a { 1588 let num_imported = self.env_module().num_imported_memories; 1589 self.all_memories() 1590 .skip(num_imported) 1591 .map(|(_i, memory)| memory) 1592 } 1593 1594 /// Lookup an item with the given index. 1595 /// 1596 /// # Panics 1597 /// 1598 /// Panics if `export` is not valid for this instance. 1599 pub fn get_export_by_index(&self, export: EntityIndex) -> Export { 1600 match export { 1601 EntityIndex::Function(i) => Export::Function(self.get_exported_func(i)), 1602 EntityIndex::Global(i) => Export::Global(self.get_exported_global(i)), 1603 EntityIndex::Table(i) => Export::Table(self.get_exported_table(i)), 1604 EntityIndex::Memory(i) => Export::Memory(self.get_exported_memory(i)), 1605 EntityIndex::Tag(i) => Export::Tag(self.get_exported_tag(i)), 1606 } 1607 } 1608 } 1609 1610 /// A handle holding an `Instance` of a WebAssembly module. 1611 #[derive(Debug)] 1612 pub struct InstanceHandle { 1613 instance: Option<SendSyncPtr<Instance>>, 1614 } 1615 1616 impl InstanceHandle { 1617 /// Creates an "empty" instance handle which internally has a null pointer 1618 /// to an instance. 1619 pub fn null() -> InstanceHandle { 1620 InstanceHandle { instance: None } 1621 } 1622 1623 /// Return a raw pointer to the vmctx used by compiled wasm code. 1624 #[inline] 1625 pub fn vmctx(&self) -> NonNull<VMContext> { 1626 self.instance().vmctx() 1627 } 1628 1629 /// Return a reference to a module. 1630 pub fn module(&self) -> &Arc<Module> { 1631 self.instance().env_module() 1632 } 1633 1634 /// Lookup a function by index. 1635 pub fn get_exported_func(&self, export: FuncIndex) -> ExportFunction { 1636 self.instance().get_exported_func(export) 1637 } 1638 1639 /// Lookup a global by index. 1640 pub fn get_exported_global(&self, export: GlobalIndex) -> ExportGlobal { 1641 self.instance().get_exported_global(export) 1642 } 1643 1644 /// Lookup a tag by index. 1645 pub fn get_exported_tag(&self, export: TagIndex) -> ExportTag { 1646 self.instance().get_exported_tag(export) 1647 } 1648 1649 /// Lookup a memory by index. 1650 pub fn get_exported_memory(&self, export: MemoryIndex) -> ExportMemory { 1651 self.instance().get_exported_memory(export) 1652 } 1653 1654 /// Lookup a table by index. 1655 pub fn get_exported_table(&self, export: TableIndex) -> ExportTable { 1656 self.instance().get_exported_table(export) 1657 } 1658 1659 /// Return an iterator over the exports of this instance. 1660 /// 1661 /// Specifically, it provides access to the key-value pairs, where the keys 1662 /// are export names, and the values are export declarations which can be 1663 /// resolved `lookup_by_declaration`. 1664 pub fn exports(&self) -> wasmparser::collections::index_map::Iter<String, EntityIndex> { 1665 self.instance().exports() 1666 } 1667 1668 /// Get a table defined locally within this module. 1669 pub fn get_defined_table(&mut self, index: DefinedTableIndex) -> *mut Table { 1670 self.instance_mut().get_defined_table(index) 1671 } 1672 1673 /// Get a table defined locally within this module, lazily 1674 /// initializing the given range first. 1675 pub fn get_defined_table_with_lazy_init( 1676 &mut self, 1677 index: DefinedTableIndex, 1678 range: impl Iterator<Item = u64>, 1679 ) -> *mut Table { 1680 let index = self.instance().env_module().table_index(index); 1681 self.instance_mut().get_table_with_lazy_init(index, range) 1682 } 1683 1684 /// Get all tables within this instance. 1685 /// 1686 /// Returns both import and defined tables. 1687 /// 1688 /// Returns both exported and non-exported tables. 1689 /// 1690 /// Gives access to the full tables space. 1691 pub fn all_tables<'a>( 1692 &'a mut self, 1693 ) -> impl ExactSizeIterator<Item = (TableIndex, ExportTable)> + 'a { 1694 let indices = (0..self.module().tables.len()) 1695 .map(|i| TableIndex::new(i)) 1696 .collect::<Vec<_>>(); 1697 indices.into_iter().map(|i| (i, self.get_exported_table(i))) 1698 } 1699 1700 /// Return the tables defined in this instance (not imported). 1701 pub fn defined_tables<'a>(&'a mut self) -> impl ExactSizeIterator<Item = ExportTable> + 'a { 1702 let num_imported = self.module().num_imported_tables; 1703 self.all_tables() 1704 .skip(num_imported) 1705 .map(|(_i, table)| table) 1706 } 1707 1708 /// Get all globals within this instance. 1709 /// 1710 /// Returns both import and defined globals. 1711 /// 1712 /// Returns both exported and non-exported globals. 1713 /// 1714 /// Gives access to the full globals space. 1715 pub fn all_globals<'a>( 1716 &'a mut self, 1717 ) -> impl ExactSizeIterator<Item = (GlobalIndex, ExportGlobal)> + 'a { 1718 self.instance_mut().all_globals() 1719 } 1720 1721 /// Get the globals defined in this instance (not imported). 1722 pub fn defined_globals<'a>( 1723 &'a mut self, 1724 ) -> impl ExactSizeIterator<Item = (DefinedGlobalIndex, ExportGlobal)> + 'a { 1725 self.instance_mut().defined_globals() 1726 } 1727 1728 /// Return a reference to the contained `Instance`. 1729 #[inline] 1730 pub(crate) fn instance(&self) -> &Instance { 1731 unsafe { &*self.instance.unwrap().as_ptr() } 1732 } 1733 1734 pub(crate) fn instance_mut(&mut self) -> &mut Instance { 1735 unsafe { &mut *self.instance.unwrap().as_ptr() } 1736 } 1737 1738 /// Get this instance's `dyn VMStore` trait object. 1739 /// 1740 /// This should only be used for initializing a vmctx's store pointer. It 1741 /// should never be used to access the store itself. Use `InstanceAndStore` 1742 /// for that instead. 1743 pub fn traitobj(&self, store: &StoreOpaque) -> NonNull<dyn VMStore> { 1744 // By requiring a store argument, we are ensuring that callers aren't 1745 // getting this trait object in order to access the store, since they 1746 // already have access. See `InstanceAndStore` and its documentation for 1747 // details about the store access patterns we want to restrict host code 1748 // to. 1749 let _ = store; 1750 1751 self.instance().store.unwrap().0 1752 } 1753 1754 /// Configure the `*mut dyn Store` internal pointer after-the-fact. 1755 /// 1756 /// This is provided for the original `Store` itself to configure the first 1757 /// self-pointer after the original `Box` has been initialized. 1758 pub unsafe fn set_store(&mut self, store: Option<NonNull<dyn VMStore>>) { 1759 self.instance_mut().set_store(store); 1760 } 1761 1762 /// Returns a clone of this instance. 1763 /// 1764 /// This is unsafe because the returned handle here is just a cheap clone 1765 /// of the internals, there's no lifetime tracking around its validity. 1766 /// You'll need to ensure that the returned handles all go out of scope at 1767 /// the same time. 1768 #[inline] 1769 pub unsafe fn clone(&self) -> InstanceHandle { 1770 InstanceHandle { 1771 instance: self.instance, 1772 } 1773 } 1774 1775 /// Performs post-initialization of an instance after its handle has been 1776 /// created and registered with a store. 1777 /// 1778 /// Failure of this function means that the instance still must persist 1779 /// within the store since failure may indicate partial failure, or some 1780 /// state could be referenced by other instances. 1781 pub fn initialize( 1782 &mut self, 1783 store: &mut StoreOpaque, 1784 module: &Module, 1785 is_bulk_memory: bool, 1786 ) -> Result<()> { 1787 allocator::initialize_instance(store, self.instance_mut(), module, is_bulk_memory) 1788 } 1789 1790 /// Attempts to convert from the host `addr` specified to a WebAssembly 1791 /// based address recorded in `WasmFault`. 1792 /// 1793 /// This method will check all linear memories that this instance contains 1794 /// to see if any of them contain `addr`. If one does then `Some` is 1795 /// returned with metadata about the wasm fault. Otherwise `None` is 1796 /// returned and `addr` doesn't belong to this instance. 1797 pub fn wasm_fault(&self, addr: usize) -> Option<WasmFault> { 1798 self.instance().wasm_fault(addr) 1799 } 1800 } 1801