1 //! Memory management for tables. 2 //! 3 //! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories. 4 5 #![cfg_attr(feature = "gc", allow(irrefutable_let_patterns))] 6 7 use crate::prelude::*; 8 use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition}; 9 use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VMStore}; 10 use core::alloc::Layout; 11 use core::mem; 12 use core::ops::Range; 13 use core::ptr::{self, NonNull}; 14 use core::slice; 15 use core::{cmp, usize}; 16 use sptr::Strict; 17 use wasmtime_environ::{ 18 FUNCREF_INIT_BIT, FUNCREF_MASK, IndexType, Trap, Tunables, WasmHeapTopType, WasmRefType, 19 }; 20 21 /// An element going into or coming out of a table. 22 /// 23 /// Table elements are stored as pointers and are default-initialized with 24 /// `ptr::null_mut`. 25 pub enum TableElement { 26 /// A `funcref`. 27 FuncRef(Option<NonNull<VMFuncRef>>), 28 29 /// A GC reference. 30 GcRef(Option<VMGcRef>), 31 32 /// An uninitialized funcref value. This should never be exposed 33 /// beyond the `wasmtime` crate boundary; the upper-level code 34 /// (which has access to the info needed for lazy initialization) 35 /// will replace it when fetched. 36 UninitFunc, 37 } 38 39 #[derive(Copy, Clone, PartialEq, Eq, Debug)] 40 pub enum TableElementType { 41 Func, 42 GcRef, 43 } 44 45 impl TableElementType { 46 fn matches(&self, val: &TableElement) -> bool { 47 match (val, self) { 48 (TableElement::FuncRef(_), TableElementType::Func) => true, 49 (TableElement::GcRef(_), TableElementType::GcRef) => true, 50 _ => false, 51 } 52 } 53 } 54 55 // The usage of `*mut VMFuncRef` is safe w.r.t. thread safety, this just relies 56 // on thread-safety of `VMGcRef` itself. 57 unsafe impl Send for TableElement where VMGcRef: Send {} 58 unsafe impl Sync for TableElement where VMGcRef: Sync {} 59 60 impl TableElement { 61 /// Consumes a table element into a pointer/reference, as it 62 /// exists outside the table itself. This strips off any tag bits 63 /// or other information that only lives inside the table. 64 /// 65 /// Can only be done to an initialized table element; lazy init 66 /// must occur first. (In other words, lazy values do not survive 67 /// beyond the table, as every table read path initializes them.) 68 /// 69 /// # Safety 70 /// 71 /// The same warnings as for `into_table_values()` apply. 72 pub(crate) unsafe fn into_func_ref_asserting_initialized(self) -> Option<NonNull<VMFuncRef>> { 73 match self { 74 Self::FuncRef(e) => e, 75 Self::UninitFunc => panic!("Uninitialized table element value outside of table slot"), 76 Self::GcRef(_) => panic!("GC reference is not a function reference"), 77 } 78 } 79 80 /// Indicates whether this value is the "uninitialized element" 81 /// value. 82 pub(crate) fn is_uninit(&self) -> bool { 83 match self { 84 Self::UninitFunc => true, 85 _ => false, 86 } 87 } 88 } 89 90 impl From<Option<NonNull<VMFuncRef>>> for TableElement { 91 fn from(f: Option<NonNull<VMFuncRef>>) -> TableElement { 92 TableElement::FuncRef(f) 93 } 94 } 95 96 impl From<Option<VMGcRef>> for TableElement { 97 fn from(x: Option<VMGcRef>) -> TableElement { 98 TableElement::GcRef(x) 99 } 100 } 101 102 impl From<VMGcRef> for TableElement { 103 fn from(x: VMGcRef) -> TableElement { 104 TableElement::GcRef(Some(x)) 105 } 106 } 107 108 #[derive(Copy, Clone)] 109 #[repr(transparent)] 110 struct TaggedFuncRef(*mut VMFuncRef); 111 112 impl TaggedFuncRef { 113 const UNINIT: TaggedFuncRef = TaggedFuncRef(ptr::null_mut()); 114 115 /// Converts the given `ptr`, a valid funcref pointer, into a tagged pointer 116 /// by adding in the `FUNCREF_INIT_BIT`. 117 fn from(ptr: Option<NonNull<VMFuncRef>>, lazy_init: bool) -> Self { 118 let ptr = ptr.map(|p| p.as_ptr()).unwrap_or(ptr::null_mut()); 119 if lazy_init { 120 let masked = Strict::map_addr(ptr, |a| a | FUNCREF_INIT_BIT); 121 TaggedFuncRef(masked) 122 } else { 123 TaggedFuncRef(ptr) 124 } 125 } 126 127 /// Converts a tagged pointer into a `TableElement`, returning `UninitFunc` 128 /// for null (not a tagged value) or `FuncRef` for otherwise tagged values. 129 fn into_table_element(self, lazy_init: bool) -> TableElement { 130 let ptr = self.0; 131 if lazy_init && ptr.is_null() { 132 TableElement::UninitFunc 133 } else { 134 // Masking off the tag bit is harmless whether the table uses lazy 135 // init or not. 136 let unmasked = Strict::map_addr(ptr, |a| a & FUNCREF_MASK); 137 TableElement::FuncRef(NonNull::new(unmasked)) 138 } 139 } 140 } 141 142 pub type FuncTableElem = Option<SendSyncPtr<VMFuncRef>>; 143 144 pub enum StaticTable { 145 Func(StaticFuncTable), 146 GcRef(StaticGcRefTable), 147 } 148 149 impl From<StaticFuncTable> for StaticTable { 150 fn from(value: StaticFuncTable) -> Self { 151 Self::Func(value) 152 } 153 } 154 155 impl From<StaticGcRefTable> for StaticTable { 156 fn from(value: StaticGcRefTable) -> Self { 157 Self::GcRef(value) 158 } 159 } 160 161 pub struct StaticFuncTable { 162 /// Where data for this table is stored. The length of this list is the 163 /// maximum size of the table. 164 data: SendSyncPtr<[FuncTableElem]>, 165 /// The current size of the table. 166 size: usize, 167 /// Whether elements of this table are initialized lazily. 168 lazy_init: bool, 169 } 170 171 pub struct StaticGcRefTable { 172 /// Where data for this table is stored. The length of this list is the 173 /// maximum size of the table. 174 data: SendSyncPtr<[Option<VMGcRef>]>, 175 /// The current size of the table. 176 size: usize, 177 } 178 179 pub enum DynamicTable { 180 Func(DynamicFuncTable), 181 GcRef(DynamicGcRefTable), 182 } 183 184 impl From<DynamicFuncTable> for DynamicTable { 185 fn from(value: DynamicFuncTable) -> Self { 186 Self::Func(value) 187 } 188 } 189 190 impl From<DynamicGcRefTable> for DynamicTable { 191 fn from(value: DynamicGcRefTable) -> Self { 192 Self::GcRef(value) 193 } 194 } 195 196 pub struct DynamicFuncTable { 197 /// Dynamically managed storage space for this table. The length of this 198 /// vector is the current size of the table. 199 elements: Vec<FuncTableElem>, 200 /// Maximum size that `elements` can grow to. 201 maximum: Option<usize>, 202 /// Whether elements of this table are initialized lazily. 203 lazy_init: bool, 204 } 205 206 pub struct DynamicGcRefTable { 207 /// Dynamically managed storage space for this table. The length of this 208 /// vector is the current size of the table. 209 elements: Vec<Option<VMGcRef>>, 210 /// Maximum size that `elements` can grow to. 211 maximum: Option<usize>, 212 } 213 214 /// Represents an instance's table. 215 pub enum Table { 216 /// A "static" table where storage space is managed externally, currently 217 /// used with the pooling allocator. 218 Static(StaticTable), 219 /// A "dynamic" table where table storage space is dynamically allocated via 220 /// `malloc` (aka Rust's `Vec`). 221 Dynamic(DynamicTable), 222 } 223 224 impl From<StaticTable> for Table { 225 fn from(value: StaticTable) -> Self { 226 Self::Static(value) 227 } 228 } 229 230 impl From<StaticFuncTable> for Table { 231 fn from(value: StaticFuncTable) -> Self { 232 let t: StaticTable = value.into(); 233 t.into() 234 } 235 } 236 237 impl From<StaticGcRefTable> for Table { 238 fn from(value: StaticGcRefTable) -> Self { 239 let t: StaticTable = value.into(); 240 t.into() 241 } 242 } 243 244 impl From<DynamicTable> for Table { 245 fn from(value: DynamicTable) -> Self { 246 Self::Dynamic(value) 247 } 248 } 249 250 impl From<DynamicFuncTable> for Table { 251 fn from(value: DynamicFuncTable) -> Self { 252 let t: DynamicTable = value.into(); 253 t.into() 254 } 255 } 256 257 impl From<DynamicGcRefTable> for Table { 258 fn from(value: DynamicGcRefTable) -> Self { 259 let t: DynamicTable = value.into(); 260 t.into() 261 } 262 } 263 264 fn wasm_to_table_type(ty: WasmRefType) -> TableElementType { 265 match ty.heap_type.top() { 266 WasmHeapTopType::Func => TableElementType::Func, 267 WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef, 268 WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. 269 } 270 } 271 272 /// Allocate dynamic table elements of the given length. 273 /// 274 /// Relies on the fact that our tables' elements are initialized to `None`, 275 /// which is represented by zero, to allocate pre-zeroed memory from the global 276 /// allocator and avoid manual zero-initialization. 277 /// 278 /// # Safety 279 /// 280 /// Should only ever be called with a `T` that is a table element type and where 281 /// `Option<T>`'s `None` variant is represented with zero. 282 unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<Vec<Option<T>>> { 283 debug_assert!( 284 core::mem::MaybeUninit::<Option<T>>::zeroed() 285 .assume_init() 286 .is_none(), 287 "null table elements are represented with zeroed memory" 288 ); 289 290 if len == 0 { 291 return Ok(vec![]); 292 } 293 294 let align = mem::align_of::<Option<T>>(); 295 296 let size = mem::size_of::<Option<T>>(); 297 let size = size.next_multiple_of(align); 298 let size = size.checked_mul(len).unwrap(); 299 300 let layout = Layout::from_size_align(size, align)?; 301 302 let ptr = alloc::alloc::alloc_zeroed(layout); 303 ensure!(!ptr.is_null(), "failed to allocate memory for table"); 304 305 let elems = Vec::<Option<T>>::from_raw_parts(ptr.cast(), len, len); 306 debug_assert!(elems.iter().all(|e| e.is_none())); 307 308 Ok(elems) 309 } 310 311 impl Table { 312 /// Create a new dynamic (movable) table instance for the specified table plan. 313 pub fn new_dynamic( 314 ty: &wasmtime_environ::Table, 315 tunables: &Tunables, 316 store: &mut dyn VMStore, 317 ) -> Result<Self> { 318 let (minimum, maximum) = Self::limit_new(ty, store)?; 319 match wasm_to_table_type(ty.ref_type) { 320 TableElementType::Func => Ok(Self::from(DynamicFuncTable { 321 elements: unsafe { alloc_dynamic_table_elements(minimum)? }, 322 maximum, 323 lazy_init: tunables.table_lazy_init, 324 })), 325 TableElementType::GcRef => Ok(Self::from(DynamicGcRefTable { 326 elements: unsafe { alloc_dynamic_table_elements(minimum)? }, 327 maximum, 328 })), 329 } 330 } 331 332 /// Create a new static (immovable) table instance for the specified table plan. 333 pub unsafe fn new_static( 334 ty: &wasmtime_environ::Table, 335 tunables: &Tunables, 336 data: SendSyncPtr<[u8]>, 337 store: &mut dyn VMStore, 338 ) -> Result<Self> { 339 let (minimum, maximum) = Self::limit_new(ty, store)?; 340 let size = minimum; 341 let max = maximum.unwrap_or(usize::MAX); 342 343 match wasm_to_table_type(ty.ref_type) { 344 TableElementType::Func => { 345 let len = { 346 let data = data.as_non_null().as_ref(); 347 let (before, data, after) = data.align_to::<FuncTableElem>(); 348 assert!(before.is_empty()); 349 assert!(after.is_empty()); 350 data.len() 351 }; 352 ensure!( 353 usize::try_from(ty.limits.min).unwrap() <= len, 354 "initial table size of {} exceeds the pooling allocator's \ 355 configured maximum table size of {len} elements", 356 ty.limits.min, 357 ); 358 let data = SendSyncPtr::new(NonNull::slice_from_raw_parts( 359 data.as_non_null().cast::<FuncTableElem>(), 360 cmp::min(len, max), 361 )); 362 Ok(Self::from(StaticFuncTable { 363 data, 364 size, 365 lazy_init: tunables.table_lazy_init, 366 })) 367 } 368 TableElementType::GcRef => { 369 let len = { 370 let data = data.as_non_null().as_ref(); 371 let (before, data, after) = data.align_to::<Option<VMGcRef>>(); 372 assert!(before.is_empty()); 373 assert!(after.is_empty()); 374 data.len() 375 }; 376 ensure!( 377 usize::try_from(ty.limits.min).unwrap() <= len, 378 "initial table size of {} exceeds the pooling allocator's \ 379 configured maximum table size of {len} elements", 380 ty.limits.min, 381 ); 382 let data = SendSyncPtr::new(NonNull::slice_from_raw_parts( 383 data.as_non_null().cast::<Option<VMGcRef>>(), 384 cmp::min(len, max), 385 )); 386 Ok(Self::from(StaticGcRefTable { data, size })) 387 } 388 } 389 } 390 391 // Calls the `store`'s limiter to optionally prevent the table from being created. 392 // 393 // Returns the minimum and maximum size of the table if the table can be created. 394 fn limit_new( 395 ty: &wasmtime_environ::Table, 396 store: &mut dyn VMStore, 397 ) -> Result<(usize, Option<usize>)> { 398 // No matter how the table limits are specified 399 // The table size is limited by the host's pointer size 400 let absolute_max = usize::MAX; 401 402 // If the minimum overflows the host's pointer size, then we can't satisfy this request. 403 // We defer the error to later so the `store` can be informed. 404 let minimum = usize::try_from(ty.limits.min).ok(); 405 406 // The maximum size of the table is limited by: 407 // * the host's pointer size. 408 // * the table's maximum size if defined. 409 // * if the table is 64-bit. 410 let maximum = match (ty.limits.max, ty.idx_type) { 411 (Some(max), _) => usize::try_from(max).ok(), 412 (None, IndexType::I64) => usize::try_from(u64::MAX).ok(), 413 (None, IndexType::I32) => usize::try_from(u32::MAX).ok(), 414 }; 415 416 // Inform the store's limiter what's about to happen. 417 if !store.table_growing(0, minimum.unwrap_or(absolute_max), maximum)? { 418 bail!( 419 "table minimum size of {} elements exceeds table limits", 420 ty.limits.min 421 ); 422 } 423 424 // At this point we need to actually handle overflows, so bail out with 425 // an error if we made it this far. 426 let minimum = minimum.ok_or_else(|| { 427 format_err!( 428 "table minimum size of {} elements exceeds table limits", 429 ty.limits.min 430 ) 431 })?; 432 Ok((minimum, maximum)) 433 } 434 435 /// Returns the type of the elements in this table. 436 pub fn element_type(&self) -> TableElementType { 437 match self { 438 Table::Static(StaticTable::Func(_)) | Table::Dynamic(DynamicTable::Func(_)) => { 439 TableElementType::Func 440 } 441 Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => { 442 TableElementType::GcRef 443 } 444 } 445 } 446 447 /// Returns whether or not the underlying storage of the table is "static". 448 #[cfg(feature = "pooling-allocator")] 449 pub(crate) fn is_static(&self) -> bool { 450 matches!(self, Table::Static(_)) 451 } 452 453 /// Returns the number of allocated elements. 454 pub fn size(&self) -> usize { 455 match self { 456 Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size, 457 Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size, 458 Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(), 459 Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { 460 elements.len() 461 } 462 } 463 } 464 465 /// Returns the maximum number of elements at runtime. 466 /// 467 /// Returns `None` if the table is unbounded. 468 /// 469 /// The runtime maximum may not be equal to the maximum from the table's Wasm type 470 /// when it is being constrained by an instance allocator. 471 pub fn maximum(&self) -> Option<usize> { 472 match self { 473 Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()), 474 Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()), 475 Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum, 476 Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum, 477 } 478 } 479 480 /// Initializes the contents of this table to the specified function 481 /// 482 /// # Panics 483 /// 484 /// Panics if the table is not a function table. 485 pub fn init_func( 486 &mut self, 487 dst: u64, 488 items: impl ExactSizeIterator<Item = Option<NonNull<VMFuncRef>>>, 489 ) -> Result<(), Trap> { 490 let dst = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?; 491 492 let (funcrefs, lazy_init) = self.funcrefs_mut(); 493 let elements = funcrefs 494 .get_mut(dst..) 495 .and_then(|s| s.get_mut(..items.len())) 496 .ok_or(Trap::TableOutOfBounds)?; 497 498 for (item, slot) in items.zip(elements) { 499 *slot = TaggedFuncRef::from(item, lazy_init); 500 } 501 Ok(()) 502 } 503 504 /// Fill `table[dst..]` with values from `items` 505 /// 506 /// Returns a trap error on out-of-bounds accesses. 507 pub fn init_gc_refs( 508 &mut self, 509 dst: u64, 510 items: impl ExactSizeIterator<Item = Option<VMGcRef>>, 511 ) -> Result<(), Trap> { 512 let dst = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?; 513 514 let elements = self 515 .gc_refs_mut() 516 .get_mut(dst..) 517 .and_then(|s| s.get_mut(..items.len())) 518 .ok_or(Trap::TableOutOfBounds)?; 519 520 for (item, slot) in items.zip(elements) { 521 *slot = item; 522 } 523 Ok(()) 524 } 525 526 /// Fill `table[dst..dst + len]` with `val`. 527 /// 528 /// Returns a trap error on out-of-bounds accesses. 529 /// 530 /// # Panics 531 /// 532 /// Panics if `val` does not have a type that matches this table, or if 533 /// `gc_store.is_none()` and this is a table of GC references. 534 pub fn fill( 535 &mut self, 536 mut gc_store: Option<&mut GcStore>, 537 dst: u64, 538 val: TableElement, 539 len: u64, 540 ) -> Result<(), Trap> { 541 let start = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?; 542 let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?; 543 let end = start 544 .checked_add(len) 545 .ok_or_else(|| Trap::TableOutOfBounds)?; 546 547 if end > self.size() { 548 return Err(Trap::TableOutOfBounds); 549 } 550 551 match val { 552 TableElement::FuncRef(f) => { 553 let (funcrefs, lazy_init) = self.funcrefs_mut(); 554 funcrefs[start..end].fill(TaggedFuncRef::from(f, lazy_init)); 555 } 556 TableElement::GcRef(r) => { 557 // Clone the init GC reference into each table slot. 558 for slot in &mut self.gc_refs_mut()[start..end] { 559 match gc_store.as_deref_mut() { 560 Some(s) => s.write_gc_ref(slot, r.as_ref()), 561 None => { 562 debug_assert!(slot.as_ref().is_none_or(|x| x.is_i31())); 563 debug_assert!(r.as_ref().is_none_or(|r| r.is_i31())); 564 *slot = r.as_ref().map(|r| r.copy_i31()); 565 } 566 } 567 } 568 569 // Drop the init GC reference, since we aren't holding onto this 570 // reference anymore, only the clones in the table. 571 if let Some(r) = r { 572 match gc_store { 573 Some(s) => s.drop_gc_ref(r), 574 None => debug_assert!(r.is_i31()), 575 } 576 } 577 } 578 TableElement::UninitFunc => { 579 let (funcrefs, _lazy_init) = self.funcrefs_mut(); 580 funcrefs[start..end].fill(TaggedFuncRef::UNINIT); 581 } 582 } 583 584 Ok(()) 585 } 586 587 /// Grow table by the specified amount of elements. 588 /// 589 /// Returns the previous size of the table if growth is successful. 590 /// 591 /// Returns `None` if table can't be grown by the specified amount of 592 /// elements, or if the `init_value` is the wrong kind of table element. 593 /// 594 /// # Panics 595 /// 596 /// Panics if `init_value` does not have a type that matches this table. 597 /// 598 /// # Unsafety 599 /// 600 /// Resizing the table can reallocate its internal elements buffer. This 601 /// table's instance's `VMContext` has raw pointers to the elements buffer 602 /// that are used by Wasm, and they need to be fixed up before we call into 603 /// Wasm again. Failure to do so will result in use-after-free inside Wasm. 604 /// 605 /// Generally, prefer using `InstanceHandle::table_grow`, which encapsulates 606 /// this unsafety. 607 pub unsafe fn grow( 608 &mut self, 609 delta: u64, 610 init_value: TableElement, 611 store: &mut dyn VMStore, 612 ) -> Result<Option<usize>, Error> { 613 let old_size = self.size(); 614 615 // Don't try to resize the table if its size isn't changing, just return 616 // success. 617 if delta == 0 { 618 return Ok(Some(old_size)); 619 } 620 // Cannot return `Trap::TableOutOfBounds` here because `impl std::error::Error for Trap` is not available in no-std. 621 let delta = 622 usize::try_from(delta).map_err(|_| format_err!("delta exceeds host pointer size"))?; 623 624 let new_size = match old_size.checked_add(delta) { 625 Some(s) => s, 626 None => { 627 store.table_grow_failed(format_err!("overflow calculating new table size"))?; 628 return Ok(None); 629 } 630 }; 631 632 if !store.table_growing(old_size, new_size, self.maximum())? { 633 return Ok(None); 634 } 635 636 // The WebAssembly spec requires failing a `table.grow` request if 637 // it exceeds the declared limits of the table. We may have set lower 638 // limits in the instance allocator as well. 639 if let Some(max) = self.maximum() { 640 if new_size > max { 641 store.table_grow_failed(format_err!("Table maximum size exceeded"))?; 642 return Ok(None); 643 } 644 } 645 646 debug_assert!(self.type_matches(&init_value)); 647 648 // First resize the storage and then fill with the init value 649 match self { 650 Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => { 651 unsafe { 652 debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none())); 653 } 654 *size = new_size; 655 } 656 Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => { 657 unsafe { 658 debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none())); 659 } 660 *size = new_size; 661 } 662 663 // These calls to `resize` could move the base address of 664 // `elements`. If this table's limits declare it to be fixed-size, 665 // then during AOT compilation we may have promised Cranelift that 666 // the table base address won't change, so it is allowed to optimize 667 // loading the base address. However, in that case the above checks 668 // that delta is non-zero and the new size doesn't exceed the 669 // maximum mean we can't get here. 670 Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => { 671 elements.resize(usize::try_from(new_size).unwrap(), None); 672 } 673 Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { 674 elements.resize_with(usize::try_from(new_size).unwrap(), || None); 675 } 676 } 677 678 self.fill( 679 store.store_opaque_mut().optional_gc_store_mut(), 680 u64::try_from(old_size).unwrap(), 681 init_value, 682 u64::try_from(delta).unwrap(), 683 ) 684 .expect("table should not be out of bounds"); 685 686 Ok(Some(old_size)) 687 } 688 689 /// Get reference to the specified element. 690 /// 691 /// Returns `None` if the index is out of bounds. 692 /// 693 /// Panics if this is a table of GC references and `gc_store` is `None`. 694 pub fn get(&self, gc_store: Option<&mut GcStore>, index: u64) -> Option<TableElement> { 695 let index = usize::try_from(index).ok()?; 696 match self.element_type() { 697 TableElementType::Func => { 698 let (funcrefs, lazy_init) = self.funcrefs(); 699 funcrefs 700 .get(index) 701 .copied() 702 .map(|e| e.into_table_element(lazy_init)) 703 } 704 TableElementType::GcRef => self.gc_refs().get(index).map(|r| { 705 let r = r.as_ref().map(|r| match gc_store { 706 Some(s) => s.clone_gc_ref(r), 707 None => r.copy_i31(), 708 }); 709 710 TableElement::GcRef(r) 711 }), 712 } 713 } 714 715 /// Set reference to the specified element. 716 /// 717 /// # Errors 718 /// 719 /// Returns an error if `index` is out of bounds or if this table type does 720 /// not match the element type. 721 /// 722 /// # Panics 723 /// 724 /// Panics if `elem` is not of the right type for this table. 725 pub fn set(&mut self, index: u64, elem: TableElement) -> Result<(), ()> { 726 let index: usize = index.try_into().map_err(|_| ())?; 727 match elem { 728 TableElement::FuncRef(f) => { 729 let (funcrefs, lazy_init) = self.funcrefs_mut(); 730 *funcrefs.get_mut(index).ok_or(())? = TaggedFuncRef::from(f, lazy_init); 731 } 732 TableElement::UninitFunc => { 733 let (funcrefs, _lazy_init) = self.funcrefs_mut(); 734 *funcrefs.get_mut(index).ok_or(())? = TaggedFuncRef::UNINIT; 735 } 736 TableElement::GcRef(e) => { 737 *self.gc_refs_mut().get_mut(index).ok_or(())? = e; 738 } 739 } 740 Ok(()) 741 } 742 743 /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`. 744 /// 745 /// # Errors 746 /// 747 /// Returns an error if the range is out of bounds of either the source or 748 /// destination tables. 749 pub unsafe fn copy( 750 gc_store: Option<&mut GcStore>, 751 dst_table: *mut Self, 752 src_table: *mut Self, 753 dst_index: u64, 754 src_index: u64, 755 len: u64, 756 ) -> Result<(), Trap> { 757 // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-copy 758 759 let src_index = usize::try_from(src_index).map_err(|_| Trap::TableOutOfBounds)?; 760 let dst_index = usize::try_from(dst_index).map_err(|_| Trap::TableOutOfBounds)?; 761 let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?; 762 763 if src_index 764 .checked_add(len) 765 .map_or(true, |n| n > (*src_table).size()) 766 || dst_index 767 .checked_add(len) 768 .map_or(true, |m| m > (*dst_table).size()) 769 { 770 return Err(Trap::TableOutOfBounds); 771 } 772 773 debug_assert!( 774 (*dst_table).element_type() == (*src_table).element_type(), 775 "table element type mismatch" 776 ); 777 778 let src_range = src_index..src_index + len; 779 let dst_range = dst_index..dst_index + len; 780 781 // Check if the tables are the same as we cannot mutably borrow and also borrow the same `RefCell` 782 if ptr::eq(dst_table, src_table) { 783 (*dst_table).copy_elements_within(gc_store, dst_range, src_range); 784 } else { 785 Self::copy_elements(gc_store, &mut *dst_table, &*src_table, dst_range, src_range); 786 } 787 788 Ok(()) 789 } 790 791 /// Return a `VMTableDefinition` for exposing the table to compiled wasm code. 792 pub fn vmtable(&mut self) -> VMTableDefinition { 793 match self { 794 Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => { 795 VMTableDefinition { 796 base: data.cast().into(), 797 current_elements: *size, 798 } 799 } 800 Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => { 801 VMTableDefinition { 802 base: data.cast().into(), 803 current_elements: *size, 804 } 805 } 806 Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => { 807 VMTableDefinition { 808 base: NonNull::<[FuncTableElem]>::from(&mut elements[..]) 809 .cast() 810 .into(), 811 current_elements: elements.len(), 812 } 813 } 814 Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { 815 VMTableDefinition { 816 base: NonNull::<[Option<VMGcRef>]>::from(&mut elements[..]) 817 .cast() 818 .into(), 819 current_elements: elements.len(), 820 } 821 } 822 } 823 } 824 825 fn type_matches(&self, val: &TableElement) -> bool { 826 self.element_type().matches(val) 827 } 828 829 fn funcrefs(&self) -> (&[TaggedFuncRef], bool) { 830 assert_eq!(self.element_type(), TableElementType::Func); 831 match self { 832 Self::Dynamic(DynamicTable::Func(DynamicFuncTable { 833 elements, 834 lazy_init, 835 .. 836 })) => ( 837 unsafe { slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) }, 838 *lazy_init, 839 ), 840 Self::Static(StaticTable::Func(StaticFuncTable { 841 data, 842 size, 843 lazy_init, 844 })) => ( 845 unsafe { 846 slice::from_raw_parts(data.as_ptr().cast(), usize::try_from(*size).unwrap()) 847 }, 848 *lazy_init, 849 ), 850 _ => unreachable!(), 851 } 852 } 853 854 fn funcrefs_mut(&mut self) -> (&mut [TaggedFuncRef], bool) { 855 assert_eq!(self.element_type(), TableElementType::Func); 856 match self { 857 Self::Dynamic(DynamicTable::Func(DynamicFuncTable { 858 elements, 859 lazy_init, 860 .. 861 })) => ( 862 unsafe { slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) }, 863 *lazy_init, 864 ), 865 Self::Static(StaticTable::Func(StaticFuncTable { 866 data, 867 size, 868 lazy_init, 869 })) => ( 870 unsafe { 871 slice::from_raw_parts_mut(data.as_ptr().cast(), usize::try_from(*size).unwrap()) 872 }, 873 *lazy_init, 874 ), 875 _ => unreachable!(), 876 } 877 } 878 879 fn gc_refs(&self) -> &[Option<VMGcRef>] { 880 assert_eq!(self.element_type(), TableElementType::GcRef); 881 match self { 882 Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements, 883 Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe { 884 &data.as_non_null().as_ref()[..usize::try_from(*size).unwrap()] 885 }, 886 _ => unreachable!(), 887 } 888 } 889 890 /// Get this table's GC references as a slice. 891 /// 892 /// Panics if this is not a table of GC references. 893 pub fn gc_refs_mut(&mut self) -> &mut [Option<VMGcRef>] { 894 assert_eq!(self.element_type(), TableElementType::GcRef); 895 match self { 896 Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements, 897 Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe { 898 &mut data.as_non_null().as_mut()[..usize::try_from(*size).unwrap()] 899 }, 900 _ => unreachable!(), 901 } 902 } 903 904 fn copy_elements( 905 gc_store: Option<&mut GcStore>, 906 dst_table: &mut Self, 907 src_table: &Self, 908 dst_range: Range<usize>, 909 src_range: Range<usize>, 910 ) { 911 // This can only be used when copying between different tables 912 debug_assert!(!ptr::eq(dst_table, src_table)); 913 914 let ty = dst_table.element_type(); 915 916 match ty { 917 TableElementType::Func => { 918 // `funcref` are `Copy`, so just do a mempcy 919 let (dst_funcrefs, _lazy_init) = dst_table.funcrefs_mut(); 920 let (src_funcrefs, _lazy_init) = src_table.funcrefs(); 921 dst_funcrefs[dst_range].copy_from_slice(&src_funcrefs[src_range]); 922 } 923 TableElementType::GcRef => { 924 assert_eq!( 925 dst_range.end - dst_range.start, 926 src_range.end - src_range.start 927 ); 928 assert!(dst_range.end <= dst_table.gc_refs().len()); 929 assert!(src_range.end <= src_table.gc_refs().len()); 930 let gc_store = gc_store.unwrap(); 931 for (dst, src) in dst_range.zip(src_range) { 932 gc_store.write_gc_ref( 933 &mut dst_table.gc_refs_mut()[dst], 934 src_table.gc_refs()[src].as_ref(), 935 ); 936 } 937 } 938 } 939 } 940 941 fn copy_elements_within( 942 &mut self, 943 gc_store: Option<&mut GcStore>, 944 dst_range: Range<usize>, 945 src_range: Range<usize>, 946 ) { 947 assert_eq!( 948 dst_range.end - dst_range.start, 949 src_range.end - src_range.start 950 ); 951 952 // This is a no-op. 953 if src_range.start == dst_range.start { 954 return; 955 } 956 957 let ty = self.element_type(); 958 match ty { 959 TableElementType::Func => { 960 // `funcref` are `Copy`, so just do a memmove 961 let (funcrefs, _lazy_init) = self.funcrefs_mut(); 962 funcrefs.copy_within(src_range, dst_range.start); 963 } 964 TableElementType::GcRef => { 965 let gc_store = gc_store.unwrap(); 966 967 // We need to clone each `externref` while handling overlapping 968 // ranges 969 let elements = self.gc_refs_mut(); 970 if dst_range.start < src_range.start { 971 for (d, s) in dst_range.zip(src_range) { 972 let (ds, ss) = elements.split_at_mut(s); 973 let dst = &mut ds[d]; 974 let src = ss[0].as_ref(); 975 gc_store.write_gc_ref(dst, src); 976 } 977 } else { 978 for (s, d) in src_range.rev().zip(dst_range.rev()) { 979 let (ss, ds) = elements.split_at_mut(d); 980 let dst = &mut ds[0]; 981 let src = ss[s].as_ref(); 982 gc_store.write_gc_ref(dst, src); 983 } 984 } 985 } 986 } 987 } 988 } 989 990 // The default table representation is an empty funcref table that cannot grow. 991 impl Default for Table { 992 fn default() -> Self { 993 Self::from(StaticFuncTable { 994 data: SendSyncPtr::new(NonNull::from(&mut [])), 995 size: 0, 996 lazy_init: false, 997 }) 998 } 999 } 1000