1 use super::{truncate_i32_to_i8, truncate_i32_to_i16}; 2 use crate::{ 3 AnyRef, ExnRef, ExternRef, Func, HeapType, RootedGcRefImpl, StorageType, Val, ValType, 4 prelude::*, 5 runtime::vm::{GcHeap, GcStore, VMGcRef}, 6 store::{AutoAssertNoGc, StoreOpaque}, 7 vm::{FuncRefTableId, SendSyncPtr}, 8 }; 9 use core::fmt; 10 use wasmtime_environ::{GcArrayLayout, VMGcKind}; 11 12 /// A `VMGcRef` that we know points to a `array`. 13 /// 14 /// Create a `VMArrayRef` via `VMGcRef::into_arrayref` and 15 /// `VMGcRef::as_arrayref`, or their untyped equivalents 16 /// `VMGcRef::into_arrayref_unchecked` and `VMGcRef::as_arrayref_unchecked`. 17 /// 18 /// Note: This is not a `TypedGcRef<_>` because each collector can have a 19 /// different concrete representation of `arrayref` that they allocate inside 20 /// their heaps. 21 #[derive(Debug, PartialEq, Eq, Hash)] 22 #[repr(transparent)] 23 pub struct VMArrayRef(VMGcRef); 24 25 impl fmt::Pointer for VMArrayRef { 26 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 27 fmt::Pointer::fmt(&self.0, f) 28 } 29 } 30 31 impl From<VMArrayRef> for VMGcRef { 32 #[inline] 33 fn from(x: VMArrayRef) -> Self { 34 x.0 35 } 36 } 37 38 impl VMGcRef { 39 /// Is this `VMGcRef` pointing to a `array`? 40 pub fn is_arrayref(&self, gc_heap: &(impl GcHeap + ?Sized)) -> bool { 41 if self.is_i31() { 42 return false; 43 } 44 45 let header = gc_heap.header(&self); 46 header.kind().matches(VMGcKind::ArrayRef) 47 } 48 49 /// Create a new `VMArrayRef` from the given `gc_ref`. 50 /// 51 /// If this is not a GC reference to an `arrayref`, `Err(self)` is 52 /// returned. 53 pub fn into_arrayref(self, gc_heap: &(impl GcHeap + ?Sized)) -> Result<VMArrayRef, VMGcRef> { 54 if self.is_arrayref(gc_heap) { 55 Ok(self.into_arrayref_unchecked()) 56 } else { 57 Err(self) 58 } 59 } 60 61 /// Create a new `VMArrayRef` from `self` without actually checking that 62 /// `self` is an `arrayref`. 63 /// 64 /// This method does not check that `self` is actually an `arrayref`, but 65 /// it should be. Failure to uphold this invariant is memory safe but will 66 /// result in general incorrectness down the line such as panics or wrong 67 /// results. 68 #[inline] 69 pub fn into_arrayref_unchecked(self) -> VMArrayRef { 70 debug_assert!(!self.is_i31()); 71 VMArrayRef(self) 72 } 73 74 /// Get this GC reference as an `arrayref` reference, if it actually is an 75 /// `arrayref` reference. 76 pub fn as_arrayref(&self, gc_heap: &(impl GcHeap + ?Sized)) -> Option<&VMArrayRef> { 77 if self.is_arrayref(gc_heap) { 78 Some(self.as_arrayref_unchecked()) 79 } else { 80 None 81 } 82 } 83 84 /// Get this GC reference as an `arrayref` reference without checking if it 85 /// actually is an `arrayref` reference. 86 /// 87 /// Calling this method on a non-`arrayref` reference is memory safe, but 88 /// will lead to general incorrectness like panics and wrong results. 89 pub fn as_arrayref_unchecked(&self) -> &VMArrayRef { 90 debug_assert!(!self.is_i31()); 91 let ptr = self as *const VMGcRef; 92 let ret = unsafe { &*ptr.cast() }; 93 assert!(matches!(ret, VMArrayRef(VMGcRef { .. }))); 94 ret 95 } 96 } 97 98 impl VMArrayRef { 99 /// Get the underlying `VMGcRef`. 100 pub fn as_gc_ref(&self) -> &VMGcRef { 101 &self.0 102 } 103 104 /// Clone this `VMArrayRef`, running any GC barriers as necessary. 105 pub fn clone(&self, gc_store: &mut GcStore) -> Self { 106 Self(gc_store.clone_gc_ref(&self.0)) 107 } 108 109 /// Explicitly drop this `arrayref`, running GC drop barriers as necessary. 110 pub fn drop(self, gc_store: &mut GcStore) { 111 gc_store.drop_gc_ref(self.0); 112 } 113 114 /// Copy this `VMArrayRef` without running the GC's clone barriers. 115 /// 116 /// Prefer calling `clone(&mut GcStore)` instead! This is mostly an internal 117 /// escape hatch for collector implementations. 118 /// 119 /// Failure to run GC barriers when they would otherwise be necessary can 120 /// lead to leaks, panics, and wrong results. It cannot lead to memory 121 /// unsafety, however. 122 pub fn unchecked_copy(&self) -> Self { 123 Self(self.0.unchecked_copy()) 124 } 125 126 /// Get the length of this array. 127 pub fn len(&self, store: &StoreOpaque) -> u32 { 128 store.unwrap_gc_store().array_len(self) 129 } 130 131 /// Read an element of the given `StorageType` into a `Val`. 132 /// 133 /// `i8` and `i16` fields are zero-extended into `Val::I32(_)`s. 134 /// 135 /// Does not check that this array's elements are actually of type 136 /// `ty`. That is the caller's responsibility. Failure to do so is memory 137 /// safe, but will lead to general incorrectness such as panics and wrong 138 /// results. 139 /// 140 /// Panics on out-of-bounds accesses. 141 pub fn read_elem( 142 &self, 143 store: &mut AutoAssertNoGc, 144 layout: &GcArrayLayout, 145 ty: &StorageType, 146 index: u32, 147 ) -> Val { 148 let offset = layout.elem_offset(index); 149 let data = store.unwrap_gc_store_mut().gc_object_data(self.as_gc_ref()); 150 match ty { 151 StorageType::I8 => Val::I32(data.read_u8(offset).into()), 152 StorageType::I16 => Val::I32(data.read_u16(offset).into()), 153 StorageType::ValType(ValType::I32) => Val::I32(data.read_i32(offset)), 154 StorageType::ValType(ValType::I64) => Val::I64(data.read_i64(offset)), 155 StorageType::ValType(ValType::F32) => Val::F32(data.read_u32(offset)), 156 StorageType::ValType(ValType::F64) => Val::F64(data.read_u64(offset)), 157 StorageType::ValType(ValType::V128) => Val::V128(data.read_v128(offset)), 158 StorageType::ValType(ValType::Ref(r)) => match r.heap_type().top() { 159 HeapType::Extern => { 160 let raw = data.read_u32(offset); 161 Val::ExternRef(ExternRef::_from_raw(store, raw)) 162 } 163 HeapType::Any => { 164 let raw = data.read_u32(offset); 165 Val::AnyRef(AnyRef::_from_raw(store, raw)) 166 } 167 HeapType::Exn => { 168 let raw = data.read_u32(offset); 169 Val::ExnRef(ExnRef::_from_raw(store, raw)) 170 } 171 HeapType::Func => { 172 let func_ref_id = data.read_u32(offset); 173 let func_ref_id = FuncRefTableId::from_raw(func_ref_id); 174 let func_ref = store 175 .unwrap_gc_store() 176 .func_ref_table 177 .get_untyped(func_ref_id); 178 Val::FuncRef(unsafe { 179 func_ref.map(|p| Func::from_vm_func_ref(store.id(), p.as_non_null())) 180 }) 181 } 182 otherwise => unreachable!("not a top type: {otherwise:?}"), 183 }, 184 } 185 } 186 187 /// Write the given value into this array at the given offset. 188 /// 189 /// Returns an error if `val` is a GC reference that has since been 190 /// unrooted. 191 /// 192 /// Does not check that `val` matches `ty`, nor that the field is actually 193 /// of type `ty`. Checking those things is the caller's responsibility. 194 /// Failure to do so is memory safe, but will lead to general incorrectness 195 /// such as panics and wrong results. 196 /// 197 /// Panics on out-of-bounds accesses. 198 pub fn write_elem( 199 &self, 200 store: &mut AutoAssertNoGc, 201 layout: &GcArrayLayout, 202 ty: &StorageType, 203 index: u32, 204 val: Val, 205 ) -> Result<()> { 206 debug_assert!(val._matches_ty(&store, &ty.unpack())?); 207 208 let offset = layout.elem_offset(index); 209 let data = store.unwrap_gc_store_mut().gc_object_data(self.as_gc_ref()); 210 match val { 211 Val::I32(i) if ty.is_i8() => data.write_i8(offset, truncate_i32_to_i8(i)), 212 Val::I32(i) if ty.is_i16() => data.write_i16(offset, truncate_i32_to_i16(i)), 213 Val::I32(i) => data.write_i32(offset, i), 214 Val::I64(i) => data.write_i64(offset, i), 215 Val::F32(f) => data.write_u32(offset, f), 216 Val::F64(f) => data.write_u64(offset, f), 217 Val::V128(v) => data.write_v128(offset, v), 218 219 // For GC-managed references, we need to take care to run the 220 // appropriate barriers, even when we are writing null references 221 // into the array. 222 // 223 // POD-read the old value into a local copy, run the GC write 224 // barrier on that local copy, and then POD-write the updated 225 // value back into the array. This avoids transmuting the inner 226 // data, which would probably be fine, but this approach is 227 // Obviously Correct and should get us by for now. If LLVM isn't 228 // able to elide some of these unnecessary copies, and this 229 // method is ever hot enough, we can always come back and clean 230 // it up in the future. 231 Val::ExternRef(e) => { 232 let raw = data.read_u32(offset); 233 let mut gc_ref = VMGcRef::from_raw_u32(raw); 234 let e = match e { 235 Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()), 236 None => None, 237 }; 238 store.gc_store_mut()?.write_gc_ref(&mut gc_ref, e.as_ref()); 239 let data = store.gc_store_mut()?.gc_object_data(self.as_gc_ref()); 240 data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32())); 241 } 242 Val::AnyRef(a) => { 243 let raw = data.read_u32(offset); 244 let mut gc_ref = VMGcRef::from_raw_u32(raw); 245 let a = match a { 246 Some(a) => Some(a.try_gc_ref(store)?.unchecked_copy()), 247 None => None, 248 }; 249 store.gc_store_mut()?.write_gc_ref(&mut gc_ref, a.as_ref()); 250 let data = store.gc_store_mut()?.gc_object_data(self.as_gc_ref()); 251 data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32())); 252 } 253 Val::ExnRef(e) => { 254 let raw = data.read_u32(offset); 255 let mut gc_ref = VMGcRef::from_raw_u32(raw); 256 let e = match e { 257 Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()), 258 None => None, 259 }; 260 store.gc_store_mut()?.write_gc_ref(&mut gc_ref, e.as_ref()); 261 let data = store.gc_store_mut()?.gc_object_data(self.as_gc_ref()); 262 data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32())); 263 } 264 265 Val::FuncRef(f) => { 266 let func_ref = match f { 267 Some(f) => Some(SendSyncPtr::new(f.vm_func_ref(store))), 268 None => None, 269 }; 270 let id = unsafe { store.gc_store_mut()?.func_ref_table.intern(func_ref) }; 271 store 272 .gc_store_mut()? 273 .gc_object_data(self.as_gc_ref()) 274 .write_u32(offset, id.into_raw()); 275 } 276 } 277 Ok(()) 278 } 279 280 /// Initialize an element in this arrayref that is currently uninitialized. 281 /// 282 /// The difference between this method and `write_elem` is that GC barriers 283 /// are handled differently. When overwriting an initialized element (aka 284 /// `write_elem`) we need to call the full write GC write barrier, which 285 /// logically drops the old GC reference and clones the new GC 286 /// reference. When we are initializing an element for the first time, there 287 /// is no old GC reference that is being overwritten and which we need to 288 /// drop, so we only need to clone the new GC reference. 289 /// 290 /// Calling this method on a arrayref that has already had the associated 291 /// element initialized will result in GC bugs. These are memory safe but 292 /// will lead to generally incorrect behavior such as panics, leaks, and 293 /// incorrect results. 294 /// 295 /// Does not check that `val` matches `ty`, nor that the field is actually 296 /// of type `ty`. Checking those things is the caller's responsibility. 297 /// Failure to do so is memory safe, but will lead to general incorrectness 298 /// such as panics and wrong results. 299 /// 300 /// Returns an error if `val` is a GC reference that has since been 301 /// unrooted. 302 /// 303 /// Panics on out-of-bounds accesses. 304 pub fn initialize_elem( 305 &self, 306 store: &mut AutoAssertNoGc, 307 layout: &GcArrayLayout, 308 ty: &StorageType, 309 index: u32, 310 val: Val, 311 ) -> Result<()> { 312 debug_assert!(val._matches_ty(&store, &ty.unpack())?); 313 let offset = layout.elem_offset(index); 314 match val { 315 Val::I32(i) if ty.is_i8() => store 316 .gc_store_mut()? 317 .gc_object_data(self.as_gc_ref()) 318 .write_i8(offset, truncate_i32_to_i8(i)), 319 Val::I32(i) if ty.is_i16() => store 320 .gc_store_mut()? 321 .gc_object_data(self.as_gc_ref()) 322 .write_i16(offset, truncate_i32_to_i16(i)), 323 Val::I32(i) => store 324 .gc_store_mut()? 325 .gc_object_data(self.as_gc_ref()) 326 .write_i32(offset, i), 327 Val::I64(i) => store 328 .gc_store_mut()? 329 .gc_object_data(self.as_gc_ref()) 330 .write_i64(offset, i), 331 Val::F32(f) => store 332 .gc_store_mut()? 333 .gc_object_data(self.as_gc_ref()) 334 .write_u32(offset, f), 335 Val::F64(f) => store 336 .gc_store_mut()? 337 .gc_object_data(self.as_gc_ref()) 338 .write_u64(offset, f), 339 Val::V128(v) => store 340 .gc_store_mut()? 341 .gc_object_data(self.as_gc_ref()) 342 .write_v128(offset, v), 343 344 // NB: We don't need to do a write barrier when initializing a 345 // field, because there is nothing being overwritten. Therefore, we 346 // just the clone barrier. 347 Val::ExternRef(x) => { 348 let x = match x { 349 None => 0, 350 Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(), 351 }; 352 store 353 .gc_store_mut()? 354 .gc_object_data(self.as_gc_ref()) 355 .write_u32(offset, x); 356 } 357 Val::AnyRef(x) => { 358 let x = match x { 359 None => 0, 360 Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(), 361 }; 362 store 363 .gc_store_mut()? 364 .gc_object_data(self.as_gc_ref()) 365 .write_u32(offset, x); 366 } 367 Val::ExnRef(x) => { 368 let x = match x { 369 None => 0, 370 Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(), 371 }; 372 store 373 .gc_store_mut()? 374 .gc_object_data(self.as_gc_ref()) 375 .write_u32(offset, x); 376 } 377 378 Val::FuncRef(f) => { 379 let func_ref = match f { 380 Some(f) => Some(SendSyncPtr::new(f.vm_func_ref(store))), 381 None => None, 382 }; 383 let id = unsafe { store.gc_store_mut()?.func_ref_table.intern(func_ref) }; 384 store 385 .gc_store_mut()? 386 .gc_object_data(self.as_gc_ref()) 387 .write_u32(offset, id.into_raw()); 388 } 389 } 390 Ok(()) 391 } 392 } 393