1 use crate::{ 2 prelude::*, 3 runtime::vm::{GcHeap, GcStore, VMGcRef}, 4 store::{AutoAssertNoGc, StoreOpaque}, 5 vm::GcArrayLayout, 6 AnyRef, ExternRef, HeapType, RootedGcRefImpl, StorageType, Val, ValType, 7 }; 8 use core::fmt; 9 use wasmtime_environ::VMGcKind; 10 11 /// A `VMGcRef` that we know points to a `array`. 12 /// 13 /// Create a `VMArrayRef` via `VMGcRef::into_arrayref` and 14 /// `VMGcRef::as_arrayref`, or their untyped equivalents 15 /// `VMGcRef::into_arrayref_unchecked` and `VMGcRef::as_arrayref_unchecked`. 16 /// 17 /// Note: This is not a `TypedGcRef<_>` because each collector can have a 18 /// different concrete representation of `arrayref` that they allocate inside 19 /// their heaps. 20 #[derive(Debug, PartialEq, Eq, Hash)] 21 #[repr(transparent)] 22 pub struct VMArrayRef(VMGcRef); 23 24 impl fmt::Pointer for VMArrayRef { 25 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 26 fmt::Pointer::fmt(&self.0, f) 27 } 28 } 29 30 impl From<VMArrayRef> for VMGcRef { 31 #[inline] 32 fn from(x: VMArrayRef) -> Self { 33 x.0 34 } 35 } 36 37 impl VMGcRef { 38 /// Is this `VMGcRef` pointing to a `array`? 39 pub fn is_arrayref(&self, gc_heap: &(impl GcHeap + ?Sized)) -> bool { 40 if self.is_i31() { 41 return false; 42 } 43 44 let header = gc_heap.header(&self); 45 header.kind().matches(VMGcKind::ArrayRef) 46 } 47 48 /// Create a new `VMArrayRef` from the given `gc_ref`. 49 /// 50 /// If this is not a GC reference to an `arrayref`, `Err(self)` is 51 /// returned. 52 pub fn into_arrayref(self, gc_heap: &impl GcHeap) -> Result<VMArrayRef, VMGcRef> { 53 if self.is_arrayref(gc_heap) { 54 Ok(self.into_arrayref_unchecked()) 55 } else { 56 Err(self) 57 } 58 } 59 60 /// Create a new `VMArrayRef` from `self` without actually checking that 61 /// `self` is an `arrayref`. 62 /// 63 /// This method does not check that `self` is actually an `arrayref`, but 64 /// it should be. Failure to uphold this invariant is memory safe but will 65 /// result in general incorrectness down the line such as panics or wrong 66 /// results. 67 #[inline] 68 pub fn into_arrayref_unchecked(self) -> VMArrayRef { 69 debug_assert!(!self.is_i31()); 70 VMArrayRef(self) 71 } 72 73 /// Get this GC reference as an `arrayref` reference, if it actually is an 74 /// `arrayref` reference. 75 pub fn as_arrayref(&self, gc_heap: &(impl GcHeap + ?Sized)) -> Option<&VMArrayRef> { 76 if self.is_arrayref(gc_heap) { 77 Some(self.as_arrayref_unchecked()) 78 } else { 79 None 80 } 81 } 82 83 /// Get this GC reference as an `arrayref` reference without checking if it 84 /// actually is an `arrayref` reference. 85 /// 86 /// Calling this method on a non-`arrayref` reference is memory safe, but 87 /// will lead to general incorrectness like panics and wrong results. 88 pub fn as_arrayref_unchecked(&self) -> &VMArrayRef { 89 debug_assert!(!self.is_i31()); 90 let ptr = self as *const VMGcRef; 91 let ret = unsafe { &*ptr.cast() }; 92 assert!(matches!(ret, VMArrayRef(VMGcRef { .. }))); 93 ret 94 } 95 } 96 97 impl VMArrayRef { 98 /// Get the underlying `VMGcRef`. 99 pub fn as_gc_ref(&self) -> &VMGcRef { 100 &self.0 101 } 102 103 /// Clone this `VMArrayRef`, running any GC barriers as necessary. 104 pub fn clone(&self, gc_store: &mut GcStore) -> Self { 105 Self(gc_store.clone_gc_ref(&self.0)) 106 } 107 108 /// Explicitly drop this `arrayref`, running GC drop barriers as necessary. 109 pub fn drop(self, gc_store: &mut GcStore) { 110 gc_store.drop_gc_ref(self.0); 111 } 112 113 /// Copy this `VMArrayRef` without running the GC's clone barriers. 114 /// 115 /// Prefer calling `clone(&mut GcStore)` instead! This is mostly an internal 116 /// escape hatch for collector implementations. 117 /// 118 /// Failure to run GC barriers when they would otherwise be necessary can 119 /// lead to leaks, panics, and wrong results. It cannot lead to memory 120 /// unsafety, however. 121 pub fn unchecked_copy(&self) -> Self { 122 Self(self.0.unchecked_copy()) 123 } 124 125 /// Get the length of this array. 126 pub fn len(&self, store: &StoreOpaque) -> u32 { 127 store.unwrap_gc_store().array_len(self) 128 } 129 130 /// Read an element of the given `StorageType` into a `Val`. 131 /// 132 /// `i8` and `i16` fields are zero-extended into `Val::I32(_)`s. 133 /// 134 /// Does not check that this array's elements are actually of type 135 /// `ty`. That is the caller's responsibility. Failure to do so is memory 136 /// safe, but will lead to general incorrectness such as panics and wrong 137 /// results. 138 /// 139 /// Panics on out-of-bounds accesses. 140 pub fn read_elem( 141 &self, 142 store: &mut AutoAssertNoGc, 143 layout: &GcArrayLayout, 144 ty: &StorageType, 145 index: u32, 146 ) -> Val { 147 let offset = layout.elems_offset + index * ty.byte_size_in_gc_heap(); 148 let data = store.unwrap_gc_store_mut().gc_object_data(self.as_gc_ref()); 149 match ty { 150 StorageType::I8 => Val::I32(data.read_u8(offset).into()), 151 StorageType::I16 => Val::I32(data.read_u16(offset).into()), 152 StorageType::ValType(ValType::I32) => Val::I32(data.read_i32(offset)), 153 StorageType::ValType(ValType::I64) => Val::I64(data.read_i64(offset)), 154 StorageType::ValType(ValType::F32) => Val::F32(data.read_u32(offset)), 155 StorageType::ValType(ValType::F64) => Val::F64(data.read_u64(offset)), 156 StorageType::ValType(ValType::V128) => Val::V128(data.read_v128(offset)), 157 StorageType::ValType(ValType::Ref(r)) => match r.heap_type().top() { 158 HeapType::Extern => { 159 let raw = data.read_u32(offset); 160 Val::ExternRef(ExternRef::_from_raw(store, raw)) 161 } 162 HeapType::Any => { 163 let raw = data.read_u32(offset); 164 Val::AnyRef(AnyRef::_from_raw(store, raw)) 165 } 166 HeapType::Func => todo!("funcrefs inside gc objects not yet implemented"), 167 otherwise => unreachable!("not a top type: {otherwise:?}"), 168 }, 169 } 170 } 171 172 /// Write the given value into this array at the given offset. 173 /// 174 /// Returns an error if `val` is a GC reference that has since been 175 /// unrooted. 176 /// 177 /// Does not check that `val` matches `ty`, nor that the field is actually 178 /// of type `ty`. Checking those things is the caller's responsibility. 179 /// Failure to do so is memory safe, but will lead to general incorrectness 180 /// such as panics and wrong results. 181 /// 182 /// Panics on out-of-bounds accesses. 183 pub fn write_elem( 184 &self, 185 store: &mut AutoAssertNoGc, 186 layout: &GcArrayLayout, 187 ty: &StorageType, 188 index: u32, 189 val: Val, 190 ) -> Result<()> { 191 debug_assert!(val._matches_ty(&store, &ty.unpack())?); 192 193 let offset = layout.elem_offset(index, ty.byte_size_in_gc_heap()); 194 let mut data = store.unwrap_gc_store_mut().gc_object_data(self.as_gc_ref()); 195 match val { 196 Val::I32(i) if ty.is_i8() => data.write_i8(offset, i as i8), 197 Val::I32(i) if ty.is_i16() => data.write_i16(offset, i as i16), 198 Val::I32(i) => data.write_i32(offset, i), 199 Val::I64(i) => data.write_i64(offset, i), 200 Val::F32(f) => data.write_u32(offset, f), 201 Val::F64(f) => data.write_u64(offset, f), 202 Val::V128(v) => data.write_v128(offset, v), 203 204 // For GC-managed references, we need to take care to run the 205 // appropriate barriers, even when we are writing null references 206 // into the array. 207 // 208 // POD-read the old value into a local copy, run the GC write 209 // barrier on that local copy, and then POD-write the updated 210 // value back into the array. This avoids transmuting the inner 211 // data, which would probably be fine, but this approach is 212 // Obviously Correct and should get us by for now. If LLVM isn't 213 // able to elide some of these unnecessary copies, and this 214 // method is ever hot enough, we can always come back and clean 215 // it up in the future. 216 Val::ExternRef(e) => { 217 let raw = data.read_u32(offset); 218 let mut gc_ref = VMGcRef::from_raw_u32(raw); 219 let e = match e { 220 Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()), 221 None => None, 222 }; 223 store.gc_store_mut()?.write_gc_ref(&mut gc_ref, e.as_ref()); 224 let mut data = store.gc_store_mut()?.gc_object_data(self.as_gc_ref()); 225 data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32())); 226 } 227 Val::AnyRef(a) => { 228 let raw = data.read_u32(offset); 229 let mut gc_ref = VMGcRef::from_raw_u32(raw); 230 let a = match a { 231 Some(a) => Some(a.try_gc_ref(store)?.unchecked_copy()), 232 None => None, 233 }; 234 store.gc_store_mut()?.write_gc_ref(&mut gc_ref, a.as_ref()); 235 let mut data = store.gc_store_mut()?.gc_object_data(self.as_gc_ref()); 236 data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32())); 237 } 238 239 Val::FuncRef(_) => todo!("funcrefs inside gc objects not yet implemented"), 240 } 241 Ok(()) 242 } 243 244 /// Initialize an element in this arrayref that is currently uninitialized. 245 /// 246 /// The difference between this method and `write_elem` is that GC barriers 247 /// are handled differently. When overwriting an initialized element (aka 248 /// `write_elem`) we need to call the full write GC write barrier, which 249 /// logically drops the old GC reference and clones the new GC 250 /// reference. When we are initializing an element for the first time, there 251 /// is no old GC reference that is being overwritten and which we need to 252 /// drop, so we only need to clone the new GC reference. 253 /// 254 /// Calling this method on a arrayref that has already had the associated 255 /// element initialized will result in GC bugs. These are memory safe but 256 /// will lead to generally incorrect behavior such as panics, leaks, and 257 /// incorrect results. 258 /// 259 /// Does not check that `val` matches `ty`, nor that the field is actually 260 /// of type `ty`. Checking those things is the caller's responsibility. 261 /// Failure to do so is memory safe, but will lead to general incorrectness 262 /// such as panics and wrong results. 263 /// 264 /// Returns an error if `val` is a GC reference that has since been 265 /// unrooted. 266 /// 267 /// Panics on out-of-bounds accesses. 268 pub fn initialize_elem( 269 &self, 270 store: &mut AutoAssertNoGc, 271 layout: &GcArrayLayout, 272 ty: &StorageType, 273 index: u32, 274 val: Val, 275 ) -> Result<()> { 276 debug_assert!(val._matches_ty(&store, &ty.unpack())?); 277 let offset = layout.elem_offset(index, ty.byte_size_in_gc_heap()); 278 match val { 279 Val::I32(i) if ty.is_i8() => store 280 .gc_store_mut()? 281 .gc_object_data(self.as_gc_ref()) 282 .write_i8(offset, i as i8), 283 Val::I32(i) if ty.is_i16() => store 284 .gc_store_mut()? 285 .gc_object_data(self.as_gc_ref()) 286 .write_i16(offset, i as i16), 287 Val::I32(i) => store 288 .gc_store_mut()? 289 .gc_object_data(self.as_gc_ref()) 290 .write_i32(offset, i), 291 Val::I64(i) => store 292 .gc_store_mut()? 293 .gc_object_data(self.as_gc_ref()) 294 .write_i64(offset, i), 295 Val::F32(f) => store 296 .gc_store_mut()? 297 .gc_object_data(self.as_gc_ref()) 298 .write_u32(offset, f), 299 Val::F64(f) => store 300 .gc_store_mut()? 301 .gc_object_data(self.as_gc_ref()) 302 .write_u64(offset, f), 303 Val::V128(v) => store 304 .gc_store_mut()? 305 .gc_object_data(self.as_gc_ref()) 306 .write_v128(offset, v), 307 308 // NB: We don't need to do a write barrier when initializing a 309 // field, because there is nothing being overwritten. Therefore, we 310 // just the clone barrier. 311 Val::ExternRef(x) => { 312 let x = match x { 313 None => 0, 314 Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(), 315 }; 316 store 317 .gc_store_mut()? 318 .gc_object_data(self.as_gc_ref()) 319 .write_u32(offset, x); 320 } 321 Val::AnyRef(x) => { 322 let x = match x { 323 None => 0, 324 Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(), 325 }; 326 store 327 .gc_store_mut()? 328 .gc_object_data(self.as_gc_ref()) 329 .write_u32(offset, x); 330 } 331 332 Val::FuncRef(_) => { 333 // TODO: we can't trust the GC heap, which means we can't read 334 // native VMFuncRef pointers out of it and trust them. That 335 // means we need to do the same side table kind of thing we do 336 // with `externref` host data here. This isn't implemented yet. 337 todo!("funcrefs in GC objects") 338 } 339 } 340 Ok(()) 341 } 342 } 343