1 use super::{ 2 ABI, RegAlloc, 3 abi::Aarch64ABI, 4 address::Address, 5 asm::{Assembler, PatchableAddToReg}, 6 regs::{self, scratch_fpr_bitset, scratch_gpr_bitset}, 7 }; 8 use crate::{ 9 Result, 10 abi::{self, align_to, calculate_frame_adjustment, local::LocalSlot, vmctx}, 11 bail, 12 codegen::{CodeGenContext, CodeGenError, Emission, FuncEnv, ptr_type_from_ptr_size}, 13 format_err, 14 isa::{ 15 CallingConvention, 16 aarch64::abi::SHADOW_STACK_POINTER_SLOT_SIZE, 17 reg::{Reg, WritableReg, writable}, 18 }, 19 masm::{ 20 CalleeKind, DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm, 21 Imm as I, IntCmpKind, IntScratch, LoadKind, MacroAssembler as Masm, MulWideKind, 22 OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, RoundingMode, SPOffset, Scratch, 23 ScratchType, ShiftKind, SplatKind, StackSlot, StoreKind, TRUSTED_FLAGS, TrapCode, 24 TruncKind, UNTRUSTED_FLAGS, V128AbsKind, V128AddKind, V128ConvertKind, V128ExtAddKind, 25 V128ExtMulKind, V128ExtendKind, V128MaxKind, V128MinKind, V128MulKind, V128NarrowKind, 26 V128NegKind, V128SubKind, V128TruncKind, VectorCompareKind, VectorEqualityKind, Zero, 27 }, 28 stack::TypedReg, 29 }; 30 use cranelift_codegen::{ 31 Final, MachBufferFinalized, MachLabel, 32 binemit::CodeOffset, 33 ir::{MemFlags, RelSourceLoc, SourceLoc, types}, 34 isa::aarch64, 35 isa::aarch64::inst::{ 36 self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, VectorSize, 37 }, 38 settings, 39 }; 40 use regalloc2::RegClass; 41 use wasmtime_environ::{PtrSize, WasmValType}; 42 43 /// Aarch64 MacroAssembler. 44 pub(crate) struct MacroAssembler { 45 /// This value represents the maximum stack size seen while compiling the 46 /// function. While the function is still being compiled its value will not 47 /// be valid (the stack will grow and shrink as space is reserved and freed 48 /// during compilation), but once all instructions have been seen this value 49 /// will be the maximum stack usage seen. 50 sp_max: u32, 51 52 /// Add-with-immediate patchable instruction sequence used to add the 53 /// constant stack max to a register. 54 stack_max_use_add: Option<PatchableAddToReg>, 55 56 /// Low level assembler. 57 asm: Assembler, 58 /// Stack pointer offset. 59 sp_offset: u32, 60 /// The target pointer size. 61 ptr_size: OperandSize, 62 /// Scratch register scope. 63 scratch_scope: RegAlloc, 64 } 65 66 impl MacroAssembler { 67 /// Create an Aarch64 MacroAssembler. new( ptr_size: impl PtrSize, shared_flags: settings::Flags, isa_flags: aarch64::settings::Flags, ) -> Result<Self>68 pub fn new( 69 ptr_size: impl PtrSize, 70 shared_flags: settings::Flags, 71 isa_flags: aarch64::settings::Flags, 72 ) -> Result<Self> { 73 Ok(Self { 74 sp_max: 0, 75 stack_max_use_add: None, 76 asm: Assembler::new(shared_flags, isa_flags), 77 sp_offset: 0u32, 78 ptr_size: ptr_type_from_ptr_size(ptr_size.size()).try_into()?, 79 scratch_scope: RegAlloc::from(scratch_gpr_bitset(), scratch_fpr_bitset()), 80 }) 81 } 82 83 /// Add the maximum stack used to a register, recording an obligation to update the 84 /// add-with-immediate instruction emitted to use the real stack max when the masm is being 85 /// finalized. add_stack_max(&mut self, reg: WritableReg, tmp: WritableReg)86 fn add_stack_max(&mut self, reg: WritableReg, tmp: WritableReg) { 87 assert!(self.stack_max_use_add.is_none()); 88 let patch = PatchableAddToReg::new(reg, tmp, self.asm.buffer_mut()); 89 self.stack_max_use_add.replace(patch); 90 } 91 92 /// Ensures that the stack pointer remains 16-byte aligned for the duration 93 /// of the provided function. This alignment is necessary for AArch64 94 /// compliance, particularly for signal handlers that may be invoked 95 /// during execution. While the compiler doesn't directly use the stack 96 /// pointer for memory addressing, maintaining this alignment is crucial 97 /// to prevent issues when handling signals. with_aligned_sp<F, T>(&mut self, f: F) -> Result<T> where F: FnOnce(&mut Self) -> Result<T>,98 pub fn with_aligned_sp<F, T>(&mut self, f: F) -> Result<T> 99 where 100 F: FnOnce(&mut Self) -> Result<T>, 101 { 102 let mut aligned = false; 103 let alignment: u32 = <Aarch64ABI as ABI>::call_stack_align().into(); 104 let addend: u32 = <Aarch64ABI as ABI>::initial_frame_size().into(); 105 let delta = calculate_frame_adjustment(self.sp_offset()?.as_u32(), addend, alignment); 106 if delta != 0 { 107 self.sub( 108 writable!(regs::sp()), 109 // Since we don't need to synchronize the shadow stack pointer 110 // when freeing stack space [^1], the stack pointer may become 111 // out of sync with the primary shadow stack pointer. Therefore, 112 // we use the shadow stack pointer as the reference for 113 // calculating any alignment delta (self.sp_offset). 114 // 115 // [1]: This approach avoids an unnecessary move instruction and 116 // maintains the invariant of not accessing memory below the 117 // current stack pointer, preventing issues with signal handlers 118 // and interrupts. 119 regs::shadow_sp(), 120 RegImm::i32(delta as i32), 121 OperandSize::S64, 122 )?; 123 124 aligned = true; 125 } 126 127 let res = f(self)?; 128 129 if aligned { 130 self.move_shadow_sp_to_sp(); 131 } 132 133 Ok(res) 134 } 135 } 136 137 impl Masm for MacroAssembler { 138 type Address = Address; 139 type Ptr = u8; 140 type ABI = Aarch64ABI; 141 frame_setup(&mut self) -> Result<()>142 fn frame_setup(&mut self) -> Result<()> { 143 let lr = regs::lr(); 144 let fp = regs::fp(); 145 let sp = regs::sp(); 146 147 let offset = SImm7Scaled::maybe_from_i64(-16, types::I64) 148 .expect("Frame pointer offset of -16 is valid for pair addressing"); 149 let addr = Address::pre_indexed_from_sp_for_pair(offset); 150 self.asm.stp(fp, lr, addr.to_pair_addressing_mode()); 151 self.asm.mov_rr(sp, writable!(fp), OperandSize::S64); 152 153 let offset = SImm9::maybe_from_i64(-(SHADOW_STACK_POINTER_SLOT_SIZE as i64)) 154 .expect("Shadow stack pointer slot size is valid for single addressing"); 155 let addr = Address::pre_indexed_from_sp(offset); 156 addr.to_addressing_mode(self, OperandSize::S64, |masm, mem| { 157 masm.asm 158 .str(regs::shadow_sp(), mem, OperandSize::S64, TRUSTED_FLAGS); 159 Ok(()) 160 })?; 161 162 self.move_sp_to_shadow_sp(); 163 Ok(()) 164 } 165 check_stack(&mut self, vmctx: Reg) -> Result<()>166 fn check_stack(&mut self, vmctx: Reg) -> Result<()> { 167 let ptr_size_u8: u8 = self.ptr_size.bytes().try_into().unwrap(); 168 169 // The PatchableAddToReg construct on aarch64 is not a single 170 // add-immediate instruction, but a 3-instruction sequence that loads an 171 // immediate using 2 mov-immediate instructions into _another_ scratch 172 // register before adding it into the target scratch register. 173 // 174 // In other words, to make this work we use _two_ scratch registers, one 175 // to hold the limit we're calculating and one helper that's just used 176 // to load the immediate. 177 // 178 // Luckily on aarch64 we have 2 available scratch registers, ip0 and 179 // ip1. 180 // NB that this in this case, we manually allocate the scratch registers 181 // as precision when it comes to its usage is 182 183 let ptr_size = self.ptr_size; 184 self.with_aligned_sp(|masm| { 185 masm.with_scratch::<IntScratch, _>(|masm, scratch_stk_limit| { 186 masm.with_scratch::<IntScratch, _>(|masm, scratch_tmp| { 187 masm.load_ptr( 188 masm.address_at_reg(vmctx, ptr_size_u8.vmcontext_store_context().into())?, 189 scratch_stk_limit.writable(), 190 )?; 191 192 masm.load_ptr( 193 Address::offset( 194 scratch_stk_limit.inner(), 195 ptr_size_u8.vmstore_context_stack_limit().into(), 196 ), 197 scratch_stk_limit.writable(), 198 )?; 199 200 masm.add_stack_max(scratch_stk_limit.writable(), scratch_tmp.writable()); 201 202 // Aarch can only do a cmp with sp in the first operand, which means we 203 // use a less-than comparison, not a greater-than (stack grows down). 204 masm.cmp(regs::sp(), scratch_stk_limit.inner().into(), ptr_size)?; 205 masm.asm 206 .trapif(IntCmpKind::LtU.into(), TrapCode::STACK_OVERFLOW); 207 208 Ok(()) 209 }) 210 }) 211 }) 212 } 213 frame_restore(&mut self) -> Result<()>214 fn frame_restore(&mut self) -> Result<()> { 215 debug_assert_eq!(self.sp_offset, 0); 216 217 // Sync the real stack pointer with the value of the shadow stack 218 // pointer. 219 self.move_shadow_sp_to_sp(); 220 221 // Pop the shadow stack pointer. It's assumed that at this point 222 // `sp_offset` is 0 and therefore the real stack pointer should be 223 // 16-byte aligned. 224 let offset = SImm9::maybe_from_i64(SHADOW_STACK_POINTER_SLOT_SIZE as i64) 225 .expect("Shadow stack pointer slot size is valid for single addressing"); 226 let addr = Address::post_indexed_from_sp(offset); 227 addr.to_addressing_mode(self, OperandSize::S64, |masm, mem| { 228 masm.asm.uload( 229 mem, 230 writable!(regs::shadow_sp()), 231 OperandSize::S64, 232 TRUSTED_FLAGS, 233 ); 234 Ok(()) 235 })?; 236 237 // Restore the link register and frame pointer. 238 let lr = regs::lr(); 239 let fp = regs::fp(); 240 let offset = SImm7Scaled::maybe_from_i64(16, types::I64) 241 .expect("Frame pointer offset 16 is valid for pair addressing"); 242 let addr = Address::post_indexed_from_sp_for_pair(offset); 243 244 self.asm.ldp(fp, lr, addr.to_pair_addressing_mode()); 245 self.asm.ret(); 246 Ok(()) 247 } 248 reserve_stack(&mut self, bytes: u32) -> Result<()>249 fn reserve_stack(&mut self, bytes: u32) -> Result<()> { 250 if bytes == 0 { 251 return Ok(()); 252 } 253 254 let ssp = regs::shadow_sp(); 255 256 match Imm12::maybe_from_u64(bytes as u64) { 257 Some(v) => self.asm.sub_ir(v, ssp, writable!(ssp), OperandSize::S64), 258 None => { 259 self.with_scratch::<IntScratch, _>(|masm, scratch| { 260 masm.asm 261 .mov_ir(scratch.writable(), I::I64(bytes as u64), OperandSize::S64); 262 masm.asm 263 .sub_rrr(scratch.inner(), ssp, writable!(ssp), OperandSize::S64); 264 }); 265 } 266 } 267 268 // Even though we're using the shadow stack pointer to reserve stack, we 269 // must ensure that the real stack pointer reflects the stack claimed so 270 // far; we can't use stack memory below the real stack pointer as it 271 // could be clobbered by interrupts or signal handlers. 272 self.move_shadow_sp_to_sp(); 273 274 self.increment_sp(bytes); 275 Ok(()) 276 } 277 free_stack(&mut self, bytes: u32) -> Result<()>278 fn free_stack(&mut self, bytes: u32) -> Result<()> { 279 if bytes == 0 { 280 return Ok(()); 281 } 282 283 let ssp = regs::shadow_sp(); 284 match Imm12::maybe_from_u64(bytes as u64) { 285 Some(v) => self.asm.add_ir(v, ssp, writable!(ssp), OperandSize::S64), 286 None => { 287 self.with_scratch::<IntScratch, _>(|masm, scratch| { 288 masm.asm 289 .mov_ir(scratch.writable(), I::I64(bytes as u64), OperandSize::S64); 290 masm.asm 291 .add_rrr(ssp, scratch.inner(), writable!(ssp), OperandSize::S64); 292 }); 293 } 294 } 295 296 // We must ensure that the real stack pointer reflects the offset 297 // tracked by `self.sp_offset`, we use such value to calculate 298 // alignment, which is crucial for calls. 299 // 300 // As an optimization: this synchronization doesn't need to happen all 301 // the time, in theory we could ensure to sync the shadow stack pointer 302 // with the stack pointer when alignment is required, like at callsites. 303 // This is the simplest approach at the time of writing, which 304 // integrates well with the rest of the aarch64 infrastructure. 305 self.move_shadow_sp_to_sp(); 306 307 self.decrement_sp(bytes); 308 Ok(()) 309 } 310 reset_stack_pointer(&mut self, offset: SPOffset) -> Result<()>311 fn reset_stack_pointer(&mut self, offset: SPOffset) -> Result<()> { 312 self.sp_offset = offset.as_u32(); 313 Ok(()) 314 } 315 local_address(&mut self, local: &LocalSlot) -> Result<Address>316 fn local_address(&mut self, local: &LocalSlot) -> Result<Address> { 317 let (reg, offset) = local 318 .addressed_from_sp() 319 .then(|| { 320 let offset = self.sp_offset.checked_sub(local.offset).expect(&format!( 321 "Invalid local offset = {}; sp offset = {}", 322 local.offset, self.sp_offset 323 )); 324 (regs::shadow_sp(), offset) 325 }) 326 .unwrap_or((regs::fp(), local.offset)); 327 328 Ok(Address::offset(reg, offset as i64)) 329 } 330 address_from_sp(&self, offset: SPOffset) -> Result<Self::Address>331 fn address_from_sp(&self, offset: SPOffset) -> Result<Self::Address> { 332 Ok(Address::from_shadow_sp( 333 (self.sp_offset - offset.as_u32()) as i64, 334 )) 335 } 336 address_at_sp(&self, offset: SPOffset) -> Result<Self::Address>337 fn address_at_sp(&self, offset: SPOffset) -> Result<Self::Address> { 338 Ok(Address::from_shadow_sp(offset.as_u32() as i64)) 339 } 340 address_at_vmctx(&self, offset: u32) -> Result<Self::Address>341 fn address_at_vmctx(&self, offset: u32) -> Result<Self::Address> { 342 Ok(Address::offset(vmctx!(Self), offset as i64)) 343 } 344 store_ptr(&mut self, src: Reg, dst: Self::Address) -> Result<()>345 fn store_ptr(&mut self, src: Reg, dst: Self::Address) -> Result<()> { 346 self.store(src.into(), dst, self.ptr_size) 347 } 348 store(&mut self, src: RegImm, dst: Address, size: OperandSize) -> Result<()>349 fn store(&mut self, src: RegImm, dst: Address, size: OperandSize) -> Result<()> { 350 match src { 351 RegImm::Imm(v) => { 352 match v { 353 I::I32(_) | I::I64(_) => { 354 self.with_scratch::<IntScratch, _>(|masm, scratch| -> Result<()> { 355 masm.asm.mov_ir(scratch.writable(), v, v.size()); 356 dst.to_addressing_mode(masm, size, |masm, mem| { 357 masm.asm.str(scratch.inner(), mem, size, TRUSTED_FLAGS); 358 Ok(()) 359 }) 360 })?; 361 } 362 imm @ (I::F32(_) | I::F64(_)) => { 363 self.with_scratch::<FloatScratch, _>(|masm, scratch| -> Result<()> { 364 masm.asm.mov_ir(scratch.writable(), imm, imm.size()); 365 dst.to_addressing_mode(masm, size, |masm, mem| { 366 masm.asm.str(scratch.inner(), mem, size, TRUSTED_FLAGS); 367 Ok(()) 368 }) 369 })?; 370 } 371 _ => bail!(CodeGenError::unsupported_wasm_type()), 372 }; 373 Ok(()) 374 } 375 RegImm::Reg(r) => dst.to_addressing_mode(self, size, |masm, mem| { 376 masm.asm.str(r, mem, size, TRUSTED_FLAGS); 377 Ok(()) 378 }), 379 } 380 } 381 wasm_store(&mut self, src: Reg, dst: Self::Address, op_kind: StoreKind) -> Result<()>382 fn wasm_store(&mut self, src: Reg, dst: Self::Address, op_kind: StoreKind) -> Result<()> { 383 self.with_aligned_sp(|masm| match op_kind { 384 StoreKind::Operand(size) => dst.to_addressing_mode(masm, size, |masm, mem| { 385 masm.asm.str(src, mem, size, UNTRUSTED_FLAGS); 386 Ok(()) 387 }), 388 StoreKind::Atomic(_size) => { 389 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 390 } 391 StoreKind::VectorLane(_selector) => { 392 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 393 } 394 }) 395 } 396 with_scratch<T: ScratchType, R>(&mut self, f: impl FnOnce(&mut Self, Scratch) -> R) -> R397 fn with_scratch<T: ScratchType, R>(&mut self, f: impl FnOnce(&mut Self, Scratch) -> R) -> R { 398 let r = self 399 .scratch_scope 400 .reg_for_class(T::reg_class(), &mut |_| Ok(())) 401 .expect("Scratch register to be available"); 402 403 let ret = f(self, Scratch::new(r)); 404 405 self.scratch_scope.free(r); 406 ret 407 } 408 call( &mut self, stack_args_size: u32, mut load_callee: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>, ) -> Result<u32>409 fn call( 410 &mut self, 411 stack_args_size: u32, 412 mut load_callee: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>, 413 ) -> Result<u32> { 414 let alignment: u32 = <Self::ABI as abi::ABI>::call_stack_align().into(); 415 let addend: u32 = <Self::ABI as abi::ABI>::initial_frame_size().into(); 416 let delta = calculate_frame_adjustment(self.sp_offset()?.as_u32(), addend, alignment); 417 let aligned_args_size = align_to(stack_args_size, alignment); 418 let total_stack = delta + aligned_args_size; 419 self.reserve_stack(total_stack)?; 420 let (callee, call_conv) = load_callee(self)?; 421 match callee { 422 CalleeKind::Indirect(reg) => self.asm.call_with_reg(reg, call_conv), 423 CalleeKind::Direct(idx) => self.asm.call_with_name(idx, call_conv), 424 } 425 426 Ok(total_stack) 427 } 428 load(&mut self, src: Address, dst: WritableReg, size: OperandSize) -> Result<()>429 fn load(&mut self, src: Address, dst: WritableReg, size: OperandSize) -> Result<()> { 430 src.to_addressing_mode(self, size, |masm, mem| { 431 Ok(masm.asm.uload(mem, dst, size, TRUSTED_FLAGS)) 432 }) 433 } 434 load_ptr(&mut self, src: Self::Address, dst: WritableReg) -> Result<()>435 fn load_ptr(&mut self, src: Self::Address, dst: WritableReg) -> Result<()> { 436 self.load(src, dst, self.ptr_size) 437 } 438 wasm_load(&mut self, src: Self::Address, dst: WritableReg, kind: LoadKind) -> Result<()>439 fn wasm_load(&mut self, src: Self::Address, dst: WritableReg, kind: LoadKind) -> Result<()> { 440 let size = kind.derive_operand_size(); 441 self.with_aligned_sp(|masm| match &kind { 442 LoadKind::Operand(_) => { 443 if size == OperandSize::S128 { 444 bail!(CodeGenError::UnimplementedWasmLoadKind) 445 } else { 446 src.to_addressing_mode(masm, size, |masm, mem| { 447 Ok(masm.asm.uload(mem, dst, size, UNTRUSTED_FLAGS)) 448 }) 449 } 450 } 451 LoadKind::Splat(_) => bail!(CodeGenError::UnimplementedWasmLoadKind), 452 LoadKind::ScalarExtend(extend_kind) => { 453 if extend_kind.signed() { 454 src.to_addressing_mode(masm, size, |masm, mem| { 455 masm.asm.sload(mem, dst, size, UNTRUSTED_FLAGS); 456 Ok(()) 457 }) 458 } else { 459 src.to_addressing_mode(masm, size, |masm, mem| { 460 // unlike x64, unused bits are set to zero so we don't need to extend 461 masm.asm.uload(mem, dst, size, UNTRUSTED_FLAGS); 462 Ok(()) 463 }) 464 } 465 } 466 LoadKind::VectorExtend(_vector_extend_kind) => { 467 bail!(CodeGenError::UnimplementedWasmLoadKind) 468 } 469 LoadKind::VectorLane(_selector) => { 470 bail!(CodeGenError::unimplemented_masm_instruction()) 471 } 472 LoadKind::Atomic(_, _) => bail!(CodeGenError::unimplemented_masm_instruction()), 473 LoadKind::VectorZero(_size) => { 474 bail!(CodeGenError::UnimplementedWasmLoadKind) 475 } 476 }) 477 } 478 compute_addr( &mut self, src: Self::Address, dst: WritableReg, size: OperandSize, ) -> Result<()>479 fn compute_addr( 480 &mut self, 481 src: Self::Address, 482 dst: WritableReg, 483 size: OperandSize, 484 ) -> Result<()> { 485 let (base, offset) = src.unwrap_offset(); 486 self.add_ir(dst, base, I::i64(offset), size) 487 } 488 pop(&mut self, dst: WritableReg, size: OperandSize) -> Result<()>489 fn pop(&mut self, dst: WritableReg, size: OperandSize) -> Result<()> { 490 let addr = self.address_from_sp(SPOffset::from_u32(self.sp_offset))?; 491 addr.to_addressing_mode(self, size, |masm, mem| { 492 masm.asm.uload(mem, dst, size, TRUSTED_FLAGS); 493 Ok(()) 494 })?; 495 self.free_stack(size.bytes()) 496 } 497 sp_offset(&self) -> Result<SPOffset>498 fn sp_offset(&self) -> Result<SPOffset> { 499 Ok(SPOffset::from_u32(self.sp_offset)) 500 } 501 finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>>502 fn finalize(mut self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>> { 503 if let Some(patch) = self.stack_max_use_add { 504 patch.finalize(i32::try_from(self.sp_max).unwrap(), self.asm.buffer_mut()); 505 } 506 507 Ok(self.asm.finalize(base)) 508 } 509 mov(&mut self, dst: WritableReg, src: RegImm, size: OperandSize) -> Result<()>510 fn mov(&mut self, dst: WritableReg, src: RegImm, size: OperandSize) -> Result<()> { 511 match (src, dst) { 512 (RegImm::Imm(v), _) => match v { 513 I::I32(_) | I::I64(_) => { 514 self.asm.mov_ir(dst, v, v.size()); 515 Ok(()) 516 } 517 imm @ (I::F32(_) | I::F64(_)) => { 518 self.asm.mov_ir(dst, imm, imm.size()); 519 Ok(()) 520 } 521 I::V128(_) => bail!(CodeGenError::unsupported_imm()), 522 }, 523 (RegImm::Reg(rs), rd) => match (rs.class(), rd.to_reg().class()) { 524 (RegClass::Int, RegClass::Int) => Ok(self.asm.mov_rr(rs, rd, size)), 525 (RegClass::Float, RegClass::Float) => Ok(self.asm.fmov_rr(rs, rd, size)), 526 (RegClass::Int, RegClass::Float) => Ok(self.asm.mov_to_fpu(rs, rd, size)), 527 _ => bail!(CodeGenError::invalid_operand_combination()), 528 }, 529 } 530 } 531 cmov( &mut self, dst: WritableReg, src: Reg, cc: IntCmpKind, size: OperandSize, ) -> Result<()>532 fn cmov( 533 &mut self, 534 dst: WritableReg, 535 src: Reg, 536 cc: IntCmpKind, 537 size: OperandSize, 538 ) -> Result<()> { 539 match (src.class(), dst.to_reg().class()) { 540 (RegClass::Int, RegClass::Int) => self.asm.csel(src, dst.to_reg(), dst, Cond::from(cc)), 541 (RegClass::Float, RegClass::Float) => { 542 self.asm 543 .fpu_csel(src, dst.to_reg(), dst, Cond::from(cc), size) 544 } 545 _ => return Err(format_err!(CodeGenError::invalid_operand_combination())), 546 } 547 548 Ok(()) 549 } 550 add(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>551 fn add(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> { 552 match (rhs, lhs, dst) { 553 (RegImm::Imm(v), rn, rd) => self.add_ir(rd, rn, v, size), 554 555 (RegImm::Reg(rm), rn, rd) => { 556 self.asm.add_rrr(rm, rn, rd, size); 557 Ok(()) 558 } 559 } 560 } 561 add_uextend( &mut self, dst: WritableReg, lhs: Reg, rhs: Reg, from_size: OperandSize, size: OperandSize, ) -> Result<()>562 fn add_uextend( 563 &mut self, 564 dst: WritableReg, 565 lhs: Reg, 566 rhs: Reg, 567 from_size: OperandSize, 568 size: OperandSize, 569 ) -> Result<()> { 570 assert!(from_size.num_bits() <= size.num_bits()); 571 let extendop = match from_size { 572 OperandSize::S8 => ExtendOp::UXTB, 573 OperandSize::S16 => ExtendOp::UXTH, 574 OperandSize::S32 => ExtendOp::UXTW, 575 OperandSize::S64 => ExtendOp::UXTX, 576 OperandSize::S128 => { 577 return Err(format_err!(CodeGenError::invalid_operand_combination())); 578 } 579 }; 580 581 self.asm.add_rrr_with_extend(rhs, lhs, dst, size, extendop); 582 Ok(()) 583 } 584 checked_uadd( &mut self, dst: WritableReg, lhs: Reg, rhs: Imm, size: OperandSize, trap: TrapCode, ) -> Result<()>585 fn checked_uadd( 586 &mut self, 587 dst: WritableReg, 588 lhs: Reg, 589 rhs: Imm, 590 size: OperandSize, 591 trap: TrapCode, 592 ) -> Result<()> { 593 // Similar to all the other potentially-trapping operations, we need to 594 // ensure that the real SP is 16-byte aligned in case control flow is 595 // transferred to a signal handler. 596 self.with_aligned_sp(|masm| { 597 // NB: we don't use `Self::add_ir` since we explicitly 598 // want to emit the add variant which sets overflow 599 // flags. 600 let imm = rhs.unwrap_as_u64(); 601 match Imm12::maybe_from_u64(imm) { 602 Some(imm12) => masm.asm.adds_ir(imm12, lhs, dst, size), 603 None => { 604 masm.with_scratch::<IntScratch, _>(|masm, scratch| { 605 masm.asm.mov_ir(scratch.writable(), rhs, rhs.size()); 606 masm.asm.adds_rrr(scratch.inner(), lhs, dst, size); 607 }); 608 } 609 } 610 masm.asm.trapif(Cond::Hs, trap); 611 Ok(()) 612 }) 613 } 614 sub(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>615 fn sub(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> { 616 match (rhs, lhs, dst) { 617 (RegImm::Imm(v), rn, rd) => { 618 let imm = v.unwrap_as_u64(); 619 match Imm12::maybe_from_u64(imm) { 620 Some(imm12) => self.asm.sub_ir(imm12, rn, rd, size), 621 None => { 622 self.with_scratch::<IntScratch, _>(|masm, scratch| { 623 masm.asm.mov_ir(scratch.writable(), v, v.size()); 624 masm.asm.sub_rrr(scratch.inner(), rn, rd, size); 625 }); 626 } 627 }; 628 629 Ok(()) 630 } 631 632 (RegImm::Reg(rm), rn, rd) => { 633 self.asm.sub_rrr(rm, rn, rd, size); 634 Ok(()) 635 } 636 } 637 } 638 mul(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>639 fn mul(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> { 640 match (rhs, lhs, dst) { 641 (RegImm::Imm(v), rn, rd) => self.with_scratch::<IntScratch, _>(|masm, scratch| { 642 masm.asm.mov_ir(scratch.writable(), v, v.size()); 643 masm.asm.mul_rrr(scratch.inner(), rn, rd, size); 644 Ok(()) 645 }), 646 647 (RegImm::Reg(rm), rn, rd) => { 648 self.asm.mul_rrr(rm, rn, rd, size); 649 Ok(()) 650 } 651 } 652 } 653 float_add(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>654 fn float_add(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> { 655 self.asm.fadd_rrr(rhs, lhs, dst, size); 656 Ok(()) 657 } 658 float_sub(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>659 fn float_sub(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> { 660 self.asm.fsub_rrr(rhs, lhs, dst, size); 661 Ok(()) 662 } 663 float_mul(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>664 fn float_mul(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> { 665 self.asm.fmul_rrr(rhs, lhs, dst, size); 666 Ok(()) 667 } 668 float_div(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>669 fn float_div(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> { 670 self.asm.fdiv_rrr(rhs, lhs, dst, size); 671 Ok(()) 672 } 673 float_min(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>674 fn float_min(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> { 675 self.asm.fmin_rrr(rhs, lhs, dst, size); 676 Ok(()) 677 } 678 float_max(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>679 fn float_max(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()> { 680 self.asm.fmax_rrr(rhs, lhs, dst, size); 681 Ok(()) 682 } 683 float_copysign( &mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize, ) -> Result<()>684 fn float_copysign( 685 &mut self, 686 dst: WritableReg, 687 lhs: Reg, 688 rhs: Reg, 689 size: OperandSize, 690 ) -> Result<()> { 691 let max_shift = match size { 692 OperandSize::S32 => 0x1f, 693 OperandSize::S64 => 0x3f, 694 _ => bail!(CodeGenError::unexpected_operand_size()), 695 }; 696 self.asm.fushr_rri(rhs, writable!(rhs), max_shift, size); 697 self.asm.fsli_rri_mod(lhs, rhs, dst, max_shift, size); 698 Ok(()) 699 } 700 float_neg(&mut self, dst: WritableReg, size: OperandSize) -> Result<()>701 fn float_neg(&mut self, dst: WritableReg, size: OperandSize) -> Result<()> { 702 self.asm.fneg_rr(dst.to_reg(), dst, size); 703 Ok(()) 704 } 705 float_abs(&mut self, dst: WritableReg, size: OperandSize) -> Result<()>706 fn float_abs(&mut self, dst: WritableReg, size: OperandSize) -> Result<()> { 707 self.asm.fabs_rr(dst.to_reg(), dst, size); 708 Ok(()) 709 } 710 float_round< F: FnMut(&mut FuncEnv<Self::Ptr>, &mut CodeGenContext<Emission>, &mut Self) -> Result<()>, >( &mut self, mode: RoundingMode, _env: &mut FuncEnv<Self::Ptr>, context: &mut CodeGenContext<Emission>, size: OperandSize, _fallback: F, ) -> Result<()>711 fn float_round< 712 F: FnMut(&mut FuncEnv<Self::Ptr>, &mut CodeGenContext<Emission>, &mut Self) -> Result<()>, 713 >( 714 &mut self, 715 mode: RoundingMode, 716 _env: &mut FuncEnv<Self::Ptr>, 717 context: &mut CodeGenContext<Emission>, 718 size: OperandSize, 719 _fallback: F, 720 ) -> Result<()> { 721 let src = context.pop_to_reg(self, None)?; 722 self.asm 723 .fround_rr(src.into(), writable!(src.into()), mode, size); 724 context.stack.push(src.into()); 725 Ok(()) 726 } 727 float_sqrt(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()>728 fn float_sqrt(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()> { 729 self.asm.fsqrt_rr(src, dst, size); 730 Ok(()) 731 } 732 and(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>733 fn and(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> { 734 match (rhs, lhs, dst) { 735 (RegImm::Imm(v), rn, rd) => { 736 let imm = v.unwrap_as_u64(); 737 let csize: inst::OperandSize = size.into(); 738 739 match ImmLogic::maybe_from_u64(imm, csize.to_ty()) { 740 Some(imml) => self.asm.and_ir(imml, rn, rd, size), 741 None => { 742 self.with_scratch::<IntScratch, _>(|masm, scratch| { 743 masm.asm.mov_ir(scratch.writable(), v, v.size()); 744 masm.asm.and_rrr(scratch.inner(), rn, rd, size); 745 }); 746 } 747 }; 748 749 Ok(()) 750 } 751 752 (RegImm::Reg(rm), rn, rd) => { 753 self.asm.and_rrr(rm, rn, rd, size); 754 Ok(()) 755 } 756 } 757 } 758 or(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>759 fn or(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> { 760 match (rhs, lhs, dst) { 761 (RegImm::Imm(v), rn, rd) => { 762 let imm = v.unwrap_as_u64(); 763 let csize: inst::OperandSize = size.into(); 764 765 match ImmLogic::maybe_from_u64(imm, csize.to_ty()) { 766 Some(imml) => self.asm.or_ir(imml, rn, rd, size), 767 None => { 768 self.with_scratch::<IntScratch, _>(|masm, scratch| { 769 masm.asm.mov_ir(scratch.writable(), v, v.size()); 770 masm.asm.or_rrr(scratch.inner(), rn, rd, size); 771 }); 772 } 773 }; 774 775 Ok(()) 776 } 777 778 (RegImm::Reg(rm), rn, rd) => { 779 self.asm.or_rrr(rm, rn, rd, size); 780 Ok(()) 781 } 782 } 783 } 784 xor(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>785 fn xor(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()> { 786 match (rhs, lhs, dst) { 787 (RegImm::Imm(v), rn, rd) => { 788 let imm = v.unwrap_as_u64(); 789 let csize: inst::OperandSize = size.into(); 790 791 match ImmLogic::maybe_from_u64(imm, csize.to_ty()) { 792 Some(imml) => self.asm.xor_ir(imml, rn, rd, size), 793 None => { 794 self.with_scratch::<IntScratch, _>(|masm, scratch| { 795 masm.asm.mov_ir(scratch.writable(), v, v.size()); 796 masm.asm.xor_rrr(scratch.inner(), rn, rd, size); 797 }); 798 } 799 }; 800 Ok(()) 801 } 802 803 (RegImm::Reg(rm), rn, rd) => { 804 self.asm.xor_rrr(rm, rn, rd, size); 805 Ok(()) 806 } 807 } 808 } 809 shift_ir( &mut self, dst: WritableReg, imm: I, lhs: Reg, kind: ShiftKind, size: OperandSize, ) -> Result<()>810 fn shift_ir( 811 &mut self, 812 dst: WritableReg, 813 imm: I, 814 lhs: Reg, 815 kind: ShiftKind, 816 size: OperandSize, 817 ) -> Result<()> { 818 match ImmShift::maybe_from_u64(imm.unwrap_as_u64()) { 819 // Immediate Ranges: 820 // 32-bit variant: 0-31 821 // 64-bit variant: 0-63 822 Some(imml) if imml.value() < size.num_bits() => { 823 self.asm.shift_ir(imml, lhs, dst, kind, size) 824 } 825 _ => { 826 self.with_scratch::<IntScratch, _>(|masm, scratch| { 827 masm.asm.mov_ir(scratch.writable(), imm, imm.size()); 828 masm.asm.shift_rrr(scratch.inner(), lhs, dst, kind, size); 829 }); 830 } 831 }; 832 Ok(()) 833 } 834 shift( &mut self, context: &mut CodeGenContext<Emission>, kind: ShiftKind, size: OperandSize, ) -> Result<()>835 fn shift( 836 &mut self, 837 context: &mut CodeGenContext<Emission>, 838 kind: ShiftKind, 839 size: OperandSize, 840 ) -> Result<()> { 841 let src = context.pop_to_reg(self, None)?; 842 let dst = context.pop_to_reg(self, None)?; 843 844 self.asm 845 .shift_rrr(src.into(), dst.into(), writable!(dst.into()), kind, size); 846 847 context.free_reg(src); 848 context.stack.push(dst.into()); 849 850 Ok(()) 851 } 852 div( &mut self, context: &mut CodeGenContext<Emission>, kind: DivKind, size: OperandSize, ) -> Result<()>853 fn div( 854 &mut self, 855 context: &mut CodeGenContext<Emission>, 856 kind: DivKind, 857 size: OperandSize, 858 ) -> Result<()> { 859 context.binop(self, size, |this, dividend, divisor, size| { 860 this.with_aligned_sp(|this| { 861 this.asm 862 .div_rrr(divisor, dividend, writable!(dividend), kind, size); 863 Ok(()) 864 })?; 865 match size { 866 OperandSize::S32 => Ok(TypedReg::new(WasmValType::I32, dividend)), 867 OperandSize::S64 => Ok(TypedReg::new(WasmValType::I64, dividend)), 868 _ => Err(format_err!(CodeGenError::unexpected_operand_size())), 869 } 870 }) 871 } 872 rem( &mut self, context: &mut CodeGenContext<Emission>, kind: RemKind, size: OperandSize, ) -> Result<()>873 fn rem( 874 &mut self, 875 context: &mut CodeGenContext<Emission>, 876 kind: RemKind, 877 size: OperandSize, 878 ) -> Result<()> { 879 context.binop(self, size, |this, dividend, divisor, size| { 880 this.with_aligned_sp(|this| { 881 this.with_scratch::<IntScratch, _>(|masm, scratch| { 882 masm.asm.rem_rrr( 883 divisor, 884 dividend, 885 writable!(dividend), 886 scratch.writable(), 887 kind, 888 size, 889 ); 890 }); 891 Ok(()) 892 })?; 893 match size { 894 OperandSize::S32 => Ok(TypedReg::new(WasmValType::I32, dividend)), 895 OperandSize::S64 => Ok(TypedReg::new(WasmValType::I64, dividend)), 896 _ => Err(format_err!(CodeGenError::unexpected_operand_size())), 897 } 898 }) 899 } 900 zero(&mut self, reg: WritableReg) -> Result<()>901 fn zero(&mut self, reg: WritableReg) -> Result<()> { 902 self.asm.mov_ir(reg, I::i64(0), OperandSize::S64); 903 Ok(()) 904 } 905 popcnt(&mut self, context: &mut CodeGenContext<Emission>, size: OperandSize) -> Result<()>906 fn popcnt(&mut self, context: &mut CodeGenContext<Emission>, size: OperandSize) -> Result<()> { 907 let src = context.pop_to_reg(self, None)?; 908 self.with_scratch::<FloatScratch, _>(|masm, tmp| { 909 masm.asm.mov_to_fpu(src.into(), tmp.writable(), size); 910 masm.asm.cnt(tmp.writable()); 911 masm.asm 912 .addv(tmp.inner(), tmp.writable(), VectorSize::Size8x8); 913 masm.asm 914 .mov_from_vec(tmp.inner(), writable!(src.into()), 0, OperandSize::S8); 915 }); 916 context.stack.push(src.into()); 917 Ok(()) 918 } 919 signed_truncate( &mut self, dst: WritableReg, src: Reg, src_size: OperandSize, dst_size: OperandSize, kind: TruncKind, ) -> Result<()>920 fn signed_truncate( 921 &mut self, 922 dst: WritableReg, 923 src: Reg, 924 src_size: OperandSize, 925 dst_size: OperandSize, 926 kind: TruncKind, 927 ) -> Result<()> { 928 self.with_aligned_sp(|masm| { 929 masm.with_scratch::<FloatScratch, _>(|masm, scratch| { 930 masm.asm 931 .fpu_to_int(dst, src, scratch.writable(), src_size, dst_size, kind, true); 932 }); 933 Ok(()) 934 }) 935 } 936 unsigned_truncate( &mut self, ctx: &mut CodeGenContext<Emission>, src_size: OperandSize, dst_size: OperandSize, kind: TruncKind, ) -> Result<()>937 fn unsigned_truncate( 938 &mut self, 939 ctx: &mut CodeGenContext<Emission>, 940 src_size: OperandSize, 941 dst_size: OperandSize, 942 kind: TruncKind, 943 ) -> Result<()> { 944 let dst_ty = match dst_size { 945 OperandSize::S32 => WasmValType::I32, 946 OperandSize::S64 => WasmValType::I64, 947 _ => bail!(CodeGenError::unexpected_operand_size()), 948 }; 949 950 ctx.convert_op(self, dst_ty, |masm, dst, src, dst_size| { 951 masm.with_aligned_sp(|masm| { 952 masm.with_scratch::<FloatScratch, _>(|masm, scratch| { 953 masm.asm.fpu_to_int( 954 writable!(dst), 955 src, 956 scratch.writable(), 957 src_size, 958 dst_size, 959 kind, 960 false, 961 ); 962 Ok(()) 963 }) 964 }) 965 }) 966 } 967 signed_convert( &mut self, dst: WritableReg, src: Reg, src_size: OperandSize, dst_size: OperandSize, ) -> Result<()>968 fn signed_convert( 969 &mut self, 970 dst: WritableReg, 971 src: Reg, 972 src_size: OperandSize, 973 dst_size: OperandSize, 974 ) -> Result<()> { 975 self.asm.cvt_sint_to_float(src, dst, src_size, dst_size); 976 Ok(()) 977 } 978 unsigned_convert( &mut self, dst: WritableReg, src: Reg, _tmp_gpr: Reg, src_size: OperandSize, dst_size: OperandSize, ) -> Result<()>979 fn unsigned_convert( 980 &mut self, 981 dst: WritableReg, 982 src: Reg, 983 _tmp_gpr: Reg, 984 src_size: OperandSize, 985 dst_size: OperandSize, 986 ) -> Result<()> { 987 self.asm.cvt_uint_to_float(src, dst, src_size, dst_size); 988 Ok(()) 989 } 990 reinterpret_float_as_int( &mut self, dst: WritableReg, src: Reg, size: OperandSize, ) -> Result<()>991 fn reinterpret_float_as_int( 992 &mut self, 993 dst: WritableReg, 994 src: Reg, 995 size: OperandSize, 996 ) -> Result<()> { 997 self.asm.mov_from_vec(src, dst, 0, size); 998 Ok(()) 999 } 1000 reinterpret_int_as_float( &mut self, dst: WritableReg, src: Reg, size: OperandSize, ) -> Result<()>1001 fn reinterpret_int_as_float( 1002 &mut self, 1003 dst: WritableReg, 1004 src: Reg, 1005 size: OperandSize, 1006 ) -> Result<()> { 1007 self.asm.mov_to_fpu(src, dst, size); 1008 Ok(()) 1009 } 1010 demote(&mut self, dst: WritableReg, src: Reg) -> Result<()>1011 fn demote(&mut self, dst: WritableReg, src: Reg) -> Result<()> { 1012 self.asm 1013 .cvt_float_to_float(src, dst, OperandSize::S64, OperandSize::S32); 1014 Ok(()) 1015 } 1016 promote(&mut self, dst: WritableReg, src: Reg) -> Result<()>1017 fn promote(&mut self, dst: WritableReg, src: Reg) -> Result<()> { 1018 self.asm 1019 .cvt_float_to_float(src, dst, OperandSize::S32, OperandSize::S64); 1020 Ok(()) 1021 } 1022 push(&mut self, reg: Reg, size: OperandSize) -> Result<StackSlot>1023 fn push(&mut self, reg: Reg, size: OperandSize) -> Result<StackSlot> { 1024 self.reserve_stack(size.bytes())?; 1025 let address = self.address_from_sp(SPOffset::from_u32(self.sp_offset))?; 1026 address.to_addressing_mode(self, size, |masm, mem| { 1027 masm.asm.str(reg, mem, size, TRUSTED_FLAGS); 1028 Ok(()) 1029 })?; 1030 1031 Ok(StackSlot { 1032 offset: SPOffset::from_u32(self.sp_offset), 1033 size: size.bytes(), 1034 }) 1035 } 1036 address_at_reg(&self, reg: Reg, offset: u32) -> Result<Self::Address>1037 fn address_at_reg(&self, reg: Reg, offset: u32) -> Result<Self::Address> { 1038 Ok(Address::offset(reg, offset as i64)) 1039 } 1040 cmp_with_set( &mut self, dst: WritableReg, src: RegImm, kind: IntCmpKind, size: OperandSize, ) -> Result<()>1041 fn cmp_with_set( 1042 &mut self, 1043 dst: WritableReg, 1044 src: RegImm, 1045 kind: IntCmpKind, 1046 size: OperandSize, 1047 ) -> Result<()> { 1048 self.cmp(dst.to_reg(), src, size)?; 1049 self.asm.cset(dst, kind.into()); 1050 Ok(()) 1051 } 1052 cmp(&mut self, src1: Reg, src2: RegImm, size: OperandSize) -> Result<()>1053 fn cmp(&mut self, src1: Reg, src2: RegImm, size: OperandSize) -> Result<()> { 1054 match src2 { 1055 RegImm::Reg(src2) => { 1056 self.asm.subs_rrr(src2, src1, size); 1057 Ok(()) 1058 } 1059 RegImm::Imm(v) => { 1060 let val = v.unwrap_as_u64(); 1061 match Imm12::maybe_from_u64(val) { 1062 Some(imm12) => self.asm.subs_ir(imm12, src1, size), 1063 None => { 1064 self.with_scratch::<IntScratch, _>(|masm, scratch| { 1065 masm.asm.mov_ir(scratch.writable(), v, v.size()); 1066 masm.asm.subs_rrr(scratch.inner(), src1, size); 1067 }); 1068 } 1069 }; 1070 Ok(()) 1071 } 1072 } 1073 } 1074 float_cmp_with_set( &mut self, dst: WritableReg, src1: Reg, src2: Reg, kind: FloatCmpKind, size: OperandSize, ) -> Result<()>1075 fn float_cmp_with_set( 1076 &mut self, 1077 dst: WritableReg, 1078 src1: Reg, 1079 src2: Reg, 1080 kind: FloatCmpKind, 1081 size: OperandSize, 1082 ) -> Result<()> { 1083 self.asm.fcmp(src1, src2, size); 1084 self.asm.cset(dst, kind.into()); 1085 Ok(()) 1086 } 1087 clz(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()>1088 fn clz(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()> { 1089 self.asm.clz(src, dst, size); 1090 Ok(()) 1091 } 1092 ctz(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()>1093 fn ctz(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()> { 1094 self.with_scratch::<IntScratch, _>(|masm, scratch| { 1095 masm.asm.rbit(src, scratch.writable(), size); 1096 masm.asm.clz(scratch.inner(), dst, size); 1097 Ok(()) 1098 }) 1099 } 1100 wrap(&mut self, dst: WritableReg, src: Reg) -> Result<()>1101 fn wrap(&mut self, dst: WritableReg, src: Reg) -> Result<()> { 1102 self.asm.mov_rr(src, dst, OperandSize::S32); 1103 Ok(()) 1104 } 1105 extend(&mut self, dst: WritableReg, src: Reg, kind: ExtendKind) -> Result<()>1106 fn extend(&mut self, dst: WritableReg, src: Reg, kind: ExtendKind) -> Result<()> { 1107 self.asm.extend(src, dst, kind); 1108 Ok(()) 1109 } 1110 get_label(&mut self) -> Result<MachLabel>1111 fn get_label(&mut self) -> Result<MachLabel> { 1112 Ok(self.asm.get_label()) 1113 } 1114 bind(&mut self, label: MachLabel) -> Result<()>1115 fn bind(&mut self, label: MachLabel) -> Result<()> { 1116 let buffer = self.asm.buffer_mut(); 1117 buffer.bind_label(label, &mut Default::default()); 1118 Ok(()) 1119 } 1120 branch( &mut self, kind: IntCmpKind, lhs: Reg, rhs: RegImm, taken: MachLabel, size: OperandSize, ) -> Result<()>1121 fn branch( 1122 &mut self, 1123 kind: IntCmpKind, 1124 lhs: Reg, 1125 rhs: RegImm, 1126 taken: MachLabel, 1127 size: OperandSize, 1128 ) -> Result<()> { 1129 use IntCmpKind::*; 1130 1131 match &(lhs, rhs) { 1132 (rlhs, RegImm::Reg(rrhs)) => { 1133 // If the comparison kind is zero or not zero and both operands 1134 // are the same register, emit an ands instruction. Else we emit 1135 // a normal comparison. 1136 if (kind == Eq || kind == Ne) && (rlhs == rrhs) { 1137 self.asm.ands_rr(*rlhs, *rrhs, size); 1138 } else { 1139 self.cmp(lhs, rhs, size)?; 1140 } 1141 } 1142 _ => self.cmp(lhs, rhs, size)?, 1143 } 1144 self.asm.jmp_if(kind.into(), taken); 1145 Ok(()) 1146 } 1147 jmp(&mut self, target: MachLabel) -> Result<()>1148 fn jmp(&mut self, target: MachLabel) -> Result<()> { 1149 self.asm.jmp(target); 1150 Ok(()) 1151 } 1152 unreachable(&mut self) -> Result<()>1153 fn unreachable(&mut self) -> Result<()> { 1154 self.with_aligned_sp(|masm| { 1155 masm.asm.udf(wasmtime_cranelift::TRAP_UNREACHABLE); 1156 Ok(()) 1157 }) 1158 } 1159 jmp_table(&mut self, targets: &[MachLabel], index: Reg, tmp: Reg) -> Result<()>1160 fn jmp_table(&mut self, targets: &[MachLabel], index: Reg, tmp: Reg) -> Result<()> { 1161 // At least one default target. 1162 debug_assert!(targets.len() >= 1); 1163 let default_index = targets.len() - 1; 1164 let max = default_index; 1165 self.asm.mov_ir( 1166 writable!(tmp), 1167 I::i32(i32::try_from(max).unwrap()), 1168 OperandSize::S32, 1169 ); 1170 // NB: We only emit the comparison instruction, since 1171 // `Assembler::jmp_table` (and the underlying Cranelift 1172 // instruction) will emit spectre mitigation and bounds 1173 // checks. 1174 self.asm.subs_rrr(tmp, index, OperandSize::S32); 1175 let default = targets[default_index]; 1176 let rest = &targets[0..default_index]; 1177 self.with_scratch::<IntScratch, _>(|masm, scratch| { 1178 masm.asm 1179 .jmp_table(rest, default, index, scratch.inner(), tmp); 1180 Ok(()) 1181 }) 1182 } 1183 trap(&mut self, code: TrapCode) -> Result<()>1184 fn trap(&mut self, code: TrapCode) -> Result<()> { 1185 self.with_aligned_sp(|masm| { 1186 masm.asm.udf(code); 1187 Ok(()) 1188 }) 1189 } 1190 trapz(&mut self, src: Reg, code: TrapCode) -> Result<()>1191 fn trapz(&mut self, src: Reg, code: TrapCode) -> Result<()> { 1192 self.with_aligned_sp(|masm| { 1193 masm.asm.trapz(src, code, OperandSize::S64); 1194 Ok(()) 1195 }) 1196 } 1197 trapif(&mut self, cc: IntCmpKind, code: TrapCode) -> Result<()>1198 fn trapif(&mut self, cc: IntCmpKind, code: TrapCode) -> Result<()> { 1199 self.with_aligned_sp(|masm| { 1200 masm.asm.trapif(cc.into(), code); 1201 Ok(()) 1202 }) 1203 } 1204 start_source_loc(&mut self, loc: RelSourceLoc) -> Result<(CodeOffset, RelSourceLoc)>1205 fn start_source_loc(&mut self, loc: RelSourceLoc) -> Result<(CodeOffset, RelSourceLoc)> { 1206 Ok(self.asm.buffer_mut().start_srcloc(loc)) 1207 } 1208 end_source_loc(&mut self) -> Result<()>1209 fn end_source_loc(&mut self) -> Result<()> { 1210 self.asm.buffer_mut().end_srcloc(); 1211 Ok(()) 1212 } 1213 current_code_offset(&self) -> Result<CodeOffset>1214 fn current_code_offset(&self) -> Result<CodeOffset> { 1215 Ok(self.asm.buffer().cur_offset()) 1216 } 1217 add128( &mut self, dst_lo: WritableReg, dst_hi: WritableReg, lhs_lo: Reg, lhs_hi: Reg, rhs_lo: Reg, rhs_hi: Reg, ) -> Result<()>1218 fn add128( 1219 &mut self, 1220 dst_lo: WritableReg, 1221 dst_hi: WritableReg, 1222 lhs_lo: Reg, 1223 lhs_hi: Reg, 1224 rhs_lo: Reg, 1225 rhs_hi: Reg, 1226 ) -> Result<()> { 1227 let _ = (dst_lo, dst_hi, lhs_lo, lhs_hi, rhs_lo, rhs_hi); 1228 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1229 } 1230 sub128( &mut self, dst_lo: WritableReg, dst_hi: WritableReg, lhs_lo: Reg, lhs_hi: Reg, rhs_lo: Reg, rhs_hi: Reg, ) -> Result<()>1231 fn sub128( 1232 &mut self, 1233 dst_lo: WritableReg, 1234 dst_hi: WritableReg, 1235 lhs_lo: Reg, 1236 lhs_hi: Reg, 1237 rhs_lo: Reg, 1238 rhs_hi: Reg, 1239 ) -> Result<()> { 1240 let _ = (dst_lo, dst_hi, lhs_lo, lhs_hi, rhs_lo, rhs_hi); 1241 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1242 } 1243 mul_wide( &mut self, context: &mut CodeGenContext<Emission>, kind: MulWideKind, ) -> Result<()>1244 fn mul_wide( 1245 &mut self, 1246 context: &mut CodeGenContext<Emission>, 1247 kind: MulWideKind, 1248 ) -> Result<()> { 1249 let _ = (context, kind); 1250 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1251 } 1252 splat(&mut self, _context: &mut CodeGenContext<Emission>, _size: SplatKind) -> Result<()>1253 fn splat(&mut self, _context: &mut CodeGenContext<Emission>, _size: SplatKind) -> Result<()> { 1254 bail!(CodeGenError::unimplemented_masm_instruction()) 1255 } 1256 shuffle(&mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _lanes: [u8; 16]) -> Result<()>1257 fn shuffle(&mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _lanes: [u8; 16]) -> Result<()> { 1258 bail!(CodeGenError::unimplemented_masm_instruction()) 1259 } 1260 swizzle(&mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg) -> Result<()>1261 fn swizzle(&mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg) -> Result<()> { 1262 bail!(CodeGenError::unimplemented_masm_instruction()) 1263 } 1264 atomic_rmw( &mut self, _context: &mut CodeGenContext<Emission>, _addr: Self::Address, _size: OperandSize, _op: RmwOp, _flags: MemFlags, _extend: Option<Extend<Zero>>, ) -> Result<()>1265 fn atomic_rmw( 1266 &mut self, 1267 _context: &mut CodeGenContext<Emission>, 1268 _addr: Self::Address, 1269 _size: OperandSize, 1270 _op: RmwOp, 1271 _flags: MemFlags, 1272 _extend: Option<Extend<Zero>>, 1273 ) -> Result<()> { 1274 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1275 } 1276 extract_lane( &mut self, _src: Reg, _dst: WritableReg, _lane: u8, _kind: ExtractLaneKind, ) -> Result<()>1277 fn extract_lane( 1278 &mut self, 1279 _src: Reg, 1280 _dst: WritableReg, 1281 _lane: u8, 1282 _kind: ExtractLaneKind, 1283 ) -> Result<()> { 1284 bail!(CodeGenError::unimplemented_masm_instruction()) 1285 } 1286 replace_lane( &mut self, _src: RegImm, _dst: WritableReg, _lane: u8, _kind: ReplaceLaneKind, ) -> Result<()>1287 fn replace_lane( 1288 &mut self, 1289 _src: RegImm, 1290 _dst: WritableReg, 1291 _lane: u8, 1292 _kind: ReplaceLaneKind, 1293 ) -> Result<()> { 1294 bail!(CodeGenError::unimplemented_masm_instruction()) 1295 } 1296 atomic_cas( &mut self, _context: &mut CodeGenContext<Emission>, _addr: Self::Address, _size: OperandSize, _flags: MemFlags, _extend: Option<Extend<Zero>>, ) -> Result<()>1297 fn atomic_cas( 1298 &mut self, 1299 _context: &mut CodeGenContext<Emission>, 1300 _addr: Self::Address, 1301 _size: OperandSize, 1302 _flags: MemFlags, 1303 _extend: Option<Extend<Zero>>, 1304 ) -> Result<()> { 1305 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1306 } 1307 v128_eq( &mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _kind: VectorEqualityKind, ) -> Result<()>1308 fn v128_eq( 1309 &mut self, 1310 _dst: WritableReg, 1311 _lhs: Reg, 1312 _rhs: Reg, 1313 _kind: VectorEqualityKind, 1314 ) -> Result<()> { 1315 bail!(CodeGenError::unimplemented_masm_instruction()) 1316 } 1317 v128_ne( &mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _kind: VectorEqualityKind, ) -> Result<()>1318 fn v128_ne( 1319 &mut self, 1320 _dst: WritableReg, 1321 _lhs: Reg, 1322 _rhs: Reg, 1323 _kind: VectorEqualityKind, 1324 ) -> Result<()> { 1325 bail!(CodeGenError::unimplemented_masm_instruction()) 1326 } 1327 v128_lt( &mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _kind: VectorCompareKind, ) -> Result<()>1328 fn v128_lt( 1329 &mut self, 1330 _dst: WritableReg, 1331 _lhs: Reg, 1332 _rhs: Reg, 1333 _kind: VectorCompareKind, 1334 ) -> Result<()> { 1335 bail!(CodeGenError::unimplemented_masm_instruction()) 1336 } 1337 v128_le( &mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _kind: VectorCompareKind, ) -> Result<()>1338 fn v128_le( 1339 &mut self, 1340 _dst: WritableReg, 1341 _lhs: Reg, 1342 _rhs: Reg, 1343 _kind: VectorCompareKind, 1344 ) -> Result<()> { 1345 bail!(CodeGenError::unimplemented_masm_instruction()) 1346 } 1347 v128_gt( &mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _kind: VectorCompareKind, ) -> Result<()>1348 fn v128_gt( 1349 &mut self, 1350 _dst: WritableReg, 1351 _lhs: Reg, 1352 _rhs: Reg, 1353 _kind: VectorCompareKind, 1354 ) -> Result<()> { 1355 bail!(CodeGenError::unimplemented_masm_instruction()) 1356 } 1357 v128_ge( &mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _kind: VectorCompareKind, ) -> Result<()>1358 fn v128_ge( 1359 &mut self, 1360 _dst: WritableReg, 1361 _lhs: Reg, 1362 _rhs: Reg, 1363 _kind: VectorCompareKind, 1364 ) -> Result<()> { 1365 bail!(CodeGenError::unimplemented_masm_instruction()) 1366 } 1367 v128_not(&mut self, _dst: WritableReg) -> Result<()>1368 fn v128_not(&mut self, _dst: WritableReg) -> Result<()> { 1369 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1370 } 1371 fence(&mut self) -> Result<()>1372 fn fence(&mut self) -> Result<()> { 1373 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1374 } 1375 v128_and(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()>1376 fn v128_and(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()> { 1377 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1378 } 1379 v128_and_not(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()>1380 fn v128_and_not(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()> { 1381 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1382 } 1383 v128_or(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()>1384 fn v128_or(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()> { 1385 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1386 } 1387 v128_xor(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()>1388 fn v128_xor(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()> { 1389 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1390 } 1391 v128_bitselect( &mut self, _src1: Reg, _src2: Reg, _mask: Reg, _dst: WritableReg, ) -> Result<()>1392 fn v128_bitselect( 1393 &mut self, 1394 _src1: Reg, 1395 _src2: Reg, 1396 _mask: Reg, 1397 _dst: WritableReg, 1398 ) -> Result<()> { 1399 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1400 } 1401 v128_any_true(&mut self, _src: Reg, _dst: WritableReg) -> Result<()>1402 fn v128_any_true(&mut self, _src: Reg, _dst: WritableReg) -> Result<()> { 1403 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1404 } 1405 v128_convert(&mut self, _src: Reg, _dst: WritableReg, _kind: V128ConvertKind) -> Result<()>1406 fn v128_convert(&mut self, _src: Reg, _dst: WritableReg, _kind: V128ConvertKind) -> Result<()> { 1407 bail!(CodeGenError::unimplemented_masm_instruction()) 1408 } 1409 v128_narrow( &mut self, _src1: Reg, _src2: Reg, _dst: WritableReg, _kind: V128NarrowKind, ) -> Result<()>1410 fn v128_narrow( 1411 &mut self, 1412 _src1: Reg, 1413 _src2: Reg, 1414 _dst: WritableReg, 1415 _kind: V128NarrowKind, 1416 ) -> Result<()> { 1417 bail!(CodeGenError::unimplemented_masm_instruction()) 1418 } 1419 v128_demote(&mut self, _src: Reg, _dst: WritableReg) -> Result<()>1420 fn v128_demote(&mut self, _src: Reg, _dst: WritableReg) -> Result<()> { 1421 bail!(CodeGenError::unimplemented_masm_instruction()) 1422 } 1423 v128_promote(&mut self, _src: Reg, _dst: WritableReg) -> Result<()>1424 fn v128_promote(&mut self, _src: Reg, _dst: WritableReg) -> Result<()> { 1425 bail!(CodeGenError::unimplemented_masm_instruction()) 1426 } 1427 v128_extend(&mut self, _src: Reg, _dst: WritableReg, _kind: V128ExtendKind) -> Result<()>1428 fn v128_extend(&mut self, _src: Reg, _dst: WritableReg, _kind: V128ExtendKind) -> Result<()> { 1429 bail!(CodeGenError::unimplemented_masm_instruction()) 1430 } 1431 v128_add( &mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg, _kind: V128AddKind, ) -> Result<()>1432 fn v128_add( 1433 &mut self, 1434 _lhs: Reg, 1435 _rhs: Reg, 1436 _dst: WritableReg, 1437 _kind: V128AddKind, 1438 ) -> Result<()> { 1439 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1440 } 1441 v128_sub( &mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg, _kind: V128SubKind, ) -> Result<()>1442 fn v128_sub( 1443 &mut self, 1444 _lhs: Reg, 1445 _rhs: Reg, 1446 _dst: WritableReg, 1447 _kind: V128SubKind, 1448 ) -> Result<()> { 1449 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1450 } 1451 v128_mul( &mut self, _context: &mut CodeGenContext<Emission>, _kind: V128MulKind, ) -> Result<()>1452 fn v128_mul( 1453 &mut self, 1454 _context: &mut CodeGenContext<Emission>, 1455 _kind: V128MulKind, 1456 ) -> Result<()> { 1457 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1458 } 1459 v128_abs(&mut self, _src: Reg, _dst: WritableReg, _kind: V128AbsKind) -> Result<()>1460 fn v128_abs(&mut self, _src: Reg, _dst: WritableReg, _kind: V128AbsKind) -> Result<()> { 1461 bail!(CodeGenError::unimplemented_masm_instruction()) 1462 } 1463 v128_neg(&mut self, _op: WritableReg, _kind: V128NegKind) -> Result<()>1464 fn v128_neg(&mut self, _op: WritableReg, _kind: V128NegKind) -> Result<()> { 1465 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1466 } 1467 v128_shift( &mut self, _context: &mut CodeGenContext<Emission>, _lane_width: OperandSize, _shift_kind: ShiftKind, ) -> Result<()>1468 fn v128_shift( 1469 &mut self, 1470 _context: &mut CodeGenContext<Emission>, 1471 _lane_width: OperandSize, 1472 _shift_kind: ShiftKind, 1473 ) -> Result<()> { 1474 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1475 } 1476 v128_q15mulr_sat_s( &mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg, _size: OperandSize, ) -> Result<()>1477 fn v128_q15mulr_sat_s( 1478 &mut self, 1479 _lhs: Reg, 1480 _rhs: Reg, 1481 _dst: WritableReg, 1482 _size: OperandSize, 1483 ) -> Result<()> { 1484 bail!(CodeGenError::unimplemented_masm_instruction()) 1485 } 1486 v128_all_true(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()>1487 fn v128_all_true(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()> { 1488 bail!(CodeGenError::unimplemented_masm_instruction()) 1489 } 1490 v128_bitmask(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()>1491 fn v128_bitmask(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()> { 1492 bail!(CodeGenError::unimplemented_masm_instruction()) 1493 } 1494 v128_trunc( &mut self, _context: &mut CodeGenContext<Emission>, _kind: V128TruncKind, ) -> Result<()>1495 fn v128_trunc( 1496 &mut self, 1497 _context: &mut CodeGenContext<Emission>, 1498 _kind: V128TruncKind, 1499 ) -> Result<()> { 1500 bail!(CodeGenError::unimplemented_masm_instruction()) 1501 } 1502 v128_min( &mut self, _src1: Reg, _src2: Reg, _dst: WritableReg, _kind: V128MinKind, ) -> Result<()>1503 fn v128_min( 1504 &mut self, 1505 _src1: Reg, 1506 _src2: Reg, 1507 _dst: WritableReg, 1508 _kind: V128MinKind, 1509 ) -> Result<()> { 1510 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1511 } 1512 v128_max( &mut self, _src1: Reg, _src2: Reg, _dst: WritableReg, _kind: V128MaxKind, ) -> Result<()>1513 fn v128_max( 1514 &mut self, 1515 _src1: Reg, 1516 _src2: Reg, 1517 _dst: WritableReg, 1518 _kind: V128MaxKind, 1519 ) -> Result<()> { 1520 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1521 } 1522 v128_extmul( &mut self, _context: &mut CodeGenContext<Emission>, _kind: V128ExtMulKind, ) -> Result<()>1523 fn v128_extmul( 1524 &mut self, 1525 _context: &mut CodeGenContext<Emission>, 1526 _kind: V128ExtMulKind, 1527 ) -> Result<()> { 1528 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1529 } 1530 v128_extadd_pairwise( &mut self, _src: Reg, _dst: WritableReg, _kind: V128ExtAddKind, ) -> Result<()>1531 fn v128_extadd_pairwise( 1532 &mut self, 1533 _src: Reg, 1534 _dst: WritableReg, 1535 _kind: V128ExtAddKind, 1536 ) -> Result<()> { 1537 Err(format_err!(CodeGenError::unimplemented_masm_instruction())) 1538 } 1539 v128_dot(&mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg) -> Result<()>1540 fn v128_dot(&mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg) -> Result<()> { 1541 bail!(CodeGenError::unimplemented_masm_instruction()) 1542 } 1543 v128_popcnt(&mut self, _context: &mut CodeGenContext<Emission>) -> Result<()>1544 fn v128_popcnt(&mut self, _context: &mut CodeGenContext<Emission>) -> Result<()> { 1545 bail!(CodeGenError::unimplemented_masm_instruction()) 1546 } 1547 v128_avgr( &mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg, _size: OperandSize, ) -> Result<()>1548 fn v128_avgr( 1549 &mut self, 1550 _lhs: Reg, 1551 _rhs: Reg, 1552 _dst: WritableReg, 1553 _size: OperandSize, 1554 ) -> Result<()> { 1555 bail!(CodeGenError::unimplemented_masm_instruction()) 1556 } 1557 v128_div( &mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg, _size: OperandSize, ) -> Result<()>1558 fn v128_div( 1559 &mut self, 1560 _lhs: Reg, 1561 _rhs: Reg, 1562 _dst: WritableReg, 1563 _size: OperandSize, 1564 ) -> Result<()> { 1565 bail!(CodeGenError::unimplemented_masm_instruction()) 1566 } 1567 v128_sqrt(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()>1568 fn v128_sqrt(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()> { 1569 bail!(CodeGenError::unimplemented_masm_instruction()) 1570 } 1571 v128_ceil(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()>1572 fn v128_ceil(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()> { 1573 bail!(CodeGenError::unimplemented_masm_instruction()) 1574 } 1575 v128_floor(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()>1576 fn v128_floor(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()> { 1577 bail!(CodeGenError::unimplemented_masm_instruction()) 1578 } 1579 v128_nearest(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()>1580 fn v128_nearest(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()> { 1581 bail!(CodeGenError::unimplemented_masm_instruction()) 1582 } 1583 v128_pmin( &mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg, _size: OperandSize, ) -> Result<()>1584 fn v128_pmin( 1585 &mut self, 1586 _lhs: Reg, 1587 _rhs: Reg, 1588 _dst: WritableReg, 1589 _size: OperandSize, 1590 ) -> Result<()> { 1591 bail!(CodeGenError::unimplemented_masm_instruction()) 1592 } 1593 v128_pmax( &mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg, _size: OperandSize, ) -> Result<()>1594 fn v128_pmax( 1595 &mut self, 1596 _lhs: Reg, 1597 _rhs: Reg, 1598 _dst: WritableReg, 1599 _size: OperandSize, 1600 ) -> Result<()> { 1601 bail!(CodeGenError::unimplemented_masm_instruction()) 1602 } 1603 } 1604 1605 impl MacroAssembler { increment_sp(&mut self, bytes: u32)1606 fn increment_sp(&mut self, bytes: u32) { 1607 self.sp_offset += bytes; 1608 1609 // NOTE: we use `max` here to track the largest stack allocation in `sp_max`. Once we have 1610 // seen the entire function, this value will represent the maximum size for the stack 1611 // frame. 1612 self.sp_max = self.sp_max.max(self.sp_offset); 1613 } 1614 decrement_sp(&mut self, bytes: u32)1615 fn decrement_sp(&mut self, bytes: u32) { 1616 self.sp_offset -= bytes; 1617 } 1618 1619 // Copies the value of the stack pointer to the shadow stack 1620 // pointer: mov x28, sp 1621 1622 // This function is called at the epilogue. move_sp_to_shadow_sp(&mut self)1623 fn move_sp_to_shadow_sp(&mut self) { 1624 let sp = regs::sp(); 1625 let shadow_sp = regs::shadow_sp(); 1626 self.asm.mov_rr(sp, writable!(shadow_sp), OperandSize::S64); 1627 } 1628 1629 /// Helper to add an immediate to a register. add_ir(&mut self, dst: WritableReg, lhs: Reg, rhs: I, size: OperandSize) -> Result<()>1630 fn add_ir(&mut self, dst: WritableReg, lhs: Reg, rhs: I, size: OperandSize) -> Result<()> { 1631 let imm = rhs.unwrap_as_u64(); 1632 match Imm12::maybe_from_u64(imm) { 1633 Some(imm12) => self.asm.add_ir(imm12, lhs, dst, size), 1634 None => { 1635 self.with_scratch::<IntScratch, _>(|masm, scratch| { 1636 masm.asm.mov_ir(scratch.writable(), rhs, rhs.size()); 1637 masm.asm.add_rrr(scratch.inner(), lhs, dst, size); 1638 }); 1639 } 1640 }; 1641 Ok(()) 1642 } 1643 1644 // Copies the value of the shadow stack pointer to the stack pointer: mov 1645 // sp, x28. 1646 // 1647 // This function is usually called when the space is claimed, e.g., via 1648 // a push, when stack space is reserved explicitly or after emitting code 1649 // that requires explicit stack pointer alignment (code that could result in 1650 // signal handling). 1651 // 1652 // This ensures the stack pointer always reflects the allocated stack space, 1653 // otherwise any space below the stack pointer could get clobbered with 1654 // interrupts and signal handlers. 1655 // 1656 // This function must also be called at the function epilogue, since the 1657 // stack pointer is used to restore the current function frame. move_shadow_sp_to_sp(&mut self)1658 fn move_shadow_sp_to_sp(&mut self) { 1659 let shadow_sp = regs::shadow_sp(); 1660 let sp = writable!(regs::sp()); 1661 let imm = Imm12::maybe_from_u64(0).unwrap(); 1662 self.asm.add_ir(imm, shadow_sp, sp, OperandSize::S64); 1663 } 1664 } 1665