1 //! Assembler library implementation for x64. 2 3 use crate::{ 4 constant_pool::ConstantPool, 5 isa::{CallingConvention, reg::Reg}, 6 masm::{ 7 DivKind, Extend, ExtendKind, ExtendType, IntCmpKind, MulWideKind, OperandSize, RemKind, 8 RoundingMode, ShiftKind, Signed, V128ExtendKind, V128LoadExtendKind, Zero, 9 }, 10 reg::writable, 11 }; 12 use cranelift_codegen::{ 13 CallInfo, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, 14 MachLabel, PatchRegion, Writable, 15 ir::{ExternalName, MemFlags, SourceLoc, TrapCode, Type, UserExternalNameRef, types}, 16 isa::{ 17 unwind::UnwindInst, 18 x64::{ 19 AtomicRmwSeqOp, EmitInfo, EmitState, Inst, 20 args::{ 21 self, Amode, Avx512Opcode, AvxOpcode, CC, ExtMode, FromWritableReg, Gpr, GprMem, 22 GprMemImm, RegMem, RegMemImm, SyntheticAmode, WritableGpr, WritableXmm, Xmm, 23 XmmMem, XmmMemImm, 24 }, 25 encoding::rex::{RexFlags, encode_modrm}, 26 external::{PairedGpr, PairedXmm}, 27 settings as x64_settings, 28 }, 29 }, 30 settings, 31 }; 32 33 use crate::reg::WritableReg; 34 use cranelift_assembler_x64 as asm; 35 use wasmtime_environ::Unsigned; 36 37 use super::address::Address; 38 use smallvec::SmallVec; 39 40 // Conversions between winch-codegen x64 types and cranelift-codegen x64 types. 41 42 impl From<Reg> for RegMemImm { 43 fn from(reg: Reg) -> Self { 44 RegMemImm::reg(reg.into()) 45 } 46 } 47 48 impl From<Reg> for RegMem { 49 fn from(value: Reg) -> Self { 50 RegMem::Reg { reg: value.into() } 51 } 52 } 53 54 impl From<Reg> for WritableGpr { 55 fn from(reg: Reg) -> Self { 56 let writable = Writable::from_reg(reg.into()); 57 WritableGpr::from_writable_reg(writable).expect("valid writable gpr") 58 } 59 } 60 61 impl From<Reg> for WritableXmm { 62 fn from(reg: Reg) -> Self { 63 let writable = Writable::from_reg(reg.into()); 64 WritableXmm::from_writable_reg(writable).expect("valid writable xmm") 65 } 66 } 67 68 /// Convert a writable GPR register to the read-write pair expected by 69 /// `cranelift-codegen`. 70 fn pair_gpr(reg: WritableReg) -> PairedGpr { 71 assert!(reg.to_reg().is_int()); 72 let read = Gpr::unwrap_new(reg.to_reg().into()); 73 let write = WritableGpr::from_reg(reg.to_reg().into()); 74 PairedGpr { read, write } 75 } 76 77 impl From<Reg> for asm::Gpr<Gpr> { 78 fn from(reg: Reg) -> Self { 79 asm::Gpr::new(reg.into()) 80 } 81 } 82 83 impl From<Reg> for asm::GprMem<Gpr, Gpr> { 84 fn from(reg: Reg) -> Self { 85 asm::GprMem::Gpr(reg.into()) 86 } 87 } 88 89 /// Convert a writable XMM register to the read-write pair expected by 90 /// `cranelift-codegen`. 91 fn pair_xmm(reg: WritableReg) -> PairedXmm { 92 assert!(reg.to_reg().is_float()); 93 let read = Xmm::unwrap_new(reg.to_reg().into()); 94 let write = WritableXmm::from_reg(reg.to_reg().into()); 95 PairedXmm { read, write } 96 } 97 98 impl From<Reg> for asm::Xmm<Xmm> { 99 fn from(reg: Reg) -> Self { 100 asm::Xmm::new(reg.into()) 101 } 102 } 103 104 impl From<Reg> for asm::XmmMem<Xmm, Gpr> { 105 fn from(reg: Reg) -> Self { 106 asm::XmmMem::Xmm(reg.into()) 107 } 108 } 109 110 impl From<Reg> for Gpr { 111 fn from(reg: Reg) -> Self { 112 Gpr::unwrap_new(reg.into()) 113 } 114 } 115 116 impl From<Reg> for GprMem { 117 fn from(value: Reg) -> Self { 118 GprMem::unwrap_new(value.into()) 119 } 120 } 121 122 impl From<Reg> for GprMemImm { 123 fn from(reg: Reg) -> Self { 124 GprMemImm::unwrap_new(reg.into()) 125 } 126 } 127 128 impl From<Reg> for Xmm { 129 fn from(reg: Reg) -> Self { 130 Xmm::unwrap_new(reg.into()) 131 } 132 } 133 134 impl From<Reg> for XmmMem { 135 fn from(value: Reg) -> Self { 136 XmmMem::unwrap_new(value.into()) 137 } 138 } 139 140 impl From<Reg> for XmmMemImm { 141 fn from(value: Reg) -> Self { 142 XmmMemImm::unwrap_new(value.into()) 143 } 144 } 145 146 impl From<OperandSize> for args::OperandSize { 147 fn from(size: OperandSize) -> Self { 148 match size { 149 OperandSize::S8 => Self::Size8, 150 OperandSize::S16 => Self::Size16, 151 OperandSize::S32 => Self::Size32, 152 OperandSize::S64 => Self::Size64, 153 s => panic!("Invalid operand size {s:?}"), 154 } 155 } 156 } 157 158 impl From<IntCmpKind> for CC { 159 fn from(value: IntCmpKind) -> Self { 160 match value { 161 IntCmpKind::Eq => CC::Z, 162 IntCmpKind::Ne => CC::NZ, 163 IntCmpKind::LtS => CC::L, 164 IntCmpKind::LtU => CC::B, 165 IntCmpKind::GtS => CC::NLE, 166 IntCmpKind::GtU => CC::NBE, 167 IntCmpKind::LeS => CC::LE, 168 IntCmpKind::LeU => CC::BE, 169 IntCmpKind::GeS => CC::NL, 170 IntCmpKind::GeU => CC::NB, 171 } 172 } 173 } 174 175 impl<T: ExtendType> From<Extend<T>> for ExtMode { 176 fn from(value: Extend<T>) -> Self { 177 match value { 178 Extend::I32Extend8 => ExtMode::BL, 179 Extend::I32Extend16 => ExtMode::WL, 180 Extend::I64Extend8 => ExtMode::BQ, 181 Extend::I64Extend16 => ExtMode::WQ, 182 Extend::I64Extend32 => ExtMode::LQ, 183 Extend::__Kind(_) => unreachable!(), 184 } 185 } 186 } 187 188 impl From<ExtendKind> for ExtMode { 189 fn from(value: ExtendKind) -> Self { 190 match value { 191 ExtendKind::Signed(s) => s.into(), 192 ExtendKind::Unsigned(u) => u.into(), 193 } 194 } 195 } 196 197 /// Kinds of extends supported by `vpmov`. 198 pub(super) enum VpmovKind { 199 /// Sign extends 8 lanes of 8-bit integers to 8 lanes of 16-bit integers. 200 E8x8S, 201 /// Zero extends 8 lanes of 8-bit integers to 8 lanes of 16-bit integers. 202 E8x8U, 203 /// Sign extends 4 lanes of 16-bit integers to 4 lanes of 32-bit integers. 204 E16x4S, 205 /// Zero extends 4 lanes of 16-bit integers to 4 lanes of 32-bit integers. 206 E16x4U, 207 /// Sign extends 2 lanes of 32-bit integers to 2 lanes of 64-bit integers. 208 E32x2S, 209 /// Zero extends 2 lanes of 32-bit integers to 2 lanes of 64-bit integers. 210 E32x2U, 211 } 212 213 impl From<V128LoadExtendKind> for VpmovKind { 214 fn from(value: V128LoadExtendKind) -> Self { 215 match value { 216 V128LoadExtendKind::E8x8S => Self::E8x8S, 217 V128LoadExtendKind::E8x8U => Self::E8x8U, 218 V128LoadExtendKind::E16x4S => Self::E16x4S, 219 V128LoadExtendKind::E16x4U => Self::E16x4U, 220 V128LoadExtendKind::E32x2S => Self::E32x2S, 221 V128LoadExtendKind::E32x2U => Self::E32x2U, 222 } 223 } 224 } 225 226 impl From<V128ExtendKind> for VpmovKind { 227 fn from(value: V128ExtendKind) -> Self { 228 match value { 229 V128ExtendKind::LowI8x16S | V128ExtendKind::HighI8x16S => Self::E8x8S, 230 V128ExtendKind::LowI8x16U => Self::E8x8U, 231 V128ExtendKind::LowI16x8S | V128ExtendKind::HighI16x8S => Self::E16x4S, 232 V128ExtendKind::LowI16x8U => Self::E16x4U, 233 V128ExtendKind::LowI32x4S | V128ExtendKind::HighI32x4S => Self::E32x2S, 234 V128ExtendKind::LowI32x4U => Self::E32x2U, 235 _ => unimplemented!(), 236 } 237 } 238 } 239 240 /// Kinds of comparisons supported by `vcmp`. 241 pub(super) enum VcmpKind { 242 /// Equal comparison. 243 Eq, 244 /// Not equal comparison. 245 Ne, 246 /// Less than comparison. 247 Lt, 248 /// Less than or equal comparison. 249 Le, 250 /// Unordered comparison. Sets result to all 1s if either source operand is 251 /// NaN. 252 Unord, 253 } 254 255 /// Kinds of conversions supported by `vcvt`. 256 pub(super) enum VcvtKind { 257 /// Converts 32-bit integers to 32-bit floats. 258 I32ToF32, 259 /// Converts doubleword integers to double precision floats. 260 I32ToF64, 261 /// Converts double precision floats to single precision floats. 262 F64ToF32, 263 // Converts double precision floats to 32-bit integers. 264 F64ToI32, 265 /// Converts single precision floats to double precision floats. 266 F32ToF64, 267 /// Converts single precision floats to 32-bit integers. 268 F32ToI32, 269 } 270 271 /// Modes supported by `vround`. 272 pub(crate) enum VroundMode { 273 /// Rounds toward nearest (ties to even). 274 TowardNearest, 275 /// Rounds toward negative infinity. 276 TowardNegativeInfinity, 277 /// Rounds toward positive infinity. 278 TowardPositiveInfinity, 279 /// Rounds toward zero. 280 TowardZero, 281 } 282 283 /// Low level assembler implementation for x64. 284 pub(crate) struct Assembler { 285 /// The machine instruction buffer. 286 buffer: MachBuffer<Inst>, 287 /// Constant emission information. 288 emit_info: EmitInfo, 289 /// Emission state. 290 emit_state: EmitState, 291 /// x64 flags. 292 isa_flags: x64_settings::Flags, 293 /// Constant pool. 294 pool: ConstantPool, 295 } 296 297 impl Assembler { 298 /// Create a new x64 assembler. 299 pub fn new(shared_flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self { 300 Self { 301 buffer: MachBuffer::<Inst>::new(), 302 emit_state: Default::default(), 303 emit_info: EmitInfo::new(shared_flags, isa_flags.clone()), 304 pool: ConstantPool::new(), 305 isa_flags, 306 } 307 } 308 309 /// Get a mutable reference to underlying 310 /// machine buffer. 311 pub fn buffer_mut(&mut self) -> &mut MachBuffer<Inst> { 312 &mut self.buffer 313 } 314 315 /// Get a reference to the underlying machine buffer. 316 pub fn buffer(&self) -> &MachBuffer<Inst> { 317 &self.buffer 318 } 319 320 /// Adds a constant to the constant pool and returns its address. 321 pub fn add_constant(&mut self, constant: &[u8]) -> Address { 322 let handle = self.pool.register(constant, &mut self.buffer); 323 Address::constant(handle) 324 } 325 326 /// Load a floating point constant, using the constant pool. 327 pub fn load_fp_const(&mut self, dst: WritableReg, constant: &[u8], size: OperandSize) { 328 let addr = self.add_constant(constant); 329 self.xmm_mov_mr(&addr, dst, size, MemFlags::trusted()); 330 } 331 332 /// Return the emitted code. 333 pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized<Final> { 334 let stencil = self 335 .buffer 336 .finish(&self.pool.constants(), self.emit_state.ctrl_plane_mut()); 337 stencil.apply_base_srcloc(loc.unwrap_or_default()) 338 } 339 340 fn emit(&mut self, inst: Inst) { 341 inst.emit(&mut self.buffer, &self.emit_info, &mut self.emit_state); 342 } 343 344 fn to_synthetic_amode(addr: &Address, memflags: MemFlags) -> SyntheticAmode { 345 match *addr { 346 Address::Offset { base, offset } => { 347 let amode = Amode::imm_reg(offset as i32, base.into()).with_flags(memflags); 348 SyntheticAmode::real(amode) 349 } 350 Address::Const(c) => SyntheticAmode::ConstantOffset(c), 351 Address::ImmRegRegShift { 352 simm32, 353 base, 354 index, 355 shift, 356 } => SyntheticAmode::Real(Amode::ImmRegRegShift { 357 simm32, 358 base: base.into(), 359 index: index.into(), 360 shift, 361 flags: memflags, 362 }), 363 } 364 } 365 366 /// Emit an unwind instruction. 367 pub fn unwind_inst(&mut self, inst: UnwindInst) { 368 self.emit(Inst::Unwind { inst }) 369 } 370 371 /// Push register. 372 pub fn push_r(&mut self, reg: Reg) { 373 let inst = asm::inst::pushq_o::new(reg).into(); 374 self.emit(Inst::External { inst }); 375 } 376 377 /// Pop to register. 378 pub fn pop_r(&mut self, dst: WritableReg) { 379 let writable: WritableGpr = dst.map(Into::into); 380 let inst = asm::inst::popq_o::new(writable).into(); 381 self.emit(Inst::External { inst }); 382 } 383 384 /// Return instruction. 385 pub fn ret(&mut self) { 386 let inst = asm::inst::retq_zo::new().into(); 387 self.emit(Inst::External { inst }); 388 } 389 390 /// Register-to-register move. 391 pub fn mov_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 392 let dst: WritableGpr = dst.map(|r| r.into()); 393 let inst = match size { 394 OperandSize::S8 => asm::inst::movb_mr::new(dst, src).into(), 395 OperandSize::S16 => asm::inst::movw_mr::new(dst, src).into(), 396 OperandSize::S32 => asm::inst::movl_mr::new(dst, src).into(), 397 OperandSize::S64 => asm::inst::movq_mr::new(dst, src).into(), 398 _ => unreachable!(), 399 }; 400 self.emit(Inst::External { inst }); 401 } 402 403 /// Register-to-memory move. 404 pub fn mov_rm(&mut self, src: Reg, addr: &Address, size: OperandSize, flags: MemFlags) { 405 assert!(addr.is_offset()); 406 let dst = Self::to_synthetic_amode(addr, flags); 407 let inst = match size { 408 OperandSize::S8 => asm::inst::movb_mr::new(dst, src).into(), 409 OperandSize::S16 => asm::inst::movw_mr::new(dst, src).into(), 410 OperandSize::S32 => asm::inst::movl_mr::new(dst, src).into(), 411 OperandSize::S64 => asm::inst::movq_mr::new(dst, src).into(), 412 _ => unreachable!(), 413 }; 414 self.emit(Inst::External { inst }); 415 } 416 417 /// Immediate-to-memory move. 418 pub fn mov_im(&mut self, src: i32, addr: &Address, size: OperandSize, flags: MemFlags) { 419 assert!(addr.is_offset()); 420 let dst = Self::to_synthetic_amode(addr, flags); 421 let inst = match size { 422 OperandSize::S8 => { 423 let src = i8::try_from(src).unwrap(); 424 asm::inst::movb_mi::new(dst, src.unsigned()).into() 425 } 426 OperandSize::S16 => { 427 let src = i16::try_from(src).unwrap(); 428 asm::inst::movw_mi::new(dst, src.unsigned()).into() 429 } 430 OperandSize::S32 => asm::inst::movl_mi::new(dst, src.unsigned()).into(), 431 OperandSize::S64 => asm::inst::movq_mi_sxl::new(dst, src).into(), 432 _ => unreachable!(), 433 }; 434 self.emit(Inst::External { inst }); 435 } 436 437 /// Immediate-to-register move. 438 pub fn mov_ir(&mut self, imm: u64, dst: WritableReg, size: OperandSize) { 439 self.emit(Inst::imm(size.into(), imm, dst.map(Into::into))); 440 } 441 442 /// Zero-extend memory-to-register load. 443 pub fn movzx_mr( 444 &mut self, 445 addr: &Address, 446 dst: WritableReg, 447 ext: Option<Extend<Zero>>, 448 memflags: MemFlags, 449 ) { 450 let src = Self::to_synthetic_amode(addr, memflags); 451 452 if let Some(ext) = ext { 453 let dst = WritableGpr::from_reg(dst.to_reg().into()); 454 let inst = match ext.into() { 455 ExtMode::BL => asm::inst::movzbl_rm::new(dst, src).into(), 456 ExtMode::BQ => asm::inst::movzbq_rm::new(dst, src).into(), 457 ExtMode::WL => asm::inst::movzwl_rm::new(dst, src).into(), 458 ExtMode::WQ => asm::inst::movzwq_rm::new(dst, src).into(), 459 ExtMode::LQ => { 460 // This instruction selection may seem strange but is 461 // correct in 64-bit mode: section 3.4.1.1 of the Intel 462 // manual says that "32-bit operands generate a 32-bit 463 // result, zero-extended to a 64-bit result in the 464 // destination general-purpose register." This is applicable 465 // beyond `mov` but we use this fact to zero-extend `src` 466 // into `dst`. 467 asm::inst::movl_rm::new(dst, src).into() 468 } 469 }; 470 self.emit(Inst::External { inst }); 471 } else { 472 let dst = WritableGpr::from_reg(dst.to_reg().into()); 473 let inst = asm::inst::movq_rm::new(dst, src).into(); 474 self.emit(Inst::External { inst }); 475 } 476 } 477 478 // Sign-extend memory-to-register load. 479 pub fn movsx_mr( 480 &mut self, 481 addr: &Address, 482 dst: WritableReg, 483 ext: Extend<Signed>, 484 memflags: MemFlags, 485 ) { 486 let src = Self::to_synthetic_amode(addr, memflags); 487 let dst = WritableGpr::from_reg(dst.to_reg().into()); 488 let inst = match ext.into() { 489 ExtMode::BL => asm::inst::movsbl_rm::new(dst, src).into(), 490 ExtMode::BQ => asm::inst::movsbq_rm::new(dst, src).into(), 491 ExtMode::WL => asm::inst::movswl_rm::new(dst, src).into(), 492 ExtMode::WQ => asm::inst::movswq_rm::new(dst, src).into(), 493 ExtMode::LQ => asm::inst::movslq_rm::new(dst, src).into(), 494 }; 495 self.emit(Inst::External { inst }); 496 } 497 498 /// Register-to-register move with zero extension. 499 pub fn movzx_rr(&mut self, src: Reg, dst: WritableReg, kind: Extend<Zero>) { 500 let dst = WritableGpr::from_reg(dst.to_reg().into()); 501 let inst = match kind.into() { 502 ExtMode::BL => asm::inst::movzbl_rm::new(dst, src).into(), 503 ExtMode::BQ => asm::inst::movzbq_rm::new(dst, src).into(), 504 ExtMode::WL => asm::inst::movzwl_rm::new(dst, src).into(), 505 ExtMode::WQ => asm::inst::movzwq_rm::new(dst, src).into(), 506 ExtMode::LQ => { 507 // This instruction selection may seem strange but is correct in 508 // 64-bit mode: section 3.4.1.1 of the Intel manual says that 509 // "32-bit operands generate a 32-bit result, zero-extended to a 510 // 64-bit result in the destination general-purpose register." 511 // This is applicable beyond `mov` but we use this fact to 512 // zero-extend `src` into `dst`. 513 asm::inst::movl_rm::new(dst, src).into() 514 } 515 }; 516 self.emit(Inst::External { inst }); 517 } 518 519 /// Register-to-register move with sign extension. 520 pub fn movsx_rr(&mut self, src: Reg, dst: WritableReg, kind: Extend<Signed>) { 521 let dst = WritableGpr::from_reg(dst.to_reg().into()); 522 let inst = match kind.into() { 523 ExtMode::BL => asm::inst::movsbl_rm::new(dst, src).into(), 524 ExtMode::BQ => asm::inst::movsbq_rm::new(dst, src).into(), 525 ExtMode::WL => asm::inst::movswl_rm::new(dst, src).into(), 526 ExtMode::WQ => asm::inst::movswq_rm::new(dst, src).into(), 527 ExtMode::LQ => asm::inst::movslq_rm::new(dst, src).into(), 528 }; 529 self.emit(Inst::External { inst }); 530 } 531 532 /// Integer register conditional move. 533 pub fn cmov(&mut self, src: Reg, dst: WritableReg, cc: IntCmpKind, size: OperandSize) { 534 use IntCmpKind::*; 535 use OperandSize::*; 536 537 let dst: WritableGpr = dst.map(Into::into); 538 let inst = match size { 539 S8 | S16 | S32 => match cc { 540 Eq => asm::inst::cmovel_rm::new(dst, src).into(), 541 Ne => asm::inst::cmovnel_rm::new(dst, src).into(), 542 LtS => asm::inst::cmovll_rm::new(dst, src).into(), 543 LtU => asm::inst::cmovbl_rm::new(dst, src).into(), 544 GtS => asm::inst::cmovgl_rm::new(dst, src).into(), 545 GtU => asm::inst::cmoval_rm::new(dst, src).into(), 546 LeS => asm::inst::cmovlel_rm::new(dst, src).into(), 547 LeU => asm::inst::cmovbel_rm::new(dst, src).into(), 548 GeS => asm::inst::cmovgel_rm::new(dst, src).into(), 549 GeU => asm::inst::cmovael_rm::new(dst, src).into(), 550 }, 551 S64 => match cc { 552 Eq => asm::inst::cmoveq_rm::new(dst, src).into(), 553 Ne => asm::inst::cmovneq_rm::new(dst, src).into(), 554 LtS => asm::inst::cmovlq_rm::new(dst, src).into(), 555 LtU => asm::inst::cmovbq_rm::new(dst, src).into(), 556 GtS => asm::inst::cmovgq_rm::new(dst, src).into(), 557 GtU => asm::inst::cmovaq_rm::new(dst, src).into(), 558 LeS => asm::inst::cmovleq_rm::new(dst, src).into(), 559 LeU => asm::inst::cmovbeq_rm::new(dst, src).into(), 560 GeS => asm::inst::cmovgeq_rm::new(dst, src).into(), 561 GeU => asm::inst::cmovaeq_rm::new(dst, src).into(), 562 }, 563 _ => unreachable!(), 564 }; 565 self.emit(Inst::External { inst }); 566 } 567 568 /// Single and double precision floating point 569 /// register-to-register move. 570 pub fn xmm_mov_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 571 let ty = match size { 572 OperandSize::S32 => types::F32, 573 OperandSize::S64 => types::F64, 574 OperandSize::S128 => types::I32X4, 575 OperandSize::S8 | OperandSize::S16 => unreachable!(), 576 }; 577 self.emit(Inst::gen_move(dst.map(|r| r.into()), src.into(), ty)); 578 } 579 580 /// Single and double precision floating point load. 581 pub fn xmm_mov_mr( 582 &mut self, 583 src: &Address, 584 dst: WritableReg, 585 size: OperandSize, 586 flags: MemFlags, 587 ) { 588 use OperandSize::*; 589 590 assert!(dst.to_reg().is_float()); 591 592 let src = Self::to_synthetic_amode(src, flags); 593 let dst: WritableXmm = dst.map(|r| r.into()); 594 let inst = match size { 595 S32 => asm::inst::movss_a_m::new(dst, src).into(), 596 S64 => asm::inst::movsd_a_m::new(dst, src).into(), 597 S128 => asm::inst::movdqu_a::new(dst, src).into(), 598 S8 | S16 => unreachable!(), 599 }; 600 self.emit(Inst::External { inst }); 601 } 602 603 /// Vector load and extend. 604 pub fn xmm_vpmov_mr( 605 &mut self, 606 src: &Address, 607 dst: WritableReg, 608 kind: VpmovKind, 609 flags: MemFlags, 610 ) { 611 assert!(dst.to_reg().is_float()); 612 let src = Self::to_synthetic_amode(src, flags); 613 let dst: WritableXmm = dst.map(|r| r.into()); 614 let inst = match kind { 615 VpmovKind::E8x8S => asm::inst::vpmovsxbw_a::new(dst, src).into(), 616 VpmovKind::E8x8U => asm::inst::vpmovzxbw_a::new(dst, src).into(), 617 VpmovKind::E16x4S => asm::inst::vpmovsxwd_a::new(dst, src).into(), 618 VpmovKind::E16x4U => asm::inst::vpmovzxwd_a::new(dst, src).into(), 619 VpmovKind::E32x2S => asm::inst::vpmovsxdq_a::new(dst, src).into(), 620 VpmovKind::E32x2U => asm::inst::vpmovzxdq_a::new(dst, src).into(), 621 }; 622 self.emit(Inst::External { inst }); 623 } 624 625 /// Extends vector of integers in `src` and puts results in `dst`. 626 pub fn xmm_vpmov_rr(&mut self, src: Reg, dst: WritableReg, kind: VpmovKind) { 627 let dst: WritableXmm = dst.map(|r| r.into()); 628 let inst = match kind { 629 VpmovKind::E8x8S => asm::inst::vpmovsxbw_a::new(dst, src).into(), 630 VpmovKind::E8x8U => asm::inst::vpmovzxbw_a::new(dst, src).into(), 631 VpmovKind::E16x4S => asm::inst::vpmovsxwd_a::new(dst, src).into(), 632 VpmovKind::E16x4U => asm::inst::vpmovzxwd_a::new(dst, src).into(), 633 VpmovKind::E32x2S => asm::inst::vpmovsxdq_a::new(dst, src).into(), 634 VpmovKind::E32x2U => asm::inst::vpmovzxdq_a::new(dst, src).into(), 635 }; 636 self.emit(Inst::External { inst }); 637 } 638 639 /// Vector load and broadcast. 640 pub fn xmm_vpbroadcast_mr( 641 &mut self, 642 src: &Address, 643 dst: WritableReg, 644 size: OperandSize, 645 flags: MemFlags, 646 ) { 647 assert!(dst.to_reg().is_float()); 648 let src = Self::to_synthetic_amode(src, flags); 649 let dst: WritableXmm = dst.map(|r| r.into()); 650 let inst = match size { 651 OperandSize::S8 => asm::inst::vpbroadcastb_a::new(dst, src).into(), 652 OperandSize::S16 => asm::inst::vpbroadcastw_a::new(dst, src).into(), 653 OperandSize::S32 => asm::inst::vpbroadcastd_a::new(dst, src).into(), 654 _ => unimplemented!(), 655 }; 656 self.emit(Inst::External { inst }); 657 } 658 659 /// Value in `src` is broadcast into lanes of `size` in `dst`. 660 pub fn xmm_vpbroadcast_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 661 assert!(src.is_float() && dst.to_reg().is_float()); 662 let dst: WritableXmm = dst.map(|r| r.into()); 663 let inst = match size { 664 OperandSize::S8 => asm::inst::vpbroadcastb_a::new(dst, src).into(), 665 OperandSize::S16 => asm::inst::vpbroadcastw_a::new(dst, src).into(), 666 OperandSize::S32 => asm::inst::vpbroadcastd_a::new(dst, src).into(), 667 _ => unimplemented!(), 668 }; 669 self.emit(Inst::External { inst }); 670 } 671 672 /// Memory to register shuffle of bytes in vector. 673 pub fn xmm_vpshuf_mr( 674 &mut self, 675 src: &Address, 676 dst: WritableReg, 677 mask: u8, 678 size: OperandSize, 679 flags: MemFlags, 680 ) { 681 let dst: WritableXmm = dst.map(|r| r.into()); 682 let src = Self::to_synthetic_amode(src, flags); 683 let inst = match size { 684 OperandSize::S32 => asm::inst::vpshufd_a::new(dst, src, mask).into(), 685 _ => unimplemented!(), 686 }; 687 self.emit(Inst::External { inst }); 688 } 689 690 /// Register to register shuffle of bytes in vector. 691 pub fn xmm_vpshuf_rr(&mut self, src: Reg, dst: WritableReg, mask: u8, size: OperandSize) { 692 let dst: WritableXmm = dst.map(|r| r.into()); 693 694 let inst = match size { 695 OperandSize::S16 => asm::inst::vpshuflw_a::new(dst, src, mask).into(), 696 OperandSize::S32 => asm::inst::vpshufd_a::new(dst, src, mask).into(), 697 _ => unimplemented!(), 698 }; 699 700 self.emit(Inst::External { inst }); 701 } 702 703 /// Single and double precision floating point store. 704 pub fn xmm_mov_rm(&mut self, src: Reg, dst: &Address, size: OperandSize, flags: MemFlags) { 705 use OperandSize::*; 706 707 assert!(src.is_float()); 708 709 let dst = Self::to_synthetic_amode(dst, flags); 710 let src: Xmm = src.into(); 711 let inst = match size { 712 S32 => asm::inst::movss_c_m::new(dst, src).into(), 713 S64 => asm::inst::movsd_c_m::new(dst, src).into(), 714 S128 => asm::inst::movdqu_b::new(dst, src).into(), 715 S16 | S8 => unreachable!(), 716 }; 717 self.emit(Inst::External { inst }) 718 } 719 720 /// Floating point register conditional move. 721 pub fn xmm_cmov(&mut self, src: Reg, dst: WritableReg, cc: IntCmpKind, size: OperandSize) { 722 let dst: WritableXmm = dst.map(Into::into); 723 let ty = match size { 724 OperandSize::S32 => types::F32, 725 OperandSize::S64 => types::F64, 726 // Move the entire 128 bits via movdqa. 727 OperandSize::S128 => types::I32X4, 728 OperandSize::S8 | OperandSize::S16 => unreachable!(), 729 }; 730 731 self.emit(Inst::XmmCmove { 732 ty, 733 cc: cc.into(), 734 consequent: Xmm::unwrap_new(src.into()), 735 alternative: dst.to_reg(), 736 dst, 737 }) 738 } 739 740 /// Subtract register and register 741 pub fn sub_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 742 let dst = pair_gpr(dst); 743 let inst = match size { 744 OperandSize::S8 => asm::inst::subb_rm::new(dst, src).into(), 745 OperandSize::S16 => asm::inst::subw_rm::new(dst, src).into(), 746 OperandSize::S32 => asm::inst::subl_rm::new(dst, src).into(), 747 OperandSize::S64 => asm::inst::subq_rm::new(dst, src).into(), 748 OperandSize::S128 => unimplemented!(), 749 }; 750 self.emit(Inst::External { inst }); 751 } 752 753 /// Subtract immediate register. 754 pub fn sub_ir(&mut self, imm: i32, dst: WritableReg, size: OperandSize) { 755 let dst = pair_gpr(dst); 756 let inst = match size { 757 OperandSize::S8 => asm::inst::subb_mi::new(dst, u8::try_from(imm).unwrap()).into(), 758 OperandSize::S16 => asm::inst::subw_mi::new(dst, u16::try_from(imm).unwrap()).into(), 759 OperandSize::S32 => asm::inst::subl_mi::new(dst, imm as u32).into(), 760 OperandSize::S64 => asm::inst::subq_mi_sxl::new(dst, imm).into(), 761 OperandSize::S128 => unimplemented!(), 762 }; 763 self.emit(Inst::External { inst }); 764 } 765 766 /// "and" two registers. 767 pub fn and_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 768 let dst = pair_gpr(dst); 769 let inst = match size { 770 OperandSize::S8 => asm::inst::andb_rm::new(dst, src).into(), 771 OperandSize::S16 => asm::inst::andw_rm::new(dst, src).into(), 772 OperandSize::S32 => asm::inst::andl_rm::new(dst, src).into(), 773 OperandSize::S64 => asm::inst::andq_rm::new(dst, src).into(), 774 OperandSize::S128 => unimplemented!(), 775 }; 776 self.emit(Inst::External { inst }); 777 } 778 779 pub fn and_ir(&mut self, imm: i32, dst: WritableReg, size: OperandSize) { 780 let dst = pair_gpr(dst); 781 let inst = match size { 782 OperandSize::S8 => asm::inst::andb_mi::new(dst, u8::try_from(imm).unwrap()).into(), 783 OperandSize::S16 => asm::inst::andw_mi::new(dst, u16::try_from(imm).unwrap()).into(), 784 OperandSize::S32 => asm::inst::andl_mi::new(dst, imm as u32).into(), 785 OperandSize::S64 => asm::inst::andq_mi_sxl::new(dst, imm).into(), 786 OperandSize::S128 => unimplemented!(), 787 }; 788 self.emit(Inst::External { inst }); 789 } 790 791 /// "and" two float registers. 792 pub fn xmm_and_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 793 let dst = pair_xmm(dst); 794 let inst = match size { 795 OperandSize::S32 => asm::inst::andps_a::new(dst, src).into(), 796 OperandSize::S64 => asm::inst::andpd_a::new(dst, src).into(), 797 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 798 }; 799 self.emit(Inst::External { inst }); 800 } 801 802 /// "and not" two float registers. 803 pub fn xmm_andn_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 804 let dst = pair_xmm(dst); 805 let inst = match size { 806 OperandSize::S32 => asm::inst::andnps_a::new(dst, src).into(), 807 OperandSize::S64 => asm::inst::andnpd_a::new(dst, src).into(), 808 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 809 }; 810 self.emit(Inst::External { inst }); 811 } 812 813 pub fn gpr_to_xmm(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 814 let dst: WritableXmm = dst.map(|r| r.into()); 815 let inst = match size { 816 OperandSize::S32 => asm::inst::movd_a::new(dst, src).into(), 817 OperandSize::S64 => asm::inst::movq_a::new(dst, src).into(), 818 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 819 }; 820 821 self.emit(Inst::External { inst }); 822 } 823 824 pub fn xmm_to_gpr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 825 let dst: WritableGpr = dst.map(Into::into); 826 let src: Xmm = src.into(); 827 let inst = match size { 828 OperandSize::S32 => asm::inst::movd_b::new(dst, src).into(), 829 OperandSize::S64 => asm::inst::movq_b::new(dst, src).into(), 830 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 831 }; 832 833 self.emit(Inst::External { inst }) 834 } 835 836 /// Convert float to signed int. 837 pub fn cvt_float_to_sint_seq( 838 &mut self, 839 src: Reg, 840 dst: WritableReg, 841 tmp_gpr: Reg, 842 tmp_xmm: Reg, 843 src_size: OperandSize, 844 dst_size: OperandSize, 845 saturating: bool, 846 ) { 847 self.emit(Inst::CvtFloatToSintSeq { 848 dst_size: dst_size.into(), 849 src_size: src_size.into(), 850 is_saturating: saturating, 851 src: src.into(), 852 dst: dst.map(Into::into), 853 tmp_gpr: tmp_gpr.into(), 854 tmp_xmm: tmp_xmm.into(), 855 }); 856 } 857 858 /// Convert float to unsigned int. 859 pub fn cvt_float_to_uint_seq( 860 &mut self, 861 src: Reg, 862 dst: WritableReg, 863 tmp_gpr: Reg, 864 tmp_xmm: Reg, 865 tmp_xmm2: Reg, 866 src_size: OperandSize, 867 dst_size: OperandSize, 868 saturating: bool, 869 ) { 870 self.emit(Inst::CvtFloatToUintSeq { 871 dst_size: dst_size.into(), 872 src_size: src_size.into(), 873 is_saturating: saturating, 874 src: src.into(), 875 dst: dst.map(Into::into), 876 tmp_gpr: tmp_gpr.into(), 877 tmp_xmm: tmp_xmm.into(), 878 tmp_xmm2: tmp_xmm2.into(), 879 }); 880 } 881 882 /// Convert signed int to float. 883 pub fn cvt_sint_to_float( 884 &mut self, 885 src: Reg, 886 dst: WritableReg, 887 src_size: OperandSize, 888 dst_size: OperandSize, 889 ) { 890 use OperandSize::*; 891 let dst = pair_xmm(dst); 892 let inst = match (src_size, dst_size) { 893 (S32, S32) => asm::inst::cvtsi2ssl_a::new(dst, src).into(), 894 (S32, S64) => asm::inst::cvtsi2sdl_a::new(dst, src).into(), 895 (S64, S32) => asm::inst::cvtsi2ssq_a::new(dst, src).into(), 896 (S64, S64) => asm::inst::cvtsi2sdq_a::new(dst, src).into(), 897 _ => unreachable!(), 898 }; 899 self.emit(Inst::External { inst }); 900 } 901 902 /// Convert unsigned 64-bit int to float. 903 pub fn cvt_uint64_to_float_seq( 904 &mut self, 905 src: Reg, 906 dst: WritableReg, 907 tmp_gpr1: Reg, 908 tmp_gpr2: Reg, 909 dst_size: OperandSize, 910 ) { 911 self.emit(Inst::CvtUint64ToFloatSeq { 912 dst_size: dst_size.into(), 913 src: src.into(), 914 dst: dst.map(Into::into), 915 tmp_gpr1: tmp_gpr1.into(), 916 tmp_gpr2: tmp_gpr2.into(), 917 }); 918 } 919 920 /// Change precision of float. 921 pub fn cvt_float_to_float( 922 &mut self, 923 src: Reg, 924 dst: WritableReg, 925 src_size: OperandSize, 926 dst_size: OperandSize, 927 ) { 928 use OperandSize::*; 929 let dst = pair_xmm(dst); 930 let inst = match (src_size, dst_size) { 931 (S32, S64) => asm::inst::cvtss2sd_a::new(dst, src).into(), 932 (S64, S32) => asm::inst::cvtsd2ss_a::new(dst, src).into(), 933 _ => unimplemented!(), 934 }; 935 self.emit(Inst::External { inst }); 936 } 937 938 pub fn or_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 939 let dst = pair_gpr(dst); 940 let inst = match size { 941 OperandSize::S8 => asm::inst::orb_rm::new(dst, src).into(), 942 OperandSize::S16 => asm::inst::orw_rm::new(dst, src).into(), 943 OperandSize::S32 => asm::inst::orl_rm::new(dst, src).into(), 944 OperandSize::S64 => asm::inst::orq_rm::new(dst, src).into(), 945 OperandSize::S128 => unimplemented!(), 946 }; 947 self.emit(Inst::External { inst }); 948 } 949 950 pub fn or_ir(&mut self, imm: i32, dst: WritableReg, size: OperandSize) { 951 let dst = pair_gpr(dst); 952 let inst = match size { 953 OperandSize::S8 => asm::inst::orb_mi::new(dst, u8::try_from(imm).unwrap()).into(), 954 OperandSize::S16 => asm::inst::orw_mi::new(dst, u16::try_from(imm).unwrap()).into(), 955 OperandSize::S32 => asm::inst::orl_mi::new(dst, imm as u32).into(), 956 OperandSize::S64 => asm::inst::orq_mi_sxl::new(dst, imm).into(), 957 OperandSize::S128 => unimplemented!(), 958 }; 959 self.emit(Inst::External { inst }); 960 } 961 962 pub fn xmm_or_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 963 let dst = pair_xmm(dst); 964 let inst = match size { 965 OperandSize::S32 => asm::inst::orps_a::new(dst, src).into(), 966 OperandSize::S64 => asm::inst::orpd_a::new(dst, src).into(), 967 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 968 }; 969 self.emit(Inst::External { inst }); 970 } 971 972 /// Logical exclusive or with registers. 973 pub fn xor_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 974 let dst = pair_gpr(dst); 975 let inst = match size { 976 OperandSize::S8 => asm::inst::xorb_rm::new(dst, src).into(), 977 OperandSize::S16 => asm::inst::xorw_rm::new(dst, src).into(), 978 OperandSize::S32 => asm::inst::xorl_rm::new(dst, src).into(), 979 OperandSize::S64 => asm::inst::xorq_rm::new(dst, src).into(), 980 OperandSize::S128 => unimplemented!(), 981 }; 982 self.emit(Inst::External { inst }); 983 } 984 985 pub fn xor_ir(&mut self, imm: i32, dst: WritableReg, size: OperandSize) { 986 let dst = pair_gpr(dst); 987 let inst = match size { 988 OperandSize::S8 => asm::inst::xorb_mi::new(dst, u8::try_from(imm).unwrap()).into(), 989 OperandSize::S16 => asm::inst::xorw_mi::new(dst, u16::try_from(imm).unwrap()).into(), 990 OperandSize::S32 => asm::inst::xorl_mi::new(dst, imm as u32).into(), 991 OperandSize::S64 => asm::inst::xorq_mi_sxl::new(dst, imm).into(), 992 OperandSize::S128 => unimplemented!(), 993 }; 994 self.emit(Inst::External { inst }); 995 } 996 997 /// Logical exclusive or with float registers. 998 pub fn xmm_xor_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 999 let dst = pair_xmm(dst); 1000 let inst = match size { 1001 OperandSize::S32 => asm::inst::xorps_a::new(dst, src).into(), 1002 OperandSize::S64 => asm::inst::xorpd_a::new(dst, src).into(), 1003 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 1004 }; 1005 self.emit(Inst::External { inst }); 1006 } 1007 1008 /// Shift with register and register. 1009 pub fn shift_rr(&mut self, src: Reg, dst: WritableReg, kind: ShiftKind, size: OperandSize) { 1010 let dst = pair_gpr(dst); 1011 let src: Gpr = src.into(); 1012 let inst = match (kind, size) { 1013 (ShiftKind::Shl, OperandSize::S32) => asm::inst::shll_mc::new(dst, src).into(), 1014 (ShiftKind::Shl, OperandSize::S64) => asm::inst::shlq_mc::new(dst, src).into(), 1015 (ShiftKind::Shl, _) => todo!(), 1016 (ShiftKind::ShrS, OperandSize::S32) => asm::inst::sarl_mc::new(dst, src).into(), 1017 (ShiftKind::ShrS, OperandSize::S64) => asm::inst::sarq_mc::new(dst, src).into(), 1018 (ShiftKind::ShrS, _) => todo!(), 1019 (ShiftKind::ShrU, OperandSize::S32) => asm::inst::shrl_mc::new(dst, src).into(), 1020 (ShiftKind::ShrU, OperandSize::S64) => asm::inst::shrq_mc::new(dst, src).into(), 1021 (ShiftKind::ShrU, _) => todo!(), 1022 (ShiftKind::Rotl, OperandSize::S32) => asm::inst::roll_mc::new(dst, src).into(), 1023 (ShiftKind::Rotl, OperandSize::S64) => asm::inst::rolq_mc::new(dst, src).into(), 1024 (ShiftKind::Rotl, _) => todo!(), 1025 (ShiftKind::Rotr, OperandSize::S32) => asm::inst::rorl_mc::new(dst, src).into(), 1026 (ShiftKind::Rotr, OperandSize::S64) => asm::inst::rorq_mc::new(dst, src).into(), 1027 (ShiftKind::Rotr, _) => todo!(), 1028 }; 1029 self.emit(Inst::External { inst }); 1030 } 1031 1032 /// Shift with immediate and register. 1033 pub fn shift_ir(&mut self, imm: u8, dst: WritableReg, kind: ShiftKind, size: OperandSize) { 1034 let dst = pair_gpr(dst); 1035 let inst = match (kind, size) { 1036 (ShiftKind::Shl, OperandSize::S32) => asm::inst::shll_mi::new(dst, imm).into(), 1037 (ShiftKind::Shl, OperandSize::S64) => asm::inst::shlq_mi::new(dst, imm).into(), 1038 (ShiftKind::Shl, _) => todo!(), 1039 (ShiftKind::ShrS, OperandSize::S32) => asm::inst::sarl_mi::new(dst, imm).into(), 1040 (ShiftKind::ShrS, OperandSize::S64) => asm::inst::sarq_mi::new(dst, imm).into(), 1041 (ShiftKind::ShrS, _) => todo!(), 1042 (ShiftKind::ShrU, OperandSize::S32) => asm::inst::shrl_mi::new(dst, imm).into(), 1043 (ShiftKind::ShrU, OperandSize::S64) => asm::inst::shrq_mi::new(dst, imm).into(), 1044 (ShiftKind::ShrU, _) => todo!(), 1045 (ShiftKind::Rotl, OperandSize::S32) => asm::inst::roll_mi::new(dst, imm).into(), 1046 (ShiftKind::Rotl, OperandSize::S64) => asm::inst::rolq_mi::new(dst, imm).into(), 1047 (ShiftKind::Rotl, _) => todo!(), 1048 (ShiftKind::Rotr, OperandSize::S32) => asm::inst::rorl_mi::new(dst, imm).into(), 1049 (ShiftKind::Rotr, OperandSize::S64) => asm::inst::rorq_mi::new(dst, imm).into(), 1050 (ShiftKind::Rotr, _) => todo!(), 1051 }; 1052 self.emit(Inst::External { inst }); 1053 } 1054 1055 /// Signed/unsigned division. 1056 /// 1057 /// Emits a sequence of instructions to ensure the correctness of 1058 /// the division invariants. This function assumes that the 1059 /// caller has correctly allocated the dividend as `(rdx:rax)` and 1060 /// accounted for the quotient to be stored in `rax`. 1061 pub fn div(&mut self, divisor: Reg, dst: (Reg, Reg), kind: DivKind, size: OperandSize) { 1062 let trap = match kind { 1063 // Signed division has two trapping conditions, integer overflow and 1064 // divide-by-zero. Check for divide-by-zero explicitly and let the 1065 // hardware detect overflow. 1066 DivKind::Signed => { 1067 self.cmp_ir(divisor, 0, size); 1068 self.emit(Inst::TrapIf { 1069 cc: CC::Z, 1070 trap_code: TrapCode::INTEGER_DIVISION_BY_ZERO, 1071 }); 1072 1073 // Sign-extend the dividend with tailor-made instructoins for 1074 // just this operation. 1075 let ext_dst: WritableGpr = dst.1.into(); 1076 let ext_src: Gpr = dst.0.into(); 1077 let inst = match size { 1078 OperandSize::S32 => asm::inst::cltd_zo::new(ext_dst, ext_src).into(), 1079 OperandSize::S64 => asm::inst::cqto_zo::new(ext_dst, ext_src).into(), 1080 _ => unimplemented!(), 1081 }; 1082 self.emit(Inst::External { inst }); 1083 TrapCode::INTEGER_OVERFLOW 1084 } 1085 1086 // Unsigned division only traps in one case, on divide-by-zero, so 1087 // defer that to the trap opcode. 1088 // 1089 // The divisor_hi reg is initialized with zero through an 1090 // xor-against-itself op. 1091 DivKind::Unsigned => { 1092 self.xor_rr(dst.1, writable!(dst.1), size); 1093 TrapCode::INTEGER_DIVISION_BY_ZERO 1094 } 1095 }; 1096 let dst0 = pair_gpr(writable!(dst.0)); 1097 let dst1 = pair_gpr(writable!(dst.1)); 1098 let inst = match (kind, size) { 1099 (DivKind::Signed, OperandSize::S32) => { 1100 asm::inst::idivl_m::new(dst0, dst1, divisor, trap).into() 1101 } 1102 (DivKind::Unsigned, OperandSize::S32) => { 1103 asm::inst::divl_m::new(dst0, dst1, divisor, trap).into() 1104 } 1105 (DivKind::Signed, OperandSize::S64) => { 1106 asm::inst::idivq_m::new(dst0, dst1, divisor, trap).into() 1107 } 1108 (DivKind::Unsigned, OperandSize::S64) => { 1109 asm::inst::divq_m::new(dst0, dst1, divisor, trap).into() 1110 } 1111 _ => todo!(), 1112 }; 1113 self.emit(Inst::External { inst }); 1114 } 1115 1116 /// Signed/unsigned remainder. 1117 /// 1118 /// Emits a sequence of instructions to ensure the correctness of the 1119 /// division invariants and ultimately calculate the remainder. 1120 /// This function assumes that the 1121 /// caller has correctly allocated the dividend as `(rdx:rax)` and 1122 /// accounted for the remainder to be stored in `rdx`. 1123 pub fn rem(&mut self, divisor: Reg, dst: (Reg, Reg), kind: RemKind, size: OperandSize) { 1124 match kind { 1125 // Signed remainder goes through a pseudo-instruction which has 1126 // some internal branching. The `dividend_hi`, or `rdx`, is 1127 // initialized here with a `SignExtendData` instruction. 1128 RemKind::Signed => { 1129 let ext_dst: WritableGpr = dst.1.into(); 1130 1131 // Initialize `dividend_hi`, or `rdx`, with a tailor-made 1132 // instruction for this operation. 1133 let ext_src: Gpr = dst.0.into(); 1134 let inst = match size { 1135 OperandSize::S32 => asm::inst::cltd_zo::new(ext_dst, ext_src).into(), 1136 OperandSize::S64 => asm::inst::cqto_zo::new(ext_dst, ext_src).into(), 1137 _ => unimplemented!(), 1138 }; 1139 self.emit(Inst::External { inst }); 1140 self.emit(Inst::CheckedSRemSeq { 1141 size: size.into(), 1142 divisor: divisor.into(), 1143 dividend_lo: dst.0.into(), 1144 dividend_hi: dst.1.into(), 1145 dst_quotient: dst.0.into(), 1146 dst_remainder: dst.1.into(), 1147 }); 1148 } 1149 1150 // Unsigned remainder initializes `dividend_hi` with zero and 1151 // then executes a normal `div` instruction. 1152 RemKind::Unsigned => { 1153 self.xor_rr(dst.1, writable!(dst.1), size); 1154 let dst0 = pair_gpr(writable!(dst.0)); 1155 let dst1 = pair_gpr(writable!(dst.1)); 1156 let trap = TrapCode::INTEGER_DIVISION_BY_ZERO; 1157 let inst = match size { 1158 OperandSize::S32 => asm::inst::divl_m::new(dst0, dst1, divisor, trap).into(), 1159 OperandSize::S64 => asm::inst::divq_m::new(dst0, dst1, divisor, trap).into(), 1160 _ => todo!(), 1161 }; 1162 self.emit(Inst::External { inst }); 1163 } 1164 } 1165 } 1166 1167 /// Multiply immediate and register. 1168 pub fn mul_ir(&mut self, imm: i32, dst: WritableReg, size: OperandSize) { 1169 use OperandSize::*; 1170 let src = dst.to_reg(); 1171 let dst: WritableGpr = dst.to_reg().into(); 1172 let inst = match size { 1173 S16 => asm::inst::imulw_rmi::new(dst, src, u16::try_from(imm).unwrap()).into(), 1174 S32 => asm::inst::imull_rmi::new(dst, src, imm as u32).into(), 1175 S64 => asm::inst::imulq_rmi_sxl::new(dst, src, imm).into(), 1176 S8 | S128 => unimplemented!(), 1177 }; 1178 self.emit(Inst::External { inst }); 1179 } 1180 1181 /// Multiply register and register. 1182 pub fn mul_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1183 use OperandSize::*; 1184 let dst = pair_gpr(dst); 1185 let inst = match size { 1186 S16 => asm::inst::imulw_rm::new(dst, src).into(), 1187 S32 => asm::inst::imull_rm::new(dst, src).into(), 1188 S64 => asm::inst::imulq_rm::new(dst, src).into(), 1189 S8 | S128 => unimplemented!(), 1190 }; 1191 self.emit(Inst::External { inst }); 1192 } 1193 1194 /// Add immediate and register. 1195 pub fn add_ir(&mut self, imm: i32, dst: WritableReg, size: OperandSize) { 1196 let dst = pair_gpr(dst); 1197 let inst = match size { 1198 OperandSize::S8 => asm::inst::addb_mi::new(dst, u8::try_from(imm).unwrap()).into(), 1199 OperandSize::S16 => asm::inst::addw_mi::new(dst, u16::try_from(imm).unwrap()).into(), 1200 OperandSize::S32 => asm::inst::addl_mi::new(dst, imm as u32).into(), 1201 OperandSize::S64 => asm::inst::addq_mi_sxl::new(dst, imm).into(), 1202 OperandSize::S128 => unimplemented!(), 1203 }; 1204 self.emit(Inst::External { inst }); 1205 } 1206 1207 /// Add register and register. 1208 pub fn add_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1209 let dst = pair_gpr(dst); 1210 let inst = match size { 1211 OperandSize::S8 => asm::inst::addb_rm::new(dst, src).into(), 1212 OperandSize::S16 => asm::inst::addw_rm::new(dst, src).into(), 1213 OperandSize::S32 => asm::inst::addl_rm::new(dst, src).into(), 1214 OperandSize::S64 => asm::inst::addq_rm::new(dst, src).into(), 1215 OperandSize::S128 => unimplemented!(), 1216 }; 1217 self.emit(Inst::External { inst }); 1218 } 1219 1220 pub fn lock_xadd( 1221 &mut self, 1222 addr: Address, 1223 dst: WritableReg, 1224 size: OperandSize, 1225 flags: MemFlags, 1226 ) { 1227 assert!(addr.is_offset()); 1228 let mem = Self::to_synthetic_amode(&addr, flags); 1229 let dst = pair_gpr(dst); 1230 let inst = match size { 1231 OperandSize::S8 => asm::inst::lock_xaddb_mr::new(mem, dst).into(), 1232 OperandSize::S16 => asm::inst::lock_xaddw_mr::new(mem, dst).into(), 1233 OperandSize::S32 => asm::inst::lock_xaddl_mr::new(mem, dst).into(), 1234 OperandSize::S64 => asm::inst::lock_xaddq_mr::new(mem, dst).into(), 1235 OperandSize::S128 => unimplemented!(), 1236 }; 1237 1238 self.emit(Inst::External { inst }); 1239 } 1240 1241 pub fn atomic_rmw_seq( 1242 &mut self, 1243 addr: Address, 1244 operand: Reg, 1245 dst: WritableReg, 1246 temp: WritableReg, 1247 size: OperandSize, 1248 flags: MemFlags, 1249 op: AtomicRmwSeqOp, 1250 ) { 1251 assert!(addr.is_offset()); 1252 let mem = Self::to_synthetic_amode(&addr, flags); 1253 self.emit(Inst::AtomicRmwSeq { 1254 ty: Type::int_with_byte_size(size.bytes() as _).unwrap(), 1255 mem, 1256 operand: operand.into(), 1257 temp: temp.map(Into::into), 1258 dst_old: dst.map(Into::into), 1259 op, 1260 }); 1261 } 1262 1263 pub fn xchg(&mut self, addr: Address, dst: WritableReg, size: OperandSize, flags: MemFlags) { 1264 assert!(addr.is_offset()); 1265 let mem = Self::to_synthetic_amode(&addr, flags); 1266 let dst = pair_gpr(dst); 1267 let inst = match size { 1268 OperandSize::S8 => asm::inst::xchgb_rm::new(dst, mem).into(), 1269 OperandSize::S16 => asm::inst::xchgw_rm::new(dst, mem).into(), 1270 OperandSize::S32 => asm::inst::xchgl_rm::new(dst, mem).into(), 1271 OperandSize::S64 => asm::inst::xchgq_rm::new(dst, mem).into(), 1272 OperandSize::S128 => unimplemented!(), 1273 }; 1274 1275 self.emit(Inst::External { inst }); 1276 } 1277 pub fn cmpxchg( 1278 &mut self, 1279 addr: Address, 1280 replacement: Reg, 1281 dst: WritableReg, 1282 size: OperandSize, 1283 flags: MemFlags, 1284 ) { 1285 assert!(addr.is_offset()); 1286 let mem = Self::to_synthetic_amode(&addr, flags); 1287 let dst = pair_gpr(dst); 1288 let inst = match size { 1289 OperandSize::S8 => asm::inst::lock_cmpxchgb_mr::new(mem, replacement, dst).into(), 1290 OperandSize::S16 => asm::inst::lock_cmpxchgw_mr::new(mem, replacement, dst).into(), 1291 OperandSize::S32 => asm::inst::lock_cmpxchgl_mr::new(mem, replacement, dst).into(), 1292 OperandSize::S64 => asm::inst::lock_cmpxchgq_mr::new(mem, replacement, dst).into(), 1293 OperandSize::S128 => unimplemented!(), 1294 }; 1295 1296 self.emit(Inst::External { inst }); 1297 } 1298 1299 pub fn cmp_ir(&mut self, src1: Reg, imm: i32, size: OperandSize) { 1300 let inst = match size { 1301 OperandSize::S8 => { 1302 let imm = i8::try_from(imm).unwrap(); 1303 asm::inst::cmpb_mi::new(src1, imm.unsigned()).into() 1304 } 1305 OperandSize::S16 => match i8::try_from(imm) { 1306 Ok(imm8) => asm::inst::cmpw_mi_sxb::new(src1, imm8).into(), 1307 Err(_) => { 1308 asm::inst::cmpw_mi::new(src1, i16::try_from(imm).unwrap().unsigned()).into() 1309 } 1310 }, 1311 OperandSize::S32 => match i8::try_from(imm) { 1312 Ok(imm8) => asm::inst::cmpl_mi_sxb::new(src1, imm8).into(), 1313 Err(_) => asm::inst::cmpl_mi::new(src1, imm.unsigned()).into(), 1314 }, 1315 OperandSize::S64 => match i8::try_from(imm) { 1316 Ok(imm8) => asm::inst::cmpq_mi_sxb::new(src1, imm8).into(), 1317 Err(_) => asm::inst::cmpq_mi::new(src1, imm).into(), 1318 }, 1319 OperandSize::S128 => unimplemented!(), 1320 }; 1321 1322 self.emit(Inst::External { inst }); 1323 } 1324 1325 pub fn cmp_rr(&mut self, src1: Reg, src2: Reg, size: OperandSize) { 1326 let inst = match size { 1327 OperandSize::S8 => asm::inst::cmpb_rm::new(src1, src2).into(), 1328 OperandSize::S16 => asm::inst::cmpw_rm::new(src1, src2).into(), 1329 OperandSize::S32 => asm::inst::cmpl_rm::new(src1, src2).into(), 1330 OperandSize::S64 => asm::inst::cmpq_rm::new(src1, src2).into(), 1331 OperandSize::S128 => unimplemented!(), 1332 }; 1333 1334 self.emit(Inst::External { inst }); 1335 } 1336 1337 /// Compares values in src1 and src2 and sets ZF, PF, and CF flags in EFLAGS 1338 /// register. 1339 pub fn ucomis(&mut self, src1: Reg, src2: Reg, size: OperandSize) { 1340 let inst = match size { 1341 OperandSize::S32 => asm::inst::ucomiss_a::new(src1, src2).into(), 1342 OperandSize::S64 => asm::inst::ucomisd_a::new(src1, src2).into(), 1343 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 1344 }; 1345 self.emit(Inst::External { inst }); 1346 } 1347 1348 pub fn popcnt(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1349 assert!( 1350 self.isa_flags.has_popcnt() && self.isa_flags.has_sse42(), 1351 "Requires has_popcnt and has_sse42 flags" 1352 ); 1353 let dst = WritableGpr::from_reg(dst.to_reg().into()); 1354 let inst = match size { 1355 OperandSize::S16 => asm::inst::popcntw_rm::new(dst, src).into(), 1356 OperandSize::S32 => asm::inst::popcntl_rm::new(dst, src).into(), 1357 OperandSize::S64 => asm::inst::popcntq_rm::new(dst, src).into(), 1358 OperandSize::S8 | OperandSize::S128 => unreachable!(), 1359 }; 1360 self.emit(Inst::External { inst }); 1361 } 1362 1363 /// Emit a test instruction with two register operands. 1364 pub fn test_rr(&mut self, src1: Reg, src2: Reg, size: OperandSize) { 1365 let inst = match size { 1366 OperandSize::S8 => asm::inst::testb_mr::new(src1, src2).into(), 1367 OperandSize::S16 => asm::inst::testw_mr::new(src1, src2).into(), 1368 OperandSize::S32 => asm::inst::testl_mr::new(src1, src2).into(), 1369 OperandSize::S64 => asm::inst::testq_mr::new(src1, src2).into(), 1370 OperandSize::S128 => unimplemented!(), 1371 }; 1372 1373 self.emit(Inst::External { inst }); 1374 } 1375 1376 /// Set value in dst to `0` or `1` based on flags in status register and 1377 /// [`CmpKind`]. 1378 pub fn setcc(&mut self, kind: IntCmpKind, dst: WritableReg) { 1379 self.setcc_impl(kind.into(), dst); 1380 } 1381 1382 /// Set value in dst to `1` if parity flag in status register is set, `0` 1383 /// otherwise. 1384 pub fn setp(&mut self, dst: WritableReg) { 1385 self.setcc_impl(CC::P, dst); 1386 } 1387 1388 /// Set value in dst to `1` if parity flag in status register is not set, 1389 /// `0` otherwise. 1390 pub fn setnp(&mut self, dst: WritableReg) { 1391 self.setcc_impl(CC::NP, dst); 1392 } 1393 1394 fn setcc_impl(&mut self, cc: CC, dst: WritableReg) { 1395 // Clear the dst register or bits 1 to 31 may be incorrectly set. 1396 // Don't use xor since it updates the status register. 1397 let dst: WritableGpr = dst.map(Into::into); 1398 let inst = asm::inst::movl_oi::new(dst, 0).into(); 1399 self.emit(Inst::External { inst }); 1400 1401 // Copy correct bit from status register into dst register. 1402 // 1403 // Note that some of these mnemonics don't match exactly and that's 1404 // intentional as there are multiple mnemonics for the same encoding in 1405 // some cases and the assembler picked ones that match Capstone rather 1406 // than Cranelift. 1407 let inst = match cc { 1408 CC::O => asm::inst::seto_m::new(dst).into(), 1409 CC::NO => asm::inst::setno_m::new(dst).into(), 1410 CC::B => asm::inst::setb_m::new(dst).into(), 1411 CC::NB => asm::inst::setae_m::new(dst).into(), // nb == ae 1412 CC::Z => asm::inst::sete_m::new(dst).into(), // z == e 1413 CC::NZ => asm::inst::setne_m::new(dst).into(), // nz == ne 1414 CC::BE => asm::inst::setbe_m::new(dst).into(), 1415 CC::NBE => asm::inst::seta_m::new(dst).into(), // nbe == a 1416 CC::S => asm::inst::sets_m::new(dst).into(), 1417 CC::NS => asm::inst::setns_m::new(dst).into(), 1418 CC::L => asm::inst::setl_m::new(dst).into(), 1419 CC::NL => asm::inst::setge_m::new(dst).into(), // nl == ge 1420 CC::LE => asm::inst::setle_m::new(dst).into(), 1421 CC::NLE => asm::inst::setg_m::new(dst).into(), // nle == g 1422 CC::P => asm::inst::setp_m::new(dst).into(), 1423 CC::NP => asm::inst::setnp_m::new(dst).into(), 1424 }; 1425 self.emit(Inst::External { inst }); 1426 } 1427 1428 /// Store the count of leading zeroes in src in dst. 1429 /// Requires `has_lzcnt` flag. 1430 pub fn lzcnt(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1431 assert!(self.isa_flags.has_lzcnt(), "Requires has_lzcnt flag"); 1432 let dst = WritableGpr::from_reg(dst.to_reg().into()); 1433 let inst = match size { 1434 OperandSize::S16 => asm::inst::lzcntw_rm::new(dst, src).into(), 1435 OperandSize::S32 => asm::inst::lzcntl_rm::new(dst, src).into(), 1436 OperandSize::S64 => asm::inst::lzcntq_rm::new(dst, src).into(), 1437 OperandSize::S8 | OperandSize::S128 => unreachable!(), 1438 }; 1439 self.emit(Inst::External { inst }); 1440 } 1441 1442 /// Store the count of trailing zeroes in src in dst. 1443 /// Requires `has_bmi1` flag. 1444 pub fn tzcnt(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1445 assert!(self.isa_flags.has_bmi1(), "Requires has_bmi1 flag"); 1446 let dst = WritableGpr::from_reg(dst.to_reg().into()); 1447 let inst = match size { 1448 OperandSize::S16 => asm::inst::tzcntw_a::new(dst, src).into(), 1449 OperandSize::S32 => asm::inst::tzcntl_a::new(dst, src).into(), 1450 OperandSize::S64 => asm::inst::tzcntq_a::new(dst, src).into(), 1451 OperandSize::S8 | OperandSize::S128 => unreachable!(), 1452 }; 1453 self.emit(Inst::External { inst }); 1454 } 1455 1456 /// Stores position of the most significant bit set in src in dst. 1457 /// Zero flag is set if src is equal to 0. 1458 pub fn bsr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1459 let dst: WritableGpr = WritableGpr::from_reg(dst.to_reg().into()); 1460 let inst = match size { 1461 OperandSize::S16 => asm::inst::bsrw_rm::new(dst, src).into(), 1462 OperandSize::S32 => asm::inst::bsrl_rm::new(dst, src).into(), 1463 OperandSize::S64 => asm::inst::bsrq_rm::new(dst, src).into(), 1464 OperandSize::S8 | OperandSize::S128 => unreachable!(), 1465 }; 1466 self.emit(Inst::External { inst }); 1467 } 1468 1469 /// Performs integer negation on `src` and places result in `dst`. 1470 pub fn neg(&mut self, read: Reg, write: WritableReg, size: OperandSize) { 1471 let gpr = PairedGpr { 1472 read: read.into(), 1473 write: WritableGpr::from_reg(write.to_reg().into()), 1474 }; 1475 let inst = match size { 1476 OperandSize::S8 => asm::inst::negb_m::new(gpr).into(), 1477 OperandSize::S16 => asm::inst::negw_m::new(gpr).into(), 1478 OperandSize::S32 => asm::inst::negl_m::new(gpr).into(), 1479 OperandSize::S64 => asm::inst::negq_m::new(gpr).into(), 1480 OperandSize::S128 => unreachable!(), 1481 }; 1482 self.emit(Inst::External { inst }); 1483 } 1484 1485 /// Stores position of the least significant bit set in src in dst. 1486 /// Zero flag is set if src is equal to 0. 1487 pub fn bsf(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1488 let dst: WritableGpr = WritableGpr::from_reg(dst.to_reg().into()); 1489 let inst = match size { 1490 OperandSize::S16 => asm::inst::bsfw_rm::new(dst, src).into(), 1491 OperandSize::S32 => asm::inst::bsfl_rm::new(dst, src).into(), 1492 OperandSize::S64 => asm::inst::bsfq_rm::new(dst, src).into(), 1493 OperandSize::S8 | OperandSize::S128 => unreachable!(), 1494 }; 1495 self.emit(Inst::External { inst }); 1496 } 1497 1498 /// Performs float addition on src and dst and places result in dst. 1499 pub fn xmm_add_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1500 let dst = pair_xmm(dst); 1501 let inst = match size { 1502 OperandSize::S32 => asm::inst::addss_a::new(dst, src).into(), 1503 OperandSize::S64 => asm::inst::addsd_a::new(dst, src).into(), 1504 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 1505 }; 1506 self.emit(Inst::External { inst }); 1507 } 1508 1509 /// Performs float subtraction on src and dst and places result in dst. 1510 pub fn xmm_sub_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1511 let dst = pair_xmm(dst); 1512 let inst = match size { 1513 OperandSize::S32 => asm::inst::subss_a::new(dst, src).into(), 1514 OperandSize::S64 => asm::inst::subsd_a::new(dst, src).into(), 1515 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 1516 }; 1517 self.emit(Inst::External { inst }); 1518 } 1519 1520 /// Performs float multiplication on src and dst and places result in dst. 1521 pub fn xmm_mul_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1522 use OperandSize::*; 1523 let dst = pair_xmm(dst); 1524 let inst = match size { 1525 S32 => asm::inst::mulss_a::new(dst, src).into(), 1526 S64 => asm::inst::mulsd_a::new(dst, src).into(), 1527 S8 | S16 | S128 => unreachable!(), 1528 }; 1529 self.emit(Inst::External { inst }); 1530 } 1531 1532 /// Performs float division on src and dst and places result in dst. 1533 pub fn xmm_div_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1534 let dst = pair_xmm(dst); 1535 let inst = match size { 1536 OperandSize::S32 => asm::inst::divss_a::new(dst, src).into(), 1537 OperandSize::S64 => asm::inst::divsd_a::new(dst, src).into(), 1538 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 1539 }; 1540 self.emit(Inst::External { inst }); 1541 } 1542 1543 /// Minimum for src and dst XMM registers with results put in dst. 1544 pub fn xmm_min_seq(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1545 self.emit(Inst::XmmMinMaxSeq { 1546 size: size.into(), 1547 is_min: true, 1548 lhs: src.into(), 1549 rhs: dst.to_reg().into(), 1550 dst: dst.map(Into::into), 1551 }); 1552 } 1553 1554 /// Maximum for src and dst XMM registers with results put in dst. 1555 pub fn xmm_max_seq(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1556 self.emit(Inst::XmmMinMaxSeq { 1557 size: size.into(), 1558 is_min: false, 1559 lhs: src.into(), 1560 rhs: dst.to_reg().into(), 1561 dst: dst.map(Into::into), 1562 }); 1563 } 1564 1565 /// Perform rounding operation on float register src and place results in 1566 /// float register dst. 1567 pub fn xmm_rounds_rr( 1568 &mut self, 1569 src: Reg, 1570 dst: WritableReg, 1571 mode: RoundingMode, 1572 size: OperandSize, 1573 ) { 1574 let dst = dst.map(|r| r.into()); 1575 1576 let imm: u8 = match mode { 1577 RoundingMode::Nearest => 0x00, 1578 RoundingMode::Down => 0x01, 1579 RoundingMode::Up => 0x02, 1580 RoundingMode::Zero => 0x03, 1581 }; 1582 1583 let inst = match size { 1584 OperandSize::S32 => asm::inst::roundss_rmi::new(dst, src, imm).into(), 1585 OperandSize::S64 => asm::inst::roundsd_rmi::new(dst, src, imm).into(), 1586 OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(), 1587 }; 1588 1589 self.emit(Inst::External { inst }); 1590 } 1591 1592 pub fn sqrt(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1593 use OperandSize::*; 1594 let dst = pair_xmm(dst); 1595 let inst = match size { 1596 S32 => asm::inst::sqrtss_a::new(dst, src).into(), 1597 S64 => asm::inst::sqrtsd_a::new(dst, src).into(), 1598 S8 | S16 | S128 => unimplemented!(), 1599 }; 1600 self.emit(Inst::External { inst }); 1601 } 1602 1603 /// Emit a call to an unknown location through a register. 1604 pub fn call_with_reg(&mut self, cc: CallingConvention, callee: Reg) { 1605 self.emit(Inst::CallUnknown { 1606 info: Box::new(CallInfo::empty(RegMem::reg(callee.into()), cc.into())), 1607 }); 1608 } 1609 1610 /// Emit a call to a locally defined function through an index. 1611 pub fn call_with_name(&mut self, cc: CallingConvention, name: UserExternalNameRef) { 1612 self.emit(Inst::CallKnown { 1613 info: Box::new(CallInfo::empty(ExternalName::user(name), cc.into())), 1614 }); 1615 } 1616 1617 /// Emits a conditional jump to the given label. 1618 pub fn jmp_if(&mut self, cc: impl Into<CC>, taken: MachLabel) { 1619 self.emit(Inst::WinchJmpIf { 1620 cc: cc.into(), 1621 taken, 1622 }); 1623 } 1624 1625 /// Performs an unconditional jump to the given label. 1626 pub fn jmp(&mut self, target: MachLabel) { 1627 self.emit(Inst::JmpKnown { dst: target }); 1628 } 1629 1630 /// Emits a jump table sequence. 1631 pub fn jmp_table( 1632 &mut self, 1633 targets: SmallVec<[MachLabel; 4]>, 1634 default: MachLabel, 1635 index: Reg, 1636 tmp1: Reg, 1637 tmp2: Reg, 1638 ) { 1639 self.emit(Inst::JmpTableSeq { 1640 idx: index.into(), 1641 tmp1: Writable::from_reg(tmp1.into()), 1642 tmp2: Writable::from_reg(tmp2.into()), 1643 default_target: default, 1644 targets: Box::new(targets.to_vec()), 1645 }) 1646 } 1647 1648 /// Emit a trap instruction. 1649 pub fn trap(&mut self, code: TrapCode) { 1650 let inst = asm::inst::ud2_zo::new(code).into(); 1651 self.emit(Inst::External { inst }); 1652 } 1653 1654 /// Conditional trap. 1655 pub fn trapif(&mut self, cc: impl Into<CC>, trap_code: TrapCode) { 1656 self.emit(Inst::TrapIf { 1657 cc: cc.into(), 1658 trap_code, 1659 }); 1660 } 1661 1662 /// Load effective address. 1663 pub fn lea(&mut self, addr: &Address, dst: WritableReg, size: OperandSize) { 1664 let addr = Self::to_synthetic_amode(addr, MemFlags::trusted()); 1665 let dst: WritableGpr = dst.map(Into::into); 1666 let inst = match size { 1667 OperandSize::S16 => asm::inst::leaw_rm::new(dst, addr).into(), 1668 OperandSize::S32 => asm::inst::leal_rm::new(dst, addr).into(), 1669 OperandSize::S64 => asm::inst::leaq_rm::new(dst, addr).into(), 1670 OperandSize::S8 | OperandSize::S128 => unimplemented!(), 1671 }; 1672 self.emit(Inst::External { inst }); 1673 } 1674 1675 pub fn adc_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1676 let dst = pair_gpr(dst); 1677 let inst = match size { 1678 OperandSize::S8 => asm::inst::adcb_rm::new(dst, src).into(), 1679 OperandSize::S16 => asm::inst::adcw_rm::new(dst, src).into(), 1680 OperandSize::S32 => asm::inst::adcl_rm::new(dst, src).into(), 1681 OperandSize::S64 => asm::inst::adcq_rm::new(dst, src).into(), 1682 OperandSize::S128 => unimplemented!(), 1683 }; 1684 self.emit(Inst::External { inst }); 1685 } 1686 1687 pub fn sbb_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1688 let dst = pair_gpr(dst); 1689 let inst = match size { 1690 OperandSize::S8 => asm::inst::sbbb_rm::new(dst, src).into(), 1691 OperandSize::S16 => asm::inst::sbbw_rm::new(dst, src).into(), 1692 OperandSize::S32 => asm::inst::sbbl_rm::new(dst, src).into(), 1693 OperandSize::S64 => asm::inst::sbbq_rm::new(dst, src).into(), 1694 OperandSize::S128 => unimplemented!(), 1695 }; 1696 self.emit(Inst::External { inst }); 1697 } 1698 1699 pub fn mul_wide( 1700 &mut self, 1701 dst_lo: WritableReg, 1702 dst_hi: WritableReg, 1703 lhs: Reg, 1704 rhs: Reg, 1705 kind: MulWideKind, 1706 size: OperandSize, 1707 ) { 1708 use MulWideKind::*; 1709 use OperandSize::*; 1710 let rax = asm::Fixed(PairedGpr { 1711 read: lhs.into(), 1712 write: WritableGpr::from_reg(dst_lo.to_reg().into()), 1713 }); 1714 let rdx = asm::Fixed(dst_hi.to_reg().into()); 1715 if size == S8 { 1716 // For `mulb` and `imulb`, both the high and low bits are written to 1717 // RAX. 1718 assert_eq!(dst_lo, dst_hi); 1719 } 1720 let inst = match (size, kind) { 1721 (S8, Unsigned) => asm::inst::mulb_m::new(rax, rhs).into(), 1722 (S8, Signed) => asm::inst::imulb_m::new(rax, rhs).into(), 1723 (S16, Unsigned) => asm::inst::mulw_m::new(rax, rdx, rhs).into(), 1724 (S16, Signed) => asm::inst::imulw_m::new(rax, rdx, rhs).into(), 1725 (S32, Unsigned) => asm::inst::mull_m::new(rax, rdx, rhs).into(), 1726 (S32, Signed) => asm::inst::imull_m::new(rax, rdx, rhs).into(), 1727 (S64, Unsigned) => asm::inst::mulq_m::new(rax, rdx, rhs).into(), 1728 (S64, Signed) => asm::inst::imulq_m::new(rax, rdx, rhs).into(), 1729 (S128, _) => unimplemented!(), 1730 }; 1731 self.emit(Inst::External { inst }); 1732 } 1733 1734 /// Shuffles bytes in `src` according to contents of `mask` and puts 1735 /// result in `dst`. 1736 pub fn xmm_vpshufb_rrm(&mut self, dst: WritableReg, src: Reg, mask: &Address) { 1737 let mask = Self::to_synthetic_amode(mask, MemFlags::trusted()); 1738 1739 self.emit(Inst::XmmRmiRVex { 1740 op: args::AvxOpcode::Vpshufb, 1741 src1: src.into(), 1742 src2: XmmMemImm::unwrap_new(RegMemImm::Mem { addr: mask }), 1743 dst: dst.to_reg().into(), 1744 }); 1745 } 1746 1747 /// Shuffles bytes in `src` according to contents of `mask` and puts 1748 /// result in `dst`. 1749 pub fn xmm_vpshufb_rrr(&mut self, dst: WritableReg, src: Reg, mask: Reg) { 1750 self.emit(Inst::XmmRmiRVex { 1751 op: args::AvxOpcode::Vpshufb, 1752 src1: src.into(), 1753 src2: XmmMemImm::unwrap_new(RegMemImm::reg(mask.into())), 1754 dst: dst.to_reg().into(), 1755 }) 1756 } 1757 1758 /// Add unsigned integers with unsigned saturation. 1759 /// 1760 /// Adds the src operands but when an individual byte result is larger than 1761 /// an unsigned byte integer, 0xFF is written instead. 1762 pub fn xmm_vpaddus_rrm( 1763 &mut self, 1764 dst: WritableReg, 1765 src1: Reg, 1766 src2: &Address, 1767 size: OperandSize, 1768 ) { 1769 let dst: WritableXmm = dst.map(|r| r.into()); 1770 let src2 = Self::to_synthetic_amode(src2, MemFlags::trusted()); 1771 let inst = match size { 1772 OperandSize::S8 => asm::inst::vpaddusb_b::new(dst, src1, src2).into(), 1773 OperandSize::S32 => asm::inst::vpaddusw_b::new(dst, src1, src2).into(), 1774 _ => unimplemented!(), 1775 }; 1776 self.emit(Inst::External { inst }); 1777 } 1778 1779 /// Add unsigned integers with unsigned saturation. 1780 /// 1781 /// Adds the src operands but when an individual byte result is larger than 1782 /// an unsigned byte integer, 0xFF is written instead. 1783 pub fn xmm_vpaddus_rrr(&mut self, dst: WritableReg, src1: Reg, src2: Reg, size: OperandSize) { 1784 let dst: WritableXmm = dst.map(|r| r.into()); 1785 let inst = match size { 1786 OperandSize::S8 => asm::inst::vpaddusb_b::new(dst, src1, src2).into(), 1787 OperandSize::S16 => asm::inst::vpaddusw_b::new(dst, src1, src2).into(), 1788 _ => unimplemented!(), 1789 }; 1790 self.emit(Inst::External { inst }); 1791 } 1792 1793 /// Add signed integers. 1794 pub fn xmm_vpadds_rrr(&mut self, dst: WritableReg, src1: Reg, src2: Reg, size: OperandSize) { 1795 let dst: WritableXmm = dst.map(|r| r.into()); 1796 let inst = match size { 1797 OperandSize::S8 => asm::inst::vpaddsb_b::new(dst, src1, src2).into(), 1798 OperandSize::S16 => asm::inst::vpaddsw_b::new(dst, src1, src2).into(), 1799 _ => unimplemented!(), 1800 }; 1801 self.emit(Inst::External { inst }); 1802 } 1803 1804 pub fn xmm_vpadd_rmr( 1805 &mut self, 1806 src1: Reg, 1807 src2: &Address, 1808 dst: WritableReg, 1809 size: OperandSize, 1810 ) { 1811 let dst: WritableXmm = dst.map(|r| r.into()); 1812 let address = Self::to_synthetic_amode(src2, MemFlags::trusted()); 1813 let inst = match size { 1814 OperandSize::S8 => asm::inst::vpaddb_b::new(dst, src1, address).into(), 1815 OperandSize::S16 => asm::inst::vpaddw_b::new(dst, src1, address).into(), 1816 OperandSize::S32 => asm::inst::vpaddd_b::new(dst, src1, address).into(), 1817 _ => unimplemented!(), 1818 }; 1819 self.emit(Inst::External { inst }); 1820 } 1821 1822 /// Adds vectors of integers in `src1` and `src2` and puts the results in 1823 /// `dst`. 1824 pub fn xmm_vpadd_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 1825 let dst: WritableXmm = dst.map(|r| r.into()); 1826 let inst = match size { 1827 OperandSize::S8 => asm::inst::vpaddb_b::new(dst, src1, src2).into(), 1828 OperandSize::S16 => asm::inst::vpaddw_b::new(dst, src1, src2).into(), 1829 OperandSize::S32 => asm::inst::vpaddd_b::new(dst, src1, src2).into(), 1830 OperandSize::S64 => asm::inst::vpaddq_b::new(dst, src1, src2).into(), 1831 _ => unimplemented!(), 1832 }; 1833 self.emit(Inst::External { inst }); 1834 } 1835 1836 pub fn mfence(&mut self) { 1837 self.emit(Inst::External { 1838 inst: asm::inst::mfence_zo::new().into(), 1839 }); 1840 } 1841 1842 /// Extract a value from `src` into `addr` determined by `lane`. 1843 pub(crate) fn xmm_vpextr_rm( 1844 &mut self, 1845 addr: &Address, 1846 src: Reg, 1847 lane: u8, 1848 size: OperandSize, 1849 flags: MemFlags, 1850 ) { 1851 assert!(addr.is_offset()); 1852 let dst = Self::to_synthetic_amode(addr, flags); 1853 let inst = match size { 1854 OperandSize::S8 => asm::inst::vpextrb_a::new(dst, src, lane).into(), 1855 OperandSize::S16 => asm::inst::vpextrw_b::new(dst, src, lane).into(), 1856 OperandSize::S32 => asm::inst::vpextrd_a::new(dst, src, lane).into(), 1857 OperandSize::S64 => asm::inst::vpextrq_a::new(dst, src, lane).into(), 1858 _ => unimplemented!(), 1859 }; 1860 self.emit(Inst::External { inst }); 1861 } 1862 1863 /// Extract a value from `src` into `dst` (zero extended) determined by `lane`. 1864 pub fn xmm_vpextr_rr(&mut self, dst: WritableReg, src: Reg, lane: u8, size: OperandSize) { 1865 let dst: WritableGpr = dst.map(|r| r.into()); 1866 let inst = match size { 1867 OperandSize::S8 => asm::inst::vpextrb_a::new(dst, src, lane).into(), 1868 OperandSize::S16 => asm::inst::vpextrw_a::new(dst, src, lane).into(), 1869 OperandSize::S32 => asm::inst::vpextrd_a::new(dst, src, lane).into(), 1870 OperandSize::S64 => asm::inst::vpextrq_a::new(dst, src, lane).into(), 1871 _ => unimplemented!(), 1872 }; 1873 self.emit(Inst::External { inst }); 1874 } 1875 1876 /// Copy value from `src2`, merge into `src1`, and put result in `dst` at 1877 /// the location specified in `count`. 1878 pub fn xmm_vpinsr_rrm( 1879 &mut self, 1880 dst: WritableReg, 1881 src1: Reg, 1882 src2: &Address, 1883 count: u8, 1884 size: OperandSize, 1885 ) { 1886 let src2 = Self::to_synthetic_amode(src2, MemFlags::trusted()); 1887 let dst: WritableXmm = dst.map(|r| r.into()); 1888 1889 let inst = match size { 1890 OperandSize::S8 => asm::inst::vpinsrb_b::new(dst, src1, src2, count).into(), 1891 OperandSize::S16 => asm::inst::vpinsrw_b::new(dst, src1, src2, count).into(), 1892 OperandSize::S32 => asm::inst::vpinsrd_b::new(dst, src1, src2, count).into(), 1893 OperandSize::S64 => asm::inst::vpinsrq_b::new(dst, src1, src2, count).into(), 1894 OperandSize::S128 => unreachable!(), 1895 }; 1896 self.emit(Inst::External { inst }); 1897 } 1898 1899 /// Copy value from `src2`, merge into `src1`, and put result in `dst` at 1900 /// the location specified in `count`. 1901 pub fn xmm_vpinsr_rrr( 1902 &mut self, 1903 dst: WritableReg, 1904 src1: Reg, 1905 src2: Reg, 1906 count: u8, 1907 size: OperandSize, 1908 ) { 1909 let dst: WritableXmm = dst.map(|r| r.into()); 1910 let inst = match size { 1911 OperandSize::S8 => asm::inst::vpinsrb_b::new(dst, src1, src2, count).into(), 1912 OperandSize::S16 => asm::inst::vpinsrw_b::new(dst, src1, src2, count).into(), 1913 OperandSize::S32 => asm::inst::vpinsrd_b::new(dst, src1, src2, count).into(), 1914 OperandSize::S64 => asm::inst::vpinsrq_b::new(dst, src1, src2, count).into(), 1915 OperandSize::S128 => unreachable!(), 1916 }; 1917 self.emit(Inst::External { inst }); 1918 } 1919 1920 /// Copy a 32-bit float in `src2`, merge into `src1`, and put result in `dst`. 1921 pub fn xmm_vinsertps_rrm(&mut self, dst: WritableReg, src1: Reg, src2: &Address, imm: u8) { 1922 let src2 = Self::to_synthetic_amode(src2, MemFlags::trusted()); 1923 1924 self.emit(Inst::XmmRmRImmVex { 1925 op: AvxOpcode::Vinsertps, 1926 src1: src1.into(), 1927 src2: XmmMem::unwrap_new(RegMem::mem(src2)), 1928 dst: dst.to_reg().into(), 1929 imm, 1930 }); 1931 } 1932 1933 /// Copy a 32-bit float in `src2`, merge into `src1`, and put result in `dst`. 1934 pub fn xmm_vinsertps_rrr(&mut self, dst: WritableReg, src1: Reg, src2: Reg, imm: u8) { 1935 self.emit(Inst::XmmRmRImmVex { 1936 op: AvxOpcode::Vinsertps, 1937 src1: src1.into(), 1938 src2: XmmMem::unwrap_new(RegMem::reg(src2.into())), 1939 dst: dst.to_reg().into(), 1940 imm, 1941 }); 1942 } 1943 1944 /// Moves lower 64-bit float from `src2` into lower 64-bits of `dst` and the 1945 /// upper 64-bits in `src1` into the upper 64-bits of `dst`. 1946 pub fn xmm_vmovsd_rrr(&mut self, dst: WritableReg, src1: Reg, src2: Reg) { 1947 self.emit(Inst::XmmRmiRVex { 1948 op: AvxOpcode::Vmovsd, 1949 src1: src1.into(), 1950 src2: XmmMemImm::unwrap_new(src2.into()), 1951 dst: dst.to_reg().into(), 1952 }) 1953 } 1954 1955 /// Moves 64-bit float from `src` into lower 64-bits of `dst`. 1956 /// Zeroes out the upper 64 bits of `dst`. 1957 pub fn xmm_vmovsd_rm(&mut self, dst: WritableReg, src: &Address) { 1958 let src = Self::to_synthetic_amode(src, MemFlags::trusted()); 1959 let dst: WritableXmm = dst.map(|r| r.into()); 1960 let inst = asm::inst::vmovsd_d::new(dst, src).into(); 1961 self.emit(Inst::External { inst }); 1962 } 1963 1964 /// Moves two 32-bit floats from `src2` to the upper 64-bits of `dst`. 1965 /// Copies two 32-bit floats from the lower 64-bits of `src1` to lower 1966 /// 64-bits of `dst`. 1967 pub fn xmm_vmovlhps_rrm(&mut self, dst: WritableReg, src1: Reg, src2: &Address) { 1968 let src2 = Self::to_synthetic_amode(src2, MemFlags::trusted()); 1969 let dst: WritableXmm = dst.map(|r| r.into()); 1970 let inst = asm::inst::vmovhps_b::new(dst, src1, src2).into(); 1971 self.emit(Inst::External { inst }); 1972 } 1973 1974 /// Moves two 32-bit floats from the lower 64-bits of `src2` to the upper 1975 /// 64-bits of `dst`. Copies two 32-bit floats from the lower 64-bits of 1976 /// `src1` to lower 64-bits of `dst`. 1977 pub fn xmm_vmovlhps_rrr(&mut self, dst: WritableReg, src1: Reg, src2: Reg) { 1978 let dst: WritableXmm = dst.map(|r| r.into()); 1979 let inst = asm::inst::vmovlhps_rvm::new(dst, src1, src2).into(); 1980 self.emit(Inst::External { inst }); 1981 } 1982 1983 /// Move unaligned packed integer values from address `src` to `dst`. 1984 pub fn xmm_vmovdqu_mr(&mut self, src: &Address, dst: WritableReg, flags: MemFlags) { 1985 let src = Self::to_synthetic_amode(src, flags); 1986 let dst: WritableXmm = dst.map(|r| r.into()); 1987 let inst = asm::inst::vmovdqu_a::new(dst, src).into(); 1988 self.emit(Inst::External { inst }); 1989 } 1990 1991 /// Move integer from `src` to xmm register `dst` using an AVX instruction. 1992 pub fn avx_gpr_to_xmm(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 1993 let dst: WritableXmm = dst.map(|r| r.into()); 1994 let inst = match size { 1995 OperandSize::S32 => asm::inst::vmovd_a::new(dst, src).into(), 1996 OperandSize::S64 => asm::inst::vmovq_a::new(dst, src).into(), 1997 _ => unreachable!(), 1998 }; 1999 2000 self.emit(Inst::External { inst }); 2001 } 2002 2003 /// Perform an AVX opcode `op` involving registers `src1` and `src2`, writing the 2004 /// result to `dst`. 2005 pub fn xmm_vex_rr(&mut self, op: AvxOpcode, src1: Reg, src2: Reg, dst: WritableReg) { 2006 self.emit(Inst::XmmRmiRVex { 2007 op, 2008 src1: src1.into(), 2009 src2: src2.into(), 2010 dst: dst.map(Into::into), 2011 }) 2012 } 2013 2014 pub fn xmm_vptest(&mut self, src1: Reg, src2: Reg) { 2015 self.emit(Inst::XmmCmpRmRVex { 2016 op: AvxOpcode::Vptest, 2017 src1: src1.into(), 2018 src2: src2.into(), 2019 }) 2020 } 2021 2022 /// Converts vector of integers into vector of floating values. 2023 pub fn xmm_vcvt_rr(&mut self, src: Reg, dst: WritableReg, kind: VcvtKind) { 2024 let dst: WritableXmm = dst.map(|x| x.into()); 2025 let inst = match kind { 2026 VcvtKind::I32ToF32 => asm::inst::vcvtdq2ps_a::new(dst, src).into(), 2027 VcvtKind::I32ToF64 => asm::inst::vcvtdq2pd_a::new(dst, src).into(), 2028 VcvtKind::F64ToF32 => asm::inst::vcvtpd2ps_a::new(dst, src).into(), 2029 VcvtKind::F64ToI32 => asm::inst::vcvttpd2dq_a::new(dst, src).into(), 2030 VcvtKind::F32ToF64 => asm::inst::vcvtps2pd_a::new(dst, src).into(), 2031 VcvtKind::F32ToI32 => asm::inst::vcvttps2dq_a::new(dst, src).into(), 2032 }; 2033 2034 self.emit(Inst::External { inst }); 2035 } 2036 2037 /// Subtract floats in vector `src1` to floats in vector `src2`. 2038 pub fn xmm_vsubp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2039 let dst: WritableXmm = dst.map(|r| r.into()); 2040 let inst = match size { 2041 OperandSize::S32 => asm::inst::vsubps_b::new(dst, src1, src2).into(), 2042 OperandSize::S64 => asm::inst::vsubpd_b::new(dst, src1, src2).into(), 2043 _ => unimplemented!(), 2044 }; 2045 self.emit(Inst::External { inst }); 2046 } 2047 2048 /// Subtract integers in vector `src1` from integers in vector `src2`. 2049 pub fn xmm_vpsub_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2050 let dst: WritableXmm = dst.map(|r| r.into()); 2051 let inst = match size { 2052 OperandSize::S8 => asm::inst::vpsubb_b::new(dst, src1, src2).into(), 2053 OperandSize::S16 => asm::inst::vpsubw_b::new(dst, src1, src2).into(), 2054 OperandSize::S32 => asm::inst::vpsubd_b::new(dst, src1, src2).into(), 2055 OperandSize::S64 => asm::inst::vpsubq_b::new(dst, src1, src2).into(), 2056 _ => unimplemented!(), 2057 }; 2058 self.emit(Inst::External { inst }); 2059 } 2060 2061 /// Substract unsigned integers with unsigned saturation. 2062 pub fn xmm_vpsubus_rrr(&mut self, dst: WritableReg, src1: Reg, src2: Reg, size: OperandSize) { 2063 let dst: WritableXmm = dst.map(|r| r.into()); 2064 let inst = match size { 2065 OperandSize::S8 => asm::inst::vpsubusb_b::new(dst, src1, src2).into(), 2066 OperandSize::S16 => asm::inst::vpsubusw_b::new(dst, src1, src2).into(), 2067 _ => unimplemented!(), 2068 }; 2069 self.emit(Inst::External { inst }); 2070 } 2071 2072 /// Subtract signed integers with signed saturation. 2073 pub fn xmm_vpsubs_rrr(&mut self, dst: WritableReg, src1: Reg, src2: Reg, size: OperandSize) { 2074 let dst: WritableXmm = dst.map(|r| r.into()); 2075 let inst = match size { 2076 OperandSize::S8 => asm::inst::vpsubsb_b::new(dst, src1, src2).into(), 2077 OperandSize::S16 => asm::inst::vpsubsw_b::new(dst, src1, src2).into(), 2078 _ => unimplemented!(), 2079 }; 2080 self.emit(Inst::External { inst }); 2081 } 2082 2083 /// Add floats in vector `src1` to floats in vector `src2`. 2084 pub fn xmm_vaddp_rrm( 2085 &mut self, 2086 src1: Reg, 2087 src2: &Address, 2088 dst: WritableReg, 2089 size: OperandSize, 2090 ) { 2091 let dst: WritableXmm = dst.map(|r| r.into()); 2092 let address = Self::to_synthetic_amode(src2, MemFlags::trusted()); 2093 let inst = match size { 2094 OperandSize::S32 => asm::inst::vaddps_b::new(dst, src1, address).into(), 2095 OperandSize::S64 => asm::inst::vaddpd_b::new(dst, src1, address).into(), 2096 _ => unimplemented!(), 2097 }; 2098 self.emit(Inst::External { inst }); 2099 } 2100 2101 /// Add floats in vector `src1` to floats in vector `src2`. 2102 pub fn xmm_vaddp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2103 let dst: WritableXmm = dst.map(|r| r.into()); 2104 let inst = match size { 2105 OperandSize::S32 => asm::inst::vaddps_b::new(dst, src1, src2).into(), 2106 OperandSize::S64 => asm::inst::vaddpd_b::new(dst, src1, src2).into(), 2107 _ => unimplemented!(), 2108 }; 2109 self.emit(Inst::External { inst }); 2110 } 2111 2112 /// Compare vector register `lhs` with a vector of integers in `rhs` for 2113 /// equality between packed integers and write the resulting vector into 2114 /// `dst`. 2115 pub fn xmm_vpcmpeq_rrm( 2116 &mut self, 2117 dst: WritableReg, 2118 lhs: Reg, 2119 address: &Address, 2120 size: OperandSize, 2121 ) { 2122 let dst: WritableXmm = dst.map(|r| r.into()); 2123 let address = Self::to_synthetic_amode(address, MemFlags::trusted()); 2124 let inst = match size { 2125 OperandSize::S8 => asm::inst::vpcmpeqb_b::new(dst, lhs, address).into(), 2126 OperandSize::S16 => asm::inst::vpcmpeqw_b::new(dst, lhs, address).into(), 2127 OperandSize::S32 => asm::inst::vpcmpeqd_b::new(dst, lhs, address).into(), 2128 OperandSize::S64 => asm::inst::vpcmpeqq_b::new(dst, lhs, address).into(), 2129 _ => unimplemented!(), 2130 }; 2131 self.emit(Inst::External { inst }); 2132 } 2133 2134 /// Compare vector registers `lhs` and `rhs` for equality between packed 2135 /// integers and write the resulting vector into `dst`. 2136 pub fn xmm_vpcmpeq_rrr(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) { 2137 let dst: WritableXmm = dst.map(|r| r.into()); 2138 let inst = match size { 2139 OperandSize::S8 => asm::inst::vpcmpeqb_b::new(dst, lhs, rhs).into(), 2140 OperandSize::S16 => asm::inst::vpcmpeqw_b::new(dst, lhs, rhs).into(), 2141 OperandSize::S32 => asm::inst::vpcmpeqd_b::new(dst, lhs, rhs).into(), 2142 OperandSize::S64 => asm::inst::vpcmpeqq_b::new(dst, lhs, rhs).into(), 2143 _ => unimplemented!(), 2144 }; 2145 self.emit(Inst::External { inst }); 2146 } 2147 2148 /// Performs a greater than comparison with vectors of signed integers in 2149 /// `lhs` and `rhs` and puts the results in `dst`. 2150 pub fn xmm_vpcmpgt_rrr(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) { 2151 let dst: WritableXmm = dst.map(|r| r.into()); 2152 let inst = match size { 2153 OperandSize::S8 => asm::inst::vpcmpgtb_b::new(dst, lhs, rhs).into(), 2154 OperandSize::S16 => asm::inst::vpcmpgtw_b::new(dst, lhs, rhs).into(), 2155 OperandSize::S32 => asm::inst::vpcmpgtd_b::new(dst, lhs, rhs).into(), 2156 OperandSize::S64 => asm::inst::vpcmpgtq_b::new(dst, lhs, rhs).into(), 2157 _ => unimplemented!(), 2158 }; 2159 self.emit(Inst::External { inst }); 2160 } 2161 2162 /// Performs a max operation with vectors of signed integers in `lhs` and 2163 /// `rhs` and puts the results in `dst`. 2164 pub fn xmm_vpmaxs_rrr(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) { 2165 let dst: WritableXmm = dst.map(|r| r.into()); 2166 let inst = match size { 2167 OperandSize::S8 => asm::inst::vpmaxsb_b::new(dst, lhs, rhs).into(), 2168 OperandSize::S16 => asm::inst::vpmaxsw_b::new(dst, lhs, rhs).into(), 2169 OperandSize::S32 => asm::inst::vpmaxsd_b::new(dst, lhs, rhs).into(), 2170 _ => unimplemented!(), 2171 }; 2172 self.emit(Inst::External { inst }); 2173 } 2174 2175 /// Performs a max operation with vectors of unsigned integers in `lhs` and 2176 /// `rhs` and puts the results in `dst`. 2177 pub fn xmm_vpmaxu_rrr(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) { 2178 let dst: WritableXmm = dst.map(|r| r.into()); 2179 let inst = match size { 2180 OperandSize::S8 => asm::inst::vpmaxub_b::new(dst, lhs, rhs).into(), 2181 OperandSize::S16 => asm::inst::vpmaxuw_b::new(dst, lhs, rhs).into(), 2182 OperandSize::S32 => asm::inst::vpmaxud_b::new(dst, lhs, rhs).into(), 2183 _ => unimplemented!(), 2184 }; 2185 self.emit(Inst::External { inst }); 2186 } 2187 2188 /// Performs a min operation with vectors of signed integers in `lhs` and 2189 /// `rhs` and puts the results in `dst`. 2190 pub fn xmm_vpmins_rrr(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) { 2191 let dst: WritableXmm = dst.map(|r| r.into()); 2192 let inst = match size { 2193 OperandSize::S8 => asm::inst::vpminsb_b::new(dst, lhs, rhs).into(), 2194 OperandSize::S16 => asm::inst::vpminsw_b::new(dst, lhs, rhs).into(), 2195 OperandSize::S32 => asm::inst::vpminsd_b::new(dst, lhs, rhs).into(), 2196 _ => unimplemented!(), 2197 }; 2198 self.emit(Inst::External { inst }); 2199 } 2200 2201 /// Performs a min operation with vectors of unsigned integers in `lhs` and 2202 /// `rhs` and puts the results in `dst`. 2203 pub fn xmm_vpminu_rrr(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) { 2204 let dst: WritableXmm = dst.map(|r| r.into()); 2205 let inst = match size { 2206 OperandSize::S8 => asm::inst::vpminub_b::new(dst, lhs, rhs).into(), 2207 OperandSize::S16 => asm::inst::vpminuw_b::new(dst, lhs, rhs).into(), 2208 OperandSize::S32 => asm::inst::vpminud_b::new(dst, lhs, rhs).into(), 2209 _ => unimplemented!(), 2210 }; 2211 self.emit(Inst::External { inst }); 2212 } 2213 2214 /// Performs a comparison operation between vectors of floats in `lhs` and 2215 /// `rhs` and puts the results in `dst`. 2216 pub fn xmm_vcmpp_rrr( 2217 &mut self, 2218 dst: WritableReg, 2219 lhs: Reg, 2220 rhs: Reg, 2221 size: OperandSize, 2222 kind: VcmpKind, 2223 ) { 2224 let dst: WritableXmm = dst.map(|r| r.into()); 2225 let imm = match kind { 2226 VcmpKind::Eq => 0, 2227 VcmpKind::Lt => 1, 2228 VcmpKind::Le => 2, 2229 VcmpKind::Unord => 3, 2230 VcmpKind::Ne => 4, 2231 }; 2232 let inst = match size { 2233 OperandSize::S32 => asm::inst::vcmpps_b::new(dst, lhs, rhs, imm).into(), 2234 OperandSize::S64 => asm::inst::vcmppd_b::new(dst, lhs, rhs, imm).into(), 2235 _ => unimplemented!(), 2236 }; 2237 self.emit(Inst::External { inst }); 2238 } 2239 2240 /// Performs a subtraction on two vectors of floats and puts the results in 2241 /// `dst`. 2242 pub fn xmm_vsub_rrm(&mut self, src1: Reg, src2: &Address, dst: WritableReg, size: OperandSize) { 2243 let dst: WritableXmm = dst.map(|r| r.into()); 2244 let address = Self::to_synthetic_amode(src2, MemFlags::trusted()); 2245 let inst = match size { 2246 OperandSize::S64 => asm::inst::vsubpd_b::new(dst, src1, address).into(), 2247 _ => unimplemented!(), 2248 }; 2249 self.emit(Inst::External { inst }); 2250 } 2251 2252 /// Performs a subtraction on two vectors of floats and puts the results in 2253 /// `dst`. 2254 pub fn xmm_vsub_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2255 let dst: WritableXmm = dst.map(|r| r.into()); 2256 let inst = match size { 2257 OperandSize::S32 => asm::inst::vsubps_b::new(dst, src1, src2).into(), 2258 OperandSize::S64 => asm::inst::vsubpd_b::new(dst, src1, src2).into(), 2259 _ => unimplemented!(), 2260 }; 2261 self.emit(Inst::External { inst }); 2262 } 2263 2264 /// Converts a vector of signed integers into a vector of narrower integers 2265 /// using saturation to handle overflow. 2266 pub fn xmm_vpackss_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2267 let dst: WritableXmm = dst.map(|r| r.into()); 2268 let inst = match size { 2269 OperandSize::S8 => asm::inst::vpacksswb_b::new(dst, src1, src2).into(), 2270 OperandSize::S16 => asm::inst::vpackssdw_b::new(dst, src1, src2).into(), 2271 _ => unimplemented!(), 2272 }; 2273 self.emit(Inst::External { inst }); 2274 } 2275 2276 /// Converts a vector of unsigned integers into a vector of narrower 2277 /// integers using saturation to handle overflow. 2278 pub fn xmm_vpackus_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2279 let dst: WritableXmm = dst.map(|r| r.into()); 2280 let inst = match size { 2281 OperandSize::S8 => asm::inst::vpackuswb_b::new(dst, src1, src2).into(), 2282 OperandSize::S16 => asm::inst::vpackusdw_b::new(dst, src1, src2).into(), 2283 _ => unimplemented!(), 2284 }; 2285 self.emit(Inst::External { inst }); 2286 } 2287 2288 /// Concatenates `src1` and `src2` and shifts right by `imm` and puts 2289 /// result in `dst`. 2290 pub fn xmm_vpalignr_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, imm: u8) { 2291 self.emit(Inst::XmmRmRImmVex { 2292 op: AvxOpcode::Vpalignr, 2293 src1: src1.into(), 2294 src2: src2.into(), 2295 dst: dst.to_reg().into(), 2296 imm, 2297 }) 2298 } 2299 2300 /// Takes the lower lanes of vectors of floats in `src1` and `src2` and 2301 /// interleaves them in `dst`. 2302 pub fn xmm_vunpcklp_rrm( 2303 &mut self, 2304 src1: Reg, 2305 src2: &Address, 2306 dst: WritableReg, 2307 size: OperandSize, 2308 ) { 2309 let dst: WritableXmm = dst.map(|r| r.into()); 2310 let address = Self::to_synthetic_amode(src2, MemFlags::trusted()); 2311 let inst = match size { 2312 OperandSize::S32 => asm::inst::vunpcklps_b::new(dst, src1, address).into(), 2313 _ => unimplemented!(), 2314 }; 2315 self.emit(Inst::External { inst }); 2316 } 2317 2318 /// Unpacks and interleaves high order data of floats in `src1` and `src2` 2319 /// and puts the results in `dst`. 2320 pub fn xmm_vunpckhp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2321 let dst: WritableXmm = dst.map(|r| r.into()); 2322 let inst = match size { 2323 OperandSize::S32 => asm::inst::vunpckhps_b::new(dst, src1, src2).into(), 2324 _ => unimplemented!(), 2325 }; 2326 self.emit(Inst::External { inst }); 2327 } 2328 2329 /// Unpacks and interleaves the lower lanes of vectors of integers in `src1` 2330 /// and `src2` and puts the results in `dst`. 2331 pub fn xmm_vpunpckl_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2332 let dst: WritableXmm = dst.map(|r| r.into()); 2333 let inst = match size { 2334 OperandSize::S8 => asm::inst::vpunpcklbw_b::new(dst, src1, src2).into(), 2335 OperandSize::S16 => asm::inst::vpunpcklwd_b::new(dst, src1, src2).into(), 2336 _ => unimplemented!(), 2337 }; 2338 self.emit(Inst::External { inst }); 2339 } 2340 2341 /// Unpacks and interleaves the higher lanes of vectors of integers in 2342 /// `src1` and `src2` and puts the results in `dst`. 2343 pub fn xmm_vpunpckh_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2344 let dst: WritableXmm = dst.map(|r| r.into()); 2345 let inst = match size { 2346 OperandSize::S8 => asm::inst::vpunpckhbw_b::new(dst, src1, src2).into(), 2347 OperandSize::S16 => asm::inst::vpunpckhwd_b::new(dst, src1, src2).into(), 2348 _ => unimplemented!(), 2349 }; 2350 self.emit(Inst::External { inst }); 2351 } 2352 2353 pub(crate) fn xmm_rm_rvex3( 2354 &mut self, 2355 op: Avx512Opcode, 2356 src1: Reg, 2357 src2: Reg, 2358 dst: WritableReg, 2359 ) { 2360 self.emit(Inst::XmmRmREvex3 { 2361 op, 2362 // `src1` reuses `dst`, and is ignored in emission 2363 src1: dst.to_reg().into(), 2364 src2: src1.into(), 2365 src3: src2.into(), 2366 dst: dst.map(Into::into), 2367 }); 2368 } 2369 2370 /// Creates a mask made up of the most significant bit of each byte of 2371 /// `src` and stores the result in `dst`. 2372 pub fn xmm_vpmovmsk_rr( 2373 &mut self, 2374 src: Reg, 2375 dst: WritableReg, 2376 src_size: OperandSize, 2377 dst_size: OperandSize, 2378 ) { 2379 assert_eq!(dst_size, OperandSize::S32); 2380 let dst: WritableGpr = dst.map(|r| r.into()); 2381 let inst = match src_size { 2382 OperandSize::S8 => asm::inst::vpmovmskb_rm::new(dst, src).into(), 2383 _ => unimplemented!(), 2384 }; 2385 2386 self.emit(Inst::External { inst }); 2387 } 2388 2389 /// Creates a mask made up of the most significant bit of each byte of 2390 /// in `src` and stores the result in `dst`. 2391 pub fn xmm_vmovskp_rr( 2392 &mut self, 2393 src: Reg, 2394 dst: WritableReg, 2395 src_size: OperandSize, 2396 dst_size: OperandSize, 2397 ) { 2398 assert_eq!(dst_size, OperandSize::S32); 2399 let dst: WritableGpr = dst.map(|r| r.into()); 2400 let inst = match src_size { 2401 OperandSize::S32 => asm::inst::vmovmskps_rm::new(dst, src).into(), 2402 OperandSize::S64 => asm::inst::vmovmskpd_rm::new(dst, src).into(), 2403 _ => unimplemented!(), 2404 }; 2405 2406 self.emit(Inst::External { inst }); 2407 } 2408 2409 /// Compute the absolute value of elements in vector `src` and put the 2410 /// results in `dst`. 2411 pub fn xmm_vpabs_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 2412 let dst: WritableXmm = dst.map(|r| r.into()); 2413 let inst = match size { 2414 OperandSize::S8 => asm::inst::vpabsb_a::new(dst, src).into(), 2415 OperandSize::S16 => asm::inst::vpabsw_a::new(dst, src).into(), 2416 OperandSize::S32 => asm::inst::vpabsd_a::new(dst, src).into(), 2417 _ => unimplemented!(), 2418 }; 2419 self.emit(Inst::External { inst }); 2420 } 2421 2422 /// Arithmetically (sign preserving) right shift on vector in `src` by 2423 /// `amount` with result written to `dst`. 2424 pub fn xmm_vpsra_rrr(&mut self, src: Reg, amount: Reg, dst: WritableReg, size: OperandSize) { 2425 let dst: WritableXmm = dst.map(|r| r.into()); 2426 let inst = match size { 2427 OperandSize::S16 => asm::inst::vpsraw_c::new(dst, src, amount).into(), 2428 OperandSize::S32 => asm::inst::vpsrad_c::new(dst, src, amount).into(), 2429 _ => unimplemented!(), 2430 }; 2431 self.emit(Inst::External { inst }); 2432 } 2433 2434 /// Arithmetically (sign preserving) right shift on vector in `src` by 2435 /// `imm` with result written to `dst`. 2436 pub fn xmm_vpsra_rri(&mut self, src: Reg, dst: WritableReg, imm: u32, size: OperandSize) { 2437 let dst: WritableXmm = dst.map(|r| r.into()); 2438 let imm = u8::try_from(imm).expect("immediate must fit in 8 bits"); 2439 let inst = match size { 2440 OperandSize::S32 => asm::inst::vpsrad_d::new(dst, src, imm).into(), 2441 _ => unimplemented!(), 2442 }; 2443 self.emit(Inst::External { inst }); 2444 } 2445 2446 /// Shift vector data left by `imm`. 2447 pub fn xmm_vpsll_rri(&mut self, src: Reg, dst: WritableReg, imm: u32, size: OperandSize) { 2448 let dst: WritableXmm = dst.map(|r| r.into()); 2449 let imm = u8::try_from(imm).expect("immediate must fit in 8 bits"); 2450 let inst = match size { 2451 OperandSize::S32 => asm::inst::vpslld_d::new(dst, src, imm).into(), 2452 OperandSize::S64 => asm::inst::vpsllq_d::new(dst, src, imm).into(), 2453 _ => unimplemented!(), 2454 }; 2455 self.emit(Inst::External { inst }); 2456 } 2457 2458 /// Shift vector data left by `amount`. 2459 pub fn xmm_vpsll_rrr(&mut self, src: Reg, amount: Reg, dst: WritableReg, size: OperandSize) { 2460 let dst: WritableXmm = dst.map(|r| r.into()); 2461 let inst = match size { 2462 OperandSize::S16 => asm::inst::vpsllw_c::new(dst, src, amount).into(), 2463 OperandSize::S32 => asm::inst::vpslld_c::new(dst, src, amount).into(), 2464 OperandSize::S64 => asm::inst::vpsllq_c::new(dst, src, amount).into(), 2465 _ => unimplemented!(), 2466 }; 2467 self.emit(Inst::External { inst }); 2468 } 2469 2470 /// Shift vector data right by `imm`. 2471 pub fn xmm_vpsrl_rri(&mut self, src: Reg, dst: WritableReg, imm: u32, size: OperandSize) { 2472 let dst: WritableXmm = dst.map(|r| r.into()); 2473 let imm = u8::try_from(imm).expect("immediate must fit in 8 bits"); 2474 let inst = match size { 2475 OperandSize::S16 => asm::inst::vpsrlw_d::new(dst, src, imm).into(), 2476 OperandSize::S32 => asm::inst::vpsrld_d::new(dst, src, imm).into(), 2477 OperandSize::S64 => asm::inst::vpsrlq_d::new(dst, src, imm).into(), 2478 _ => unimplemented!(), 2479 }; 2480 self.emit(Inst::External { inst }); 2481 } 2482 2483 /// Shift vector data right by `amount`. 2484 pub fn xmm_vpsrl_rrr(&mut self, src: Reg, amount: Reg, dst: WritableReg, size: OperandSize) { 2485 let dst: WritableXmm = dst.map(|r| r.into()); 2486 let inst = match size { 2487 OperandSize::S16 => asm::inst::vpsrlw_c::new(dst, src, amount).into(), 2488 OperandSize::S32 => asm::inst::vpsrld_c::new(dst, src, amount).into(), 2489 OperandSize::S64 => asm::inst::vpsrlq_c::new(dst, src, amount).into(), 2490 _ => unimplemented!(), 2491 }; 2492 self.emit(Inst::External { inst }); 2493 } 2494 2495 /// Perform an `and` operation on vectors of floats in `src1` and `src2` 2496 /// and put the results in `dst`. 2497 pub fn xmm_vandp_rrm( 2498 &mut self, 2499 src1: Reg, 2500 src2: &Address, 2501 dst: WritableReg, 2502 size: OperandSize, 2503 ) { 2504 let dst: WritableXmm = dst.map(|r| r.into()); 2505 let address = Self::to_synthetic_amode(src2, MemFlags::trusted()); 2506 let inst = match size { 2507 OperandSize::S32 => asm::inst::vandps_b::new(dst, src1, address).into(), 2508 OperandSize::S64 => asm::inst::vandpd_b::new(dst, src1, address).into(), 2509 _ => unimplemented!(), 2510 }; 2511 self.emit(Inst::External { inst }); 2512 } 2513 2514 /// Perform an `and` operation on vectors of floats in `src1` and `src2` 2515 /// and put the results in `dst`. 2516 pub fn xmm_vandp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2517 let dst: WritableXmm = dst.map(|r| r.into()); 2518 let inst = match size { 2519 OperandSize::S32 => asm::inst::vandps_b::new(dst, src1, src2).into(), 2520 OperandSize::S64 => asm::inst::vandpd_b::new(dst, src1, src2).into(), 2521 _ => unimplemented!(), 2522 }; 2523 self.emit(Inst::External { inst }); 2524 } 2525 2526 /// Performs a bitwise `and` operation on the vectors in `src1` and `src2` 2527 /// and stores the results in `dst`. 2528 pub fn xmm_vpand_rrm(&mut self, src1: Reg, src2: &Address, dst: WritableReg) { 2529 let dst: WritableXmm = dst.map(|r| r.into()); 2530 let address = Self::to_synthetic_amode(&src2, MemFlags::trusted()); 2531 let inst = asm::inst::vpand_b::new(dst, src1, address).into(); 2532 self.emit(Inst::External { inst }); 2533 } 2534 2535 /// Performs a bitwise `and` operation on the vectors in `src1` and `src2` 2536 /// and stores the results in `dst`. 2537 pub fn xmm_vpand_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg) { 2538 let dst: WritableXmm = dst.map(|r| r.into()); 2539 let inst = asm::inst::vpand_b::new(dst, src1, src2).into(); 2540 self.emit(Inst::External { inst }); 2541 } 2542 2543 /// Perform an `and not` operation on vectors of floats in `src1` and 2544 /// `src2` and put the results in `dst`. 2545 pub fn xmm_vandnp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2546 let dst: WritableXmm = dst.map(|r| r.into()); 2547 let inst = match size { 2548 OperandSize::S32 => asm::inst::vandnps_b::new(dst, src1, src2).into(), 2549 OperandSize::S64 => asm::inst::vandnpd_b::new(dst, src1, src2).into(), 2550 _ => unimplemented!(), 2551 }; 2552 self.emit(Inst::External { inst }); 2553 } 2554 2555 /// Perform an `and not` operation on vectors in `src1` and `src2` and put 2556 /// the results in `dst`. 2557 pub fn xmm_vpandn_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg) { 2558 let dst: WritableXmm = dst.map(|r| r.into()); 2559 let inst = asm::inst::vpandn_b::new(dst, src1, src2).into(); 2560 self.emit(Inst::External { inst }); 2561 } 2562 2563 /// Perform an or operation for the vectors of floats in `src1` and `src2` 2564 /// and put the results in `dst`. 2565 pub fn xmm_vorp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2566 let dst: WritableXmm = dst.map(|r| r.into()); 2567 let inst = match size { 2568 OperandSize::S32 => asm::inst::vorps_b::new(dst, src1, src2).into(), 2569 OperandSize::S64 => asm::inst::vorpd_b::new(dst, src1, src2).into(), 2570 _ => unimplemented!(), 2571 }; 2572 self.emit(Inst::External { inst }); 2573 } 2574 2575 /// Bitwise OR of `src1` and `src2`. 2576 pub fn xmm_vpor_rrr(&mut self, dst: WritableReg, src1: Reg, src2: Reg) { 2577 let dst: WritableXmm = dst.map(|r| r.into()); 2578 let inst = asm::inst::vpor_b::new(dst, src1, src2).into(); 2579 self.emit(Inst::External { inst }); 2580 } 2581 2582 /// Bitwise logical xor of vectors of floats in `src1` and `src2` and puts 2583 /// the results in `dst`. 2584 pub fn xmm_vxorp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2585 let dst: WritableXmm = dst.map(|r| r.into()); 2586 let inst = match size { 2587 OperandSize::S32 => asm::inst::vxorps_b::new(dst, src1, src2).into(), 2588 OperandSize::S64 => asm::inst::vxorpd_b::new(dst, src1, src2).into(), 2589 _ => unimplemented!(), 2590 }; 2591 self.emit(Inst::External { inst }); 2592 } 2593 2594 /// Perform a logical on vector in `src` and in `address` and put the 2595 /// results in `dst`. 2596 pub fn xmm_vpxor_rmr(&mut self, src: Reg, address: &Address, dst: WritableReg) { 2597 let dst: WritableXmm = dst.map(|r| r.into()); 2598 let address = Self::to_synthetic_amode(address, MemFlags::trusted()); 2599 let inst = asm::inst::vpxor_b::new(dst, src, address).into(); 2600 self.emit(Inst::External { inst }); 2601 } 2602 2603 /// Perform a logical on vectors in `src1` and `src2` and put the results in 2604 /// `dst`. 2605 pub fn xmm_vpxor_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg) { 2606 let dst: WritableXmm = dst.map(|r| r.into()); 2607 let inst = asm::inst::vpxor_b::new(dst, src1, src2).into(); 2608 self.emit(Inst::External { inst }); 2609 } 2610 2611 /// Perform a max operation across two vectors of floats and put the 2612 /// results in `dst`. 2613 pub fn xmm_vmaxp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2614 let dst: WritableXmm = dst.map(|r| r.into()); 2615 let inst = match size { 2616 OperandSize::S32 => asm::inst::vmaxps_b::new(dst, src1, src2).into(), 2617 OperandSize::S64 => asm::inst::vmaxpd_b::new(dst, src1, src2).into(), 2618 _ => unimplemented!(), 2619 }; 2620 self.emit(Inst::External { inst }); 2621 } 2622 2623 // Perform a min operation across two vectors of floats and put the 2624 // results in `dst`. 2625 pub fn xmm_vminp_rrm( 2626 &mut self, 2627 src1: Reg, 2628 src2: &Address, 2629 dst: WritableReg, 2630 size: OperandSize, 2631 ) { 2632 let dst: WritableXmm = dst.map(|r| r.into()); 2633 let address = Self::to_synthetic_amode(src2, MemFlags::trusted()); 2634 let inst = match size { 2635 OperandSize::S32 => asm::inst::vminps_b::new(dst, src1, address).into(), 2636 OperandSize::S64 => asm::inst::vminpd_b::new(dst, src1, address).into(), 2637 _ => unimplemented!(), 2638 }; 2639 self.emit(Inst::External { inst }); 2640 } 2641 2642 // Perform a min operation across two vectors of floats and put the 2643 // results in `dst`. 2644 pub fn xmm_vminp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2645 let dst: WritableXmm = dst.map(|r| r.into()); 2646 let inst = match size { 2647 OperandSize::S32 => asm::inst::vminps_b::new(dst, src1, src2).into(), 2648 OperandSize::S64 => asm::inst::vminpd_b::new(dst, src1, src2).into(), 2649 _ => unimplemented!(), 2650 }; 2651 self.emit(Inst::External { inst }); 2652 } 2653 2654 // Round a vector of floats. 2655 pub fn xmm_vroundp_rri( 2656 &mut self, 2657 src: Reg, 2658 dst: WritableReg, 2659 mode: VroundMode, 2660 size: OperandSize, 2661 ) { 2662 let dst: WritableXmm = dst.map(|r| r.into()); 2663 let imm = match mode { 2664 VroundMode::TowardNearest => 0, 2665 VroundMode::TowardNegativeInfinity => 1, 2666 VroundMode::TowardPositiveInfinity => 2, 2667 VroundMode::TowardZero => 3, 2668 }; 2669 2670 let inst = match size { 2671 OperandSize::S32 => asm::inst::vroundps_rmi::new(dst, src, imm).into(), 2672 OperandSize::S64 => asm::inst::vroundpd_rmi::new(dst, src, imm).into(), 2673 _ => unimplemented!(), 2674 }; 2675 2676 self.emit(Inst::External { inst }); 2677 } 2678 2679 /// Shuffle of vectors of floats. 2680 pub fn xmm_vshufp_rrri( 2681 &mut self, 2682 src1: Reg, 2683 src2: Reg, 2684 dst: WritableReg, 2685 imm: u8, 2686 size: OperandSize, 2687 ) { 2688 let op = match size { 2689 OperandSize::S32 => AvxOpcode::Vshufps, 2690 _ => unimplemented!(), 2691 }; 2692 2693 self.emit(Inst::XmmRmRImmVex { 2694 op, 2695 src1: src1.into(), 2696 src2: src2.into(), 2697 dst: dst.to_reg().into(), 2698 imm, 2699 }); 2700 } 2701 2702 /// Each lane in `src1` is multiplied by the corresponding lane in `src2` 2703 /// producing intermediate 32-bit operands. Each intermediate 32-bit 2704 /// operand is truncated to 18 most significant bits. Rounding is performed 2705 /// by adding 1 to the least significant bit of the 18-bit intermediate 2706 /// result. The 16 bits immediately to the right of the most significant 2707 /// bit of each 18-bit intermediate result is placed in each lane of `dst`. 2708 pub fn xmm_vpmulhrs_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2709 let dst: WritableXmm = dst.map(|r| r.into()); 2710 let inst = match size { 2711 OperandSize::S16 => asm::inst::vpmulhrsw_b::new(dst, src1, src2).into(), 2712 _ => unimplemented!(), 2713 }; 2714 self.emit(Inst::External { inst }); 2715 } 2716 2717 pub fn xmm_vpmuldq_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg) { 2718 let dst: WritableXmm = dst.map(|r| r.into()); 2719 let inst = asm::inst::vpmuldq_b::new(dst, src1, src2).into(); 2720 self.emit(Inst::External { inst }); 2721 } 2722 2723 pub fn xmm_vpmuludq_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg) { 2724 let dst: WritableXmm = dst.map(|r| r.into()); 2725 let inst = asm::inst::vpmuludq_b::new(dst, src1, src2).into(); 2726 self.emit(Inst::External { inst }); 2727 } 2728 2729 pub fn xmm_vpmull_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2730 let dst: WritableXmm = dst.map(|r| r.into()); 2731 let inst = match size { 2732 OperandSize::S16 => asm::inst::vpmullw_b::new(dst, src1, src2).into(), 2733 OperandSize::S32 => asm::inst::vpmulld_b::new(dst, src1, src2).into(), 2734 _ => unimplemented!(), 2735 }; 2736 self.emit(Inst::External { inst }); 2737 } 2738 2739 pub fn xmm_vmulp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2740 let dst: WritableXmm = dst.map(|r| r.into()); 2741 let inst = match size { 2742 OperandSize::S32 => asm::inst::vmulps_b::new(dst, src1, src2).into(), 2743 OperandSize::S64 => asm::inst::vmulpd_b::new(dst, src1, src2).into(), 2744 _ => unimplemented!(), 2745 }; 2746 self.emit(Inst::External { inst }); 2747 } 2748 2749 /// Perform an average operation for the vector of unsigned integers in 2750 /// `src1` and `src2` and put the results in `dst`. 2751 pub fn xmm_vpavg_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2752 let dst: WritableXmm = dst.map(|r| r.into()); 2753 let inst = match size { 2754 OperandSize::S8 => asm::inst::vpavgb_b::new(dst, src1, src2).into(), 2755 OperandSize::S16 => asm::inst::vpavgw_b::new(dst, src1, src2).into(), 2756 _ => unimplemented!(), 2757 }; 2758 self.emit(Inst::External { inst }); 2759 } 2760 2761 /// Divide the vector of floats in `src1` by the vector of floats in `src2` 2762 /// and put the results in `dst`. 2763 pub fn xmm_vdivp_rrr(&mut self, src1: Reg, src2: Reg, dst: WritableReg, size: OperandSize) { 2764 let dst: WritableXmm = dst.map(|r| r.into()); 2765 let inst = match size { 2766 OperandSize::S32 => asm::inst::vdivps_b::new(dst, src1, src2).into(), 2767 OperandSize::S64 => asm::inst::vdivpd_b::new(dst, src1, src2).into(), 2768 _ => unimplemented!(), 2769 }; 2770 self.emit(Inst::External { inst }); 2771 } 2772 2773 /// Compute square roots of vector of floats in `src` and put the results 2774 /// in `dst`. 2775 pub fn xmm_vsqrtp_rr(&mut self, src: Reg, dst: WritableReg, size: OperandSize) { 2776 let dst: WritableXmm = dst.map(|r| r.into()); 2777 let inst = match size { 2778 OperandSize::S32 => asm::inst::vsqrtps_b::new(dst, src).into(), 2779 OperandSize::S64 => asm::inst::vsqrtpd_b::new(dst, src).into(), 2780 _ => unimplemented!(), 2781 }; 2782 self.emit(Inst::External { inst }); 2783 } 2784 2785 /// Multiply and add packed signed and unsigned bytes. 2786 pub fn xmm_vpmaddubs_rmr( 2787 &mut self, 2788 src: Reg, 2789 address: &Address, 2790 dst: WritableReg, 2791 size: OperandSize, 2792 ) { 2793 let address = Self::to_synthetic_amode(address, MemFlags::trusted()); 2794 2795 let op = match size { 2796 OperandSize::S16 => AvxOpcode::Vpmaddubsw, 2797 _ => unimplemented!(), 2798 }; 2799 2800 self.emit(Inst::XmmRmiRVex { 2801 op, 2802 src1: src.into(), 2803 src2: XmmMemImm::unwrap_new(RegMemImm::mem(address)), 2804 dst: dst.to_reg().into(), 2805 }); 2806 } 2807 2808 /// Multiple and add packed integers. 2809 pub fn xmm_vpmaddwd_rmr(&mut self, src: Reg, address: &Address, dst: WritableReg) { 2810 let address = Self::to_synthetic_amode(address, MemFlags::trusted()); 2811 2812 self.emit(Inst::XmmRmiRVex { 2813 op: AvxOpcode::Vpmaddwd, 2814 src1: src.into(), 2815 src2: XmmMemImm::unwrap_new(RegMemImm::mem(address)), 2816 dst: dst.to_reg().into(), 2817 }) 2818 } 2819 } 2820 2821 /// Captures the region in a MachBuffer where an add-with-immediate instruction would be emitted, 2822 /// but the immediate is not yet known. Currently, this implementation expects a 32-bit immediate, 2823 /// so 8 and 16 bit operand sizes are not supported. 2824 pub(crate) struct PatchableAddToReg { 2825 /// The region to be patched in the [`MachBuffer`]. It must contain a valid add instruction 2826 /// sequence, accepting a 32-bit immediate. 2827 region: PatchRegion, 2828 2829 /// The offset into the patchable region where the patchable constant begins. 2830 constant_offset: usize, 2831 } 2832 2833 impl PatchableAddToReg { 2834 /// Create a new [`PatchableAddToReg`] by capturing a region in the output buffer where the 2835 /// add-with-immediate occurs. The [`MachBuffer`] will have and add-with-immediate instruction 2836 /// present in that region, though it will add `0` until the `::finalize` method is called. 2837 /// 2838 /// Currently this implementation expects to be able to patch a 32-bit immediate, which means 2839 /// that 8 and 16-bit addition cannot be supported. 2840 pub(crate) fn new(reg: Reg, size: OperandSize, buf: &mut MachBuffer<Inst>) -> Self { 2841 let open = buf.start_patchable(); 2842 2843 // Emit the opcode and register use for the add instruction. 2844 let start = buf.cur_offset(); 2845 Self::add_inst_bytes(reg, size, buf); 2846 let constant_offset = usize::try_from(buf.cur_offset() - start).unwrap(); 2847 2848 // Emit a placeholder for the 32-bit immediate. 2849 buf.put4(0); 2850 2851 let region = buf.end_patchable(open); 2852 2853 Self { 2854 region, 2855 constant_offset, 2856 } 2857 } 2858 2859 /// Generate the prefix of the add instruction (rex byte (depending on register use), opcode, 2860 /// and register reference). 2861 fn add_inst_bytes(reg: Reg, size: OperandSize, buf: &mut MachBuffer<Inst>) { 2862 match size { 2863 OperandSize::S32 | OperandSize::S64 => {} 2864 _ => { 2865 panic!( 2866 "{}-bit addition is not supported, please see the comment on PatchableAddToReg::new", 2867 size.num_bits(), 2868 ) 2869 } 2870 } 2871 2872 let enc_g = 0; 2873 2874 debug_assert!(reg.is_int()); 2875 let enc_e = u8::try_from(reg.hw_enc()).unwrap(); 2876 2877 RexFlags::from(args::OperandSize::from(size)).emit_two_op(buf, enc_g, enc_e); 2878 2879 // the opcode for an add 2880 buf.put1(0x81); 2881 2882 // the modrm byte 2883 buf.put1(encode_modrm(0b11, enc_g & 7, enc_e & 7)); 2884 } 2885 2886 /// Patch the [`MachBuffer`] with the known constant to be added to the register. The final 2887 /// value is passed in as an i32, but the instruction encoding is fixed when 2888 /// [`PatchableAddToReg::new`] is called. 2889 pub(crate) fn finalize(self, val: i32, buffer: &mut MachBuffer<Inst>) { 2890 let slice = self.region.patch(buffer); 2891 debug_assert_eq!(slice.len(), self.constant_offset + 4); 2892 slice[self.constant_offset..].copy_from_slice(val.to_le_bytes().as_slice()); 2893 } 2894 } 2895