1 //! This module defines x86_64-specific machine instruction types. 2 3 pub use emit_state::EmitState; 4 5 use crate::binemit::{Addend, CodeOffset, Reloc}; 6 use crate::ir::{ExternalName, LibCall, TrapCode, Type, types}; 7 use crate::isa::x64::abi::X64ABIMachineSpec; 8 use crate::isa::x64::inst::regs::{pretty_print_reg, show_ireg_sized}; 9 use crate::isa::x64::settings as x64_settings; 10 use crate::isa::{CallConv, FunctionAlignment}; 11 use crate::{CodegenError, CodegenResult, settings}; 12 use crate::{machinst::*, trace}; 13 use alloc::boxed::Box; 14 use alloc::vec::Vec; 15 use core::slice; 16 use cranelift_assembler_x64 as asm; 17 use cranelift_entity::{Signed, Unsigned}; 18 use smallvec::{SmallVec, smallvec}; 19 use std::fmt::{self, Write}; 20 use std::string::{String, ToString}; 21 22 pub mod args; 23 mod emit; 24 mod emit_state; 25 #[cfg(test)] 26 mod emit_tests; 27 pub mod external; 28 pub mod regs; 29 mod stack_switch; 30 pub mod unwind; 31 32 use args::*; 33 34 //============================================================================= 35 // Instructions (top level): definition 36 37 // `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here. 38 pub use super::lower::isle::generated_code::AtomicRmwSeqOp; 39 pub use super::lower::isle::generated_code::MInst as Inst; 40 41 /// Out-of-line data for return-calls, to keep the size of `Inst` down. 42 #[derive(Clone, Debug)] 43 pub struct ReturnCallInfo<T> { 44 /// Where this call is going. 45 pub dest: T, 46 47 /// The size of the argument area for this return-call, potentially smaller than that of the 48 /// caller, but never larger. 49 pub new_stack_arg_size: u32, 50 51 /// The in-register arguments and their constraints. 52 pub uses: CallArgList, 53 54 /// A temporary for use when moving the return address. 55 pub tmp: WritableGpr, 56 } 57 58 #[test] 59 #[cfg(target_pointer_width = "64")] 60 fn inst_size_test() { 61 // This test will help with unintentionally growing the size 62 // of the Inst enum. 63 assert_eq!(48, std::mem::size_of::<Inst>()); 64 } 65 66 impl Inst { 67 /// Retrieve a list of ISA feature sets in which the instruction is available. An empty list 68 /// indicates that the instruction is available in the baseline feature set (i.e. SSE2 and 69 /// below); more than one `InstructionSet` in the list indicates that the instruction is present 70 /// *any* of the included ISA feature sets. 71 fn available_in_any_isa(&self) -> SmallVec<[InstructionSet; 2]> { 72 match self { 73 // These instructions are part of SSE2, which is a basic requirement in Cranelift, and 74 // don't have to be checked. 75 Inst::AtomicRmwSeq { .. } 76 | Inst::CallKnown { .. } 77 | Inst::CallUnknown { .. } 78 | Inst::ReturnCallKnown { .. } 79 | Inst::ReturnCallUnknown { .. } 80 | Inst::CheckedSRemSeq { .. } 81 | Inst::CheckedSRemSeq8 { .. } 82 | Inst::CvtFloatToSintSeq { .. } 83 | Inst::CvtFloatToUintSeq { .. } 84 | Inst::CvtUint64ToFloatSeq { .. } 85 | Inst::JmpCond { .. } 86 | Inst::JmpCondOr { .. } 87 | Inst::WinchJmpIf { .. } 88 | Inst::JmpKnown { .. } 89 | Inst::JmpTableSeq { .. } 90 | Inst::JmpUnknown { .. } 91 | Inst::LoadExtName { .. } 92 | Inst::MovFromPReg { .. } 93 | Inst::MovToPReg { .. } 94 | Inst::StackProbeLoop { .. } 95 | Inst::Args { .. } 96 | Inst::Rets { .. } 97 | Inst::StackSwitchBasic { .. } 98 | Inst::TrapIf { .. } 99 | Inst::TrapIfAnd { .. } 100 | Inst::TrapIfOr { .. } 101 | Inst::XmmCmove { .. } 102 | Inst::XmmCmpRmR { .. } 103 | Inst::XmmMinMaxSeq { .. } 104 | Inst::XmmUninitializedValue { .. } 105 | Inst::GprUninitializedValue { .. } 106 | Inst::ElfTlsGetAddr { .. } 107 | Inst::MachOTlsGetAddr { .. } 108 | Inst::CoffTlsGetAddr { .. } 109 | Inst::Unwind { .. } 110 | Inst::DummyUse { .. } => smallvec![], 111 112 Inst::Atomic128RmwSeq { .. } | Inst::Atomic128XchgSeq { .. } => { 113 smallvec![InstructionSet::CMPXCHG16b] 114 } 115 116 // These use dynamic SSE opcodes. 117 Inst::XmmRmR { op, .. } 118 | Inst::XmmRmRUnaligned { op, .. } 119 | Inst::XmmRmRImm { op, .. } 120 | Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()], 121 122 Inst::XmmUnaryRmREvex { op, .. } 123 | Inst::XmmRmREvex { op, .. } 124 | Inst::XmmRmREvex3 { op, .. } 125 | Inst::XmmUnaryRmRImmEvex { op, .. } => op.available_from(), 126 127 Inst::XmmRmiRVex { op, .. } 128 | Inst::XmmRmRImmVex { op, .. } 129 | Inst::XmmMovRMVex { op, .. } 130 | Inst::XmmCmpRmRVex { op, .. } => op.available_from(), 131 132 Inst::External { inst } => { 133 use cranelift_assembler_x64::Feature::*; 134 let mut features = smallvec![]; 135 for f in inst.features() { 136 match f { 137 _64b | compat => {} 138 sse => features.push(InstructionSet::SSE), 139 sse2 => features.push(InstructionSet::SSE2), 140 sse3 => features.push(InstructionSet::SSE3), 141 ssse3 => features.push(InstructionSet::SSSE3), 142 sse41 => features.push(InstructionSet::SSE41), 143 sse42 => features.push(InstructionSet::SSE42), 144 bmi1 => features.push(InstructionSet::BMI1), 145 bmi2 => features.push(InstructionSet::BMI2), 146 lzcnt => features.push(InstructionSet::Lzcnt), 147 popcnt => features.push(InstructionSet::Popcnt), 148 avx => features.push(InstructionSet::AVX), 149 avx2 => features.push(InstructionSet::AVX2), 150 cmpxchg16b => features.push(InstructionSet::CMPXCHG16b), 151 fma => features.push(InstructionSet::FMA), 152 } 153 } 154 features 155 } 156 } 157 } 158 } 159 160 // Handy constructors for Insts. 161 162 impl Inst { 163 pub(crate) fn nop(len: u8) -> Self { 164 assert!(len > 0 && len <= 9); 165 let inst = match len { 166 1 => asm::inst::nop_1b::new().into(), 167 2 => asm::inst::nop_2b::new().into(), 168 3 => asm::inst::nop_3b::new().into(), 169 4 => asm::inst::nop_4b::new().into(), 170 5 => asm::inst::nop_5b::new().into(), 171 6 => asm::inst::nop_6b::new().into(), 172 7 => asm::inst::nop_7b::new().into(), 173 8 => asm::inst::nop_8b::new().into(), 174 9 => asm::inst::nop_9b::new().into(), 175 _ => unreachable!("nop length must be between 1 and 9"), 176 }; 177 Self::External { inst } 178 } 179 180 pub(crate) fn addq_mi(dst: Writable<Reg>, simm32: i32) -> Self { 181 let inst = if let Ok(simm8) = i8::try_from(simm32) { 182 asm::inst::addq_mi_sxb::new(dst, simm8).into() 183 } else { 184 asm::inst::addq_mi_sxl::new(dst, simm32).into() 185 }; 186 Inst::External { inst } 187 } 188 189 pub(crate) fn subq_mi(dst: Writable<Reg>, simm32: i32) -> Self { 190 let inst = if let Ok(simm8) = i8::try_from(simm32) { 191 asm::inst::subq_mi_sxb::new(dst, simm8).into() 192 } else { 193 asm::inst::subq_mi_sxl::new(dst, simm32).into() 194 }; 195 Inst::External { inst } 196 } 197 198 /// Writes the `simm64` immedaite into `dst`. 199 /// 200 /// Note that if `dst_size` is less than 64-bits then the upper bits of 201 /// `simm64` will be converted to zero. 202 pub fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst { 203 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); 204 debug_assert!(dst.to_reg().class() == RegClass::Int); 205 let dst = WritableGpr::from_writable_reg(dst).unwrap(); 206 let inst = match dst_size { 207 OperandSize::Size64 => match u32::try_from(simm64) { 208 // If `simm64` is zero-extended use `movl` which zeros the 209 // upper bits. 210 Ok(imm32) => asm::inst::movl_oi::new(dst, imm32).into(), 211 _ => match i32::try_from(simm64.signed()) { 212 // If `simm64` is sign-extended use `movq` which sign the 213 // upper bits. 214 Ok(simm32) => asm::inst::movq_mi_sxl::new(dst, simm32).into(), 215 // fall back to embedding the entire immediate. 216 _ => asm::inst::movabsq_oi::new(dst, simm64).into(), 217 }, 218 }, 219 // FIXME: the input to this function is a logical `simm64` stored 220 // as `u64`. That means that ideally what we would do here is cast 221 // the `simm64` to an `i64`, perform a `i32::try_from()`, then cast 222 // that back to `u32`. That would ensure that the immediate loses 223 // no meaning and has the same logical value. Currently though 224 // Cranelift relies on discarding the upper bits because literals 225 // like `0x8000_0000_u64` fail to convert to an `i32`. In theory 226 // the input to this function should change to `i64`. In the 227 // meantime this is documented as discarding the upper bits, 228 // although this is an old function so that's unlikely to help 229 // much. 230 _ => asm::inst::movl_oi::new(dst, simm64 as u32).into(), 231 }; 232 Inst::External { inst } 233 } 234 235 /// Convenient helper for unary float operations. 236 #[cfg(test)] 237 pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst { 238 src.assert_regclass_is(RegClass::Float); 239 debug_assert!(dst.to_reg().class() == RegClass::Float); 240 Inst::XmmUnaryRmR { 241 op, 242 src: XmmMemAligned::unwrap_new(src), 243 dst: WritableXmm::from_writable_reg(dst).unwrap(), 244 } 245 } 246 247 #[cfg(test)] 248 pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self { 249 src.assert_regclass_is(RegClass::Float); 250 debug_assert!(dst.to_reg().class() == RegClass::Float); 251 Inst::XmmRmR { 252 op, 253 src1: Xmm::unwrap_new(dst.to_reg()), 254 src2: XmmMemAligned::unwrap_new(src), 255 dst: WritableXmm::from_writable_reg(dst).unwrap(), 256 } 257 } 258 259 #[allow(dead_code)] 260 pub(crate) fn xmm_min_max_seq( 261 size: OperandSize, 262 is_min: bool, 263 lhs: Reg, 264 rhs: Reg, 265 dst: Writable<Reg>, 266 ) -> Inst { 267 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64])); 268 debug_assert_eq!(lhs.class(), RegClass::Float); 269 debug_assert_eq!(rhs.class(), RegClass::Float); 270 debug_assert_eq!(dst.to_reg().class(), RegClass::Float); 271 Inst::XmmMinMaxSeq { 272 size, 273 is_min, 274 lhs: Xmm::unwrap_new(lhs), 275 rhs: Xmm::unwrap_new(rhs), 276 dst: WritableXmm::from_writable_reg(dst).unwrap(), 277 } 278 } 279 280 pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst { 281 src.assert_regclass_is(RegClass::Int); 282 debug_assert!(dst.to_reg().class() == RegClass::Int); 283 let src = match src { 284 RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()), 285 RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()), 286 }; 287 let inst = match ext_mode { 288 ExtMode::BL => asm::inst::movzbl_rm::new(dst, src).into(), 289 ExtMode::BQ => asm::inst::movzbq_rm::new(dst, src).into(), 290 ExtMode::WL => asm::inst::movzwl_rm::new(dst, src).into(), 291 ExtMode::WQ => asm::inst::movzwq_rm::new(dst, src).into(), 292 ExtMode::LQ => { 293 // This instruction selection may seem strange but is correct in 294 // 64-bit mode: section 3.4.1.1 of the Intel manual says that 295 // "32-bit operands generate a 32-bit result, zero-extended to a 296 // 64-bit result in the destination general-purpose register." 297 // This is applicable beyond `mov` but we use this fact to 298 // zero-extend `src` into `dst`. 299 asm::inst::movl_rm::new(dst, src).into() 300 } 301 }; 302 Inst::External { inst } 303 } 304 305 pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst { 306 src.assert_regclass_is(RegClass::Int); 307 debug_assert!(dst.to_reg().class() == RegClass::Int); 308 let src = match src { 309 RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()), 310 RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()), 311 }; 312 let inst = match ext_mode { 313 ExtMode::BL => asm::inst::movsbl_rm::new(dst, src).into(), 314 ExtMode::BQ => asm::inst::movsbq_rm::new(dst, src).into(), 315 ExtMode::WL => asm::inst::movswl_rm::new(dst, src).into(), 316 ExtMode::WQ => asm::inst::movswq_rm::new(dst, src).into(), 317 ExtMode::LQ => asm::inst::movslq_rm::new(dst, src).into(), 318 }; 319 Inst::External { inst } 320 } 321 322 /// Compares `src1` against `src2` 323 pub(crate) fn cmp_mi_sxb(size: OperandSize, src1: Gpr, src2: i8) -> Inst { 324 let inst = match size { 325 OperandSize::Size8 => asm::inst::cmpb_mi::new(src1, src2.unsigned()).into(), 326 OperandSize::Size16 => asm::inst::cmpw_mi_sxb::new(src1, src2).into(), 327 OperandSize::Size32 => asm::inst::cmpl_mi_sxb::new(src1, src2).into(), 328 OperandSize::Size64 => asm::inst::cmpq_mi_sxb::new(src1, src2).into(), 329 }; 330 Inst::External { inst } 331 } 332 333 pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst { 334 Inst::TrapIf { cc, trap_code } 335 } 336 337 pub(crate) fn call_known(info: Box<CallInfo<ExternalName>>) -> Inst { 338 Inst::CallKnown { info } 339 } 340 341 pub(crate) fn call_unknown(info: Box<CallInfo<RegMem>>) -> Inst { 342 info.dest.assert_regclass_is(RegClass::Int); 343 Inst::CallUnknown { info } 344 } 345 346 pub(crate) fn jmp_known(dst: MachLabel) -> Inst { 347 Inst::JmpKnown { dst } 348 } 349 350 pub(crate) fn jmp_unknown(target: RegMem) -> Inst { 351 target.assert_regclass_is(RegClass::Int); 352 Inst::JmpUnknown { target } 353 } 354 355 /// Choose which instruction to use for loading a register value from memory. For loads smaller 356 /// than 64 bits, this method expects a way to extend the value (i.e. [ExtKind::SignExtend], 357 /// [ExtKind::ZeroExtend]); loads with no extension necessary will ignore this. 358 pub(crate) fn load( 359 ty: Type, 360 from_addr: impl Into<SyntheticAmode>, 361 to_reg: Writable<Reg>, 362 ext_kind: ExtKind, 363 ) -> Inst { 364 let rc = to_reg.to_reg().class(); 365 match rc { 366 RegClass::Int => { 367 let ext_mode = match ty.bytes() { 368 1 => Some(ExtMode::BQ), 369 2 => Some(ExtMode::WQ), 370 4 => Some(ExtMode::LQ), 371 8 => None, 372 _ => unreachable!("the type should never use a scalar load: {}", ty), 373 }; 374 if let Some(ext_mode) = ext_mode { 375 // Values smaller than 64 bits must be extended in some way. 376 match ext_kind { 377 ExtKind::SignExtend => { 378 Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg) 379 } 380 ExtKind::ZeroExtend => { 381 Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg) 382 } 383 ExtKind::None => { 384 panic!("expected an extension kind for extension mode: {ext_mode:?}") 385 } 386 } 387 } else { 388 // 64-bit values can be moved directly. 389 let from_addr = asm::GprMem::from(from_addr.into()); 390 Inst::External { 391 inst: asm::inst::movq_rm::new(to_reg, from_addr).into(), 392 } 393 } 394 } 395 RegClass::Float => { 396 let to_reg = to_reg.map(|r| Xmm::new(r).unwrap()); 397 let from_addr = from_addr.into(); 398 let inst = match ty { 399 types::F16 | types::I8X2 => { 400 panic!("loading a f16 or i8x2 requires multiple instructions") 401 } 402 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => { 403 asm::inst::movss_a_m::new(to_reg, from_addr).into() 404 } 405 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => { 406 asm::inst::movsd_a_m::new(to_reg, from_addr).into() 407 } 408 types::F32X4 => asm::inst::movups_a::new(to_reg, from_addr).into(), 409 types::F64X2 => asm::inst::movupd_a::new(to_reg, from_addr).into(), 410 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => { 411 asm::inst::movdqu_a::new(to_reg, from_addr).into() 412 } 413 _ => unimplemented!("unable to load type: {}", ty), 414 }; 415 Inst::External { inst } 416 } 417 RegClass::Vector => unreachable!(), 418 } 419 } 420 421 /// Choose which instruction to use for storing a register value to memory. 422 pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst { 423 let rc = from_reg.class(); 424 let to_addr = to_addr.into(); 425 let inst = match rc { 426 RegClass::Int => { 427 let from_reg = Gpr::unwrap_new(from_reg); 428 match ty { 429 types::I8 => asm::inst::movb_mr::new(to_addr, from_reg).into(), 430 types::I16 => asm::inst::movw_mr::new(to_addr, from_reg).into(), 431 types::I32 => asm::inst::movl_mr::new(to_addr, from_reg).into(), 432 types::I64 => asm::inst::movq_mr::new(to_addr, from_reg).into(), 433 _ => unreachable!(), 434 } 435 } 436 RegClass::Float => { 437 let from_reg = Xmm::new(from_reg).unwrap(); 438 match ty { 439 types::F16 | types::I8X2 => { 440 panic!("storing a f16 or i8x2 requires multiple instructions") 441 } 442 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => { 443 asm::inst::movss_c_m::new(to_addr, from_reg).into() 444 } 445 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => { 446 asm::inst::movsd_c_m::new(to_addr, from_reg).into() 447 } 448 types::F32X4 => asm::inst::movups_b::new(to_addr, from_reg).into(), 449 types::F64X2 => asm::inst::movupd_b::new(to_addr, from_reg).into(), 450 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => { 451 asm::inst::movdqu_b::new(to_addr, from_reg).into() 452 } 453 _ => unimplemented!("unable to store type: {}", ty), 454 } 455 } 456 RegClass::Vector => unreachable!(), 457 }; 458 Inst::External { inst } 459 } 460 } 461 462 //============================================================================= 463 // Instructions: printing 464 465 impl PrettyPrint for Inst { 466 fn pretty_print(&self, _size: u8) -> String { 467 fn ljustify(s: String) -> String { 468 let w = 7; 469 if s.len() >= w { 470 s 471 } else { 472 let need = usize::min(w, w - s.len()); 473 s + &format!("{nil: <width$}", nil = "", width = need) 474 } 475 } 476 477 fn ljustify2(s1: String, s2: String) -> String { 478 ljustify(s1 + &s2) 479 } 480 481 match self { 482 Inst::CheckedSRemSeq { 483 size, 484 divisor, 485 dividend_lo, 486 dividend_hi, 487 dst_quotient, 488 dst_remainder, 489 } => { 490 let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes()); 491 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes()); 492 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes()); 493 let dst_quotient = 494 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes()); 495 let dst_remainder = 496 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes()); 497 format!( 498 "checked_srem_seq {dividend_lo}, {dividend_hi}, \ 499 {divisor}, {dst_quotient}, {dst_remainder}", 500 ) 501 } 502 503 Inst::CheckedSRemSeq8 { 504 divisor, 505 dividend, 506 dst, 507 } => { 508 let divisor = pretty_print_reg(divisor.to_reg(), 1); 509 let dividend = pretty_print_reg(dividend.to_reg(), 1); 510 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1); 511 format!("checked_srem_seq {dividend}, {divisor}, {dst}") 512 } 513 514 Inst::XmmUnaryRmR { op, src, dst, .. } => { 515 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size()); 516 let src = src.pretty_print(op.src_size()); 517 let op = ljustify(op.to_string()); 518 format!("{op} {src}, {dst}") 519 } 520 521 Inst::XmmUnaryRmREvex { op, src, dst, .. } => { 522 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 523 let src = src.pretty_print(8); 524 let op = ljustify(op.to_string()); 525 format!("{op} {src}, {dst}") 526 } 527 528 Inst::XmmUnaryRmRImmEvex { 529 op, src, dst, imm, .. 530 } => { 531 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 532 let src = src.pretty_print(8); 533 let op = ljustify(op.to_string()); 534 format!("{op} ${imm}, {src}, {dst}") 535 } 536 537 Inst::XmmMovRMVex { op, src, dst, .. } => { 538 let src = pretty_print_reg(src.to_reg(), 8); 539 let dst = dst.pretty_print(8); 540 let op = ljustify(op.to_string()); 541 format!("{op} {src}, {dst}") 542 } 543 544 Inst::XmmRmR { 545 op, 546 src1, 547 src2, 548 dst, 549 .. 550 } => { 551 let src1 = pretty_print_reg(src1.to_reg(), 8); 552 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 553 let src2 = src2.pretty_print(8); 554 let op = ljustify(op.to_string()); 555 format!("{op} {src1}, {src2}, {dst}") 556 } 557 558 Inst::XmmRmRUnaligned { 559 op, 560 src1, 561 src2, 562 dst, 563 .. 564 } => { 565 let src1 = pretty_print_reg(src1.to_reg(), 8); 566 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 567 let src2 = src2.pretty_print(8); 568 let op = ljustify(op.to_string()); 569 format!("{op} {src1}, {src2}, {dst}") 570 } 571 572 Inst::XmmRmiRVex { 573 op, 574 src1, 575 src2, 576 dst, 577 .. 578 } => { 579 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 580 let src1 = pretty_print_reg(src1.to_reg(), 8); 581 let src2 = src2.pretty_print(8); 582 let op = ljustify(op.to_string()); 583 format!("{op} {src1}, {src2}, {dst}") 584 } 585 586 Inst::XmmRmRImmVex { 587 op, 588 src1, 589 src2, 590 dst, 591 imm, 592 .. 593 } => { 594 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 595 let src1 = pretty_print_reg(src1.to_reg(), 8); 596 let src2 = src2.pretty_print(8); 597 let op = ljustify(op.to_string()); 598 format!("{op} ${imm}, {src1}, {src2}, {dst}") 599 } 600 601 Inst::XmmRmREvex { 602 op, 603 src1, 604 src2, 605 dst, 606 .. 607 } => { 608 let src1 = pretty_print_reg(src1.to_reg(), 8); 609 let src2 = src2.pretty_print(8); 610 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 611 let op = ljustify(op.to_string()); 612 format!("{op} {src2}, {src1}, {dst}") 613 } 614 615 Inst::XmmRmREvex3 { 616 op, 617 src1, 618 src2, 619 src3, 620 dst, 621 .. 622 } => { 623 let src1 = pretty_print_reg(src1.to_reg(), 8); 624 let src2 = pretty_print_reg(src2.to_reg(), 8); 625 let src3 = src3.pretty_print(8); 626 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 627 let op = ljustify(op.to_string()); 628 format!("{op} {src3}, {src2}, {src1}, {dst}") 629 } 630 631 Inst::XmmMinMaxSeq { 632 lhs, 633 rhs, 634 dst, 635 is_min, 636 size, 637 } => { 638 let rhs = pretty_print_reg(rhs.to_reg(), 8); 639 let lhs = pretty_print_reg(lhs.to_reg(), 8); 640 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 641 let op = ljustify2( 642 if *is_min { 643 "xmm min seq ".to_string() 644 } else { 645 "xmm max seq ".to_string() 646 }, 647 format!("f{}", size.to_bits()), 648 ); 649 format!("{op} {lhs}, {rhs}, {dst}") 650 } 651 652 Inst::XmmRmRImm { 653 op, 654 src1, 655 src2, 656 dst, 657 imm, 658 size, 659 .. 660 } => { 661 let src1 = pretty_print_reg(*src1, 8); 662 let dst = pretty_print_reg(dst.to_reg(), 8); 663 let src2 = src2.pretty_print(8); 664 let op = ljustify(format!( 665 "{}{}", 666 op.to_string(), 667 if *size == OperandSize::Size64 { 668 ".w" 669 } else { 670 "" 671 } 672 )); 673 format!("{op} ${imm}, {src1}, {src2}, {dst}") 674 } 675 676 Inst::XmmUninitializedValue { dst } => { 677 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 678 let op = ljustify("uninit".into()); 679 format!("{op} {dst}") 680 } 681 682 Inst::GprUninitializedValue { dst } => { 683 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 684 let op = ljustify("uninit".into()); 685 format!("{op} {dst}") 686 } 687 688 Inst::XmmCmpRmR { op, src1, src2 } => { 689 let src1 = pretty_print_reg(src1.to_reg(), 8); 690 let src2 = src2.pretty_print(8); 691 let op = ljustify(op.to_string()); 692 format!("{op} {src2}, {src1}") 693 } 694 695 Inst::XmmCmpRmRVex { op, src1, src2 } => { 696 let src1 = pretty_print_reg(src1.to_reg(), 8); 697 let src2 = src2.pretty_print(8); 698 format!("{} {src2}, {src1}", ljustify(op.to_string())) 699 } 700 701 Inst::CvtUint64ToFloatSeq { 702 src, 703 dst, 704 dst_size, 705 tmp_gpr1, 706 tmp_gpr2, 707 .. 708 } => { 709 let src = pretty_print_reg(src.to_reg(), 8); 710 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes()); 711 let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8); 712 let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8); 713 let op = ljustify(format!( 714 "u64_to_{}_seq", 715 if *dst_size == OperandSize::Size64 { 716 "f64" 717 } else { 718 "f32" 719 } 720 )); 721 format!("{op} {src}, {dst}, {tmp_gpr1}, {tmp_gpr2}") 722 } 723 724 Inst::CvtFloatToSintSeq { 725 src, 726 dst, 727 src_size, 728 dst_size, 729 tmp_xmm, 730 tmp_gpr, 731 is_saturating, 732 } => { 733 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes()); 734 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes()); 735 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8); 736 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8); 737 let op = ljustify(format!( 738 "cvt_float{}_to_sint{}{}_seq", 739 src_size.to_bits(), 740 dst_size.to_bits(), 741 if *is_saturating { "_sat" } else { "" }, 742 )); 743 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}") 744 } 745 746 Inst::CvtFloatToUintSeq { 747 src, 748 dst, 749 src_size, 750 dst_size, 751 tmp_gpr, 752 tmp_xmm, 753 tmp_xmm2, 754 is_saturating, 755 } => { 756 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes()); 757 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes()); 758 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8); 759 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8); 760 let tmp_xmm2 = pretty_print_reg(tmp_xmm2.to_reg().to_reg(), 8); 761 let op = ljustify(format!( 762 "cvt_float{}_to_uint{}{}_seq", 763 src_size.to_bits(), 764 dst_size.to_bits(), 765 if *is_saturating { "_sat" } else { "" }, 766 )); 767 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}, {tmp_xmm2}") 768 } 769 770 Inst::MovFromPReg { src, dst } => { 771 let src: Reg = (*src).into(); 772 let src = regs::show_ireg_sized(src, 8); 773 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 774 let op = ljustify("movq".to_string()); 775 format!("{op} {src}, {dst}") 776 } 777 778 Inst::MovToPReg { src, dst } => { 779 let src = pretty_print_reg(src.to_reg(), 8); 780 let dst: Reg = (*dst).into(); 781 let dst = regs::show_ireg_sized(dst, 8); 782 let op = ljustify("movq".to_string()); 783 format!("{op} {src}, {dst}") 784 } 785 786 Inst::XmmCmove { 787 ty, 788 cc, 789 consequent, 790 alternative, 791 dst, 792 .. 793 } => { 794 let size = u8::try_from(ty.bytes()).unwrap(); 795 let alternative = pretty_print_reg(alternative.to_reg(), size); 796 let dst = pretty_print_reg(dst.to_reg().to_reg(), size); 797 let consequent = pretty_print_reg(consequent.to_reg(), size); 798 let suffix = match *ty { 799 types::F64 => "sd", 800 types::F32 => "ss", 801 types::F16 => "ss", 802 types::F32X4 => "aps", 803 types::F64X2 => "apd", 804 _ => "dqa", 805 }; 806 let cc = cc.invert(); 807 format!( 808 "mov{suffix} {alternative}, {dst}; \ 809 j{cc} $next; \ 810 mov{suffix} {consequent}, {dst}; \ 811 $next:" 812 ) 813 } 814 815 Inst::StackProbeLoop { 816 tmp, 817 frame_size, 818 guard_size, 819 } => { 820 let tmp = pretty_print_reg(tmp.to_reg(), 8); 821 let op = ljustify("stack_probe_loop".to_string()); 822 format!("{op} {tmp}, frame_size={frame_size}, guard_size={guard_size}") 823 } 824 825 Inst::CallKnown { info } => { 826 let op = ljustify("call".to_string()); 827 let try_call = info 828 .try_call_info 829 .as_ref() 830 .map(|tci| pretty_print_try_call(tci)) 831 .unwrap_or_default(); 832 format!("{op} {:?}{try_call}", info.dest) 833 } 834 835 Inst::CallUnknown { info } => { 836 let dest = info.dest.pretty_print(8); 837 let op = ljustify("call".to_string()); 838 let try_call = info 839 .try_call_info 840 .as_ref() 841 .map(|tci| pretty_print_try_call(tci)) 842 .unwrap_or_default(); 843 format!("{op} *{dest}{try_call}") 844 } 845 846 Inst::ReturnCallKnown { info } => { 847 let ReturnCallInfo { 848 uses, 849 new_stack_arg_size, 850 tmp, 851 dest, 852 } = &**info; 853 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8); 854 let mut s = format!("return_call_known {dest:?} ({new_stack_arg_size}) tmp={tmp}"); 855 for ret in uses { 856 let preg = regs::show_reg(ret.preg); 857 let vreg = pretty_print_reg(ret.vreg, 8); 858 write!(&mut s, " {vreg}={preg}").unwrap(); 859 } 860 s 861 } 862 863 Inst::ReturnCallUnknown { info } => { 864 let ReturnCallInfo { 865 uses, 866 new_stack_arg_size, 867 tmp, 868 dest, 869 } = &**info; 870 let callee = pretty_print_reg(*dest, 8); 871 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8); 872 let mut s = 873 format!("return_call_unknown {callee} ({new_stack_arg_size}) tmp={tmp}"); 874 for ret in uses { 875 let preg = regs::show_reg(ret.preg); 876 let vreg = pretty_print_reg(ret.vreg, 8); 877 write!(&mut s, " {vreg}={preg}").unwrap(); 878 } 879 s 880 } 881 882 Inst::Args { args } => { 883 let mut s = "args".to_string(); 884 for arg in args { 885 let preg = regs::show_reg(arg.preg); 886 let def = pretty_print_reg(arg.vreg.to_reg(), 8); 887 write!(&mut s, " {def}={preg}").unwrap(); 888 } 889 s 890 } 891 892 Inst::Rets { rets } => { 893 let mut s = "rets".to_string(); 894 for ret in rets { 895 let preg = regs::show_reg(ret.preg); 896 let vreg = pretty_print_reg(ret.vreg, 8); 897 write!(&mut s, " {vreg}={preg}").unwrap(); 898 } 899 s 900 } 901 902 Inst::StackSwitchBasic { 903 store_context_ptr, 904 load_context_ptr, 905 in_payload0, 906 out_payload0, 907 } => { 908 let store_context_ptr = pretty_print_reg(**store_context_ptr, 8); 909 let load_context_ptr = pretty_print_reg(**load_context_ptr, 8); 910 let in_payload0 = pretty_print_reg(**in_payload0, 8); 911 let out_payload0 = pretty_print_reg(*out_payload0.to_reg(), 8); 912 format!( 913 "{out_payload0} = stack_switch_basic {store_context_ptr}, {load_context_ptr}, {in_payload0}" 914 ) 915 } 916 917 Inst::JmpKnown { dst } => { 918 let op = ljustify("jmp".to_string()); 919 let dst = dst.to_string(); 920 format!("{op} {dst}") 921 } 922 923 Inst::WinchJmpIf { cc, taken } => { 924 let taken = taken.to_string(); 925 let op = ljustify2("j".to_string(), cc.to_string()); 926 format!("{op} {taken}") 927 } 928 929 Inst::JmpCondOr { 930 cc1, 931 cc2, 932 taken, 933 not_taken, 934 } => { 935 let taken = taken.to_string(); 936 let not_taken = not_taken.to_string(); 937 let op = ljustify(format!("j{cc1},{cc2}")); 938 format!("{op} {taken}; j {not_taken}") 939 } 940 941 Inst::JmpCond { 942 cc, 943 taken, 944 not_taken, 945 } => { 946 let taken = taken.to_string(); 947 let not_taken = not_taken.to_string(); 948 let op = ljustify2("j".to_string(), cc.to_string()); 949 format!("{op} {taken}; j {not_taken}") 950 } 951 952 Inst::JmpTableSeq { 953 idx, tmp1, tmp2, .. 954 } => { 955 let idx = pretty_print_reg(*idx, 8); 956 let tmp1 = pretty_print_reg(tmp1.to_reg(), 8); 957 let tmp2 = pretty_print_reg(tmp2.to_reg(), 8); 958 let op = ljustify("br_table".into()); 959 format!("{op} {idx}, {tmp1}, {tmp2}") 960 } 961 962 Inst::JmpUnknown { target } => { 963 let target = target.pretty_print(8); 964 let op = ljustify("jmp".to_string()); 965 format!("{op} *{target}") 966 } 967 968 Inst::TrapIf { cc, trap_code, .. } => { 969 format!("j{cc} #trap={trap_code}") 970 } 971 972 Inst::TrapIfAnd { 973 cc1, 974 cc2, 975 trap_code, 976 .. 977 } => { 978 let cc1 = cc1.invert(); 979 let cc2 = cc2.invert(); 980 format!("trap_if_and {cc1}, {cc2}, {trap_code}") 981 } 982 983 Inst::TrapIfOr { 984 cc1, 985 cc2, 986 trap_code, 987 .. 988 } => { 989 let cc2 = cc2.invert(); 990 format!("trap_if_or {cc1}, {cc2}, {trap_code}") 991 } 992 993 Inst::LoadExtName { 994 dst, name, offset, .. 995 } => { 996 let dst = pretty_print_reg(*dst.to_reg(), 8); 997 let name = name.display(None); 998 let op = ljustify("load_ext_name".into()); 999 format!("{op} {name}+{offset}, {dst}") 1000 } 1001 1002 Inst::AtomicRmwSeq { ty, op, .. } => { 1003 let ty = ty.bits(); 1004 format!( 1005 "atomically {{ {ty}_bits_at_[%r9] {op:?}= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }}" 1006 ) 1007 } 1008 1009 Inst::Atomic128RmwSeq { 1010 op, 1011 mem, 1012 operand_low, 1013 operand_high, 1014 temp_low, 1015 temp_high, 1016 dst_old_low, 1017 dst_old_high, 1018 } => { 1019 let operand_low = pretty_print_reg(**operand_low, 8); 1020 let operand_high = pretty_print_reg(**operand_high, 8); 1021 let temp_low = pretty_print_reg(*temp_low.to_reg(), 8); 1022 let temp_high = pretty_print_reg(*temp_high.to_reg(), 8); 1023 let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8); 1024 let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8); 1025 let mem = mem.pretty_print(16); 1026 format!( 1027 "atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem} = {temp_high}:{temp_low} }}" 1028 ) 1029 } 1030 1031 Inst::Atomic128XchgSeq { 1032 mem, 1033 operand_low, 1034 operand_high, 1035 dst_old_low, 1036 dst_old_high, 1037 } => { 1038 let operand_low = pretty_print_reg(**operand_low, 8); 1039 let operand_high = pretty_print_reg(**operand_high, 8); 1040 let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8); 1041 let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8); 1042 let mem = mem.pretty_print(16); 1043 format!( 1044 "atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {mem} = {operand_high}:{operand_low} }}" 1045 ) 1046 } 1047 1048 Inst::ElfTlsGetAddr { symbol, dst } => { 1049 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 1050 format!("{dst} = elf_tls_get_addr {symbol:?}") 1051 } 1052 1053 Inst::MachOTlsGetAddr { symbol, dst } => { 1054 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 1055 format!("{dst} = macho_tls_get_addr {symbol:?}") 1056 } 1057 1058 Inst::CoffTlsGetAddr { symbol, dst, tmp } => { 1059 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8); 1060 let tmp = tmp.to_reg().to_reg(); 1061 1062 let mut s = format!("{dst} = coff_tls_get_addr {symbol:?}"); 1063 if tmp.is_virtual() { 1064 let tmp = show_ireg_sized(tmp, 8); 1065 write!(&mut s, ", {tmp}").unwrap(); 1066 }; 1067 1068 s 1069 } 1070 1071 Inst::Unwind { inst } => format!("unwind {inst:?}"), 1072 1073 Inst::DummyUse { reg } => { 1074 let reg = pretty_print_reg(*reg, 8); 1075 format!("dummy_use {reg}") 1076 } 1077 1078 Inst::External { inst } => { 1079 format!("{inst}") 1080 } 1081 } 1082 } 1083 } 1084 1085 fn pretty_print_try_call(info: &TryCallInfo) -> String { 1086 let dests = info 1087 .exception_dests 1088 .iter() 1089 .map(|(tag, label)| format!("{tag:?}: {label:?}")) 1090 .collect::<Vec<_>>() 1091 .join(", "); 1092 format!("; jmp {:?}; catch [{dests}]", info.continuation) 1093 } 1094 1095 impl fmt::Debug for Inst { 1096 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { 1097 write!(fmt, "{}", self.pretty_print_inst(&mut Default::default())) 1098 } 1099 } 1100 1101 fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { 1102 // Note: because we need to statically know the indices of each 1103 // reg in the operands list in order to fetch its allocation 1104 // later, we put the variable-operand-count bits (the RegMem, 1105 // RegMemImm, etc args) last. regalloc2 doesn't care what order 1106 // the operands come in; they can be freely reordered. 1107 1108 // N.B.: we MUST keep the below in careful sync with (i) emission, 1109 // in `emit.rs`, and (ii) pretty-printing, in the `pretty_print` 1110 // method above. 1111 match inst { 1112 Inst::CheckedSRemSeq { 1113 divisor, 1114 dividend_lo, 1115 dividend_hi, 1116 dst_quotient, 1117 dst_remainder, 1118 .. 1119 } => { 1120 collector.reg_use(divisor); 1121 collector.reg_fixed_use(dividend_lo, regs::rax()); 1122 collector.reg_fixed_use(dividend_hi, regs::rdx()); 1123 collector.reg_fixed_def(dst_quotient, regs::rax()); 1124 collector.reg_fixed_def(dst_remainder, regs::rdx()); 1125 } 1126 Inst::CheckedSRemSeq8 { 1127 divisor, 1128 dividend, 1129 dst, 1130 .. 1131 } => { 1132 collector.reg_use(divisor); 1133 collector.reg_fixed_use(dividend, regs::rax()); 1134 collector.reg_fixed_def(dst, regs::rax()); 1135 } 1136 Inst::XmmUnaryRmR { src, dst, .. } => { 1137 collector.reg_def(dst); 1138 src.get_operands(collector); 1139 } 1140 Inst::XmmUnaryRmREvex { src, dst, .. } | Inst::XmmUnaryRmRImmEvex { src, dst, .. } => { 1141 collector.reg_def(dst); 1142 src.get_operands(collector); 1143 } 1144 Inst::XmmRmR { 1145 src1, src2, dst, .. 1146 } => { 1147 collector.reg_use(src1); 1148 collector.reg_reuse_def(dst, 0); 1149 src2.get_operands(collector); 1150 } 1151 Inst::XmmRmRUnaligned { 1152 src1, src2, dst, .. 1153 } => { 1154 collector.reg_use(src1); 1155 collector.reg_reuse_def(dst, 0); 1156 src2.get_operands(collector); 1157 } 1158 Inst::XmmRmiRVex { 1159 src1, src2, dst, .. 1160 } => { 1161 collector.reg_def(dst); 1162 collector.reg_use(src1); 1163 src2.get_operands(collector); 1164 } 1165 Inst::XmmRmRImmVex { 1166 src1, src2, dst, .. 1167 } => { 1168 collector.reg_def(dst); 1169 collector.reg_use(src1); 1170 src2.get_operands(collector); 1171 } 1172 Inst::XmmRmREvex { 1173 op, 1174 src1, 1175 src2, 1176 dst, 1177 .. 1178 } => { 1179 assert_ne!(*op, Avx512Opcode::Vpermi2b); 1180 collector.reg_use(src1); 1181 src2.get_operands(collector); 1182 collector.reg_def(dst); 1183 } 1184 Inst::XmmRmREvex3 { 1185 op, 1186 src1, 1187 src2, 1188 src3, 1189 dst, 1190 .. 1191 } => { 1192 assert_eq!(*op, Avx512Opcode::Vpermi2b); 1193 collector.reg_use(src1); 1194 collector.reg_use(src2); 1195 src3.get_operands(collector); 1196 collector.reg_reuse_def(dst, 0); // Reuse `src1`. 1197 } 1198 Inst::XmmRmRImm { 1199 src1, src2, dst, .. 1200 } => { 1201 collector.reg_use(src1); 1202 collector.reg_reuse_def(dst, 0); 1203 src2.get_operands(collector); 1204 } 1205 Inst::XmmUninitializedValue { dst } => collector.reg_def(dst), 1206 Inst::GprUninitializedValue { dst } => collector.reg_def(dst), 1207 Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => { 1208 collector.reg_use(rhs); 1209 collector.reg_use(lhs); 1210 collector.reg_reuse_def(dst, 0); // Reuse RHS. 1211 } 1212 Inst::XmmMovRMVex { src, dst, .. } => { 1213 collector.reg_use(src); 1214 dst.get_operands(collector); 1215 } 1216 Inst::XmmCmpRmR { src1, src2, .. } => { 1217 collector.reg_use(src1); 1218 src2.get_operands(collector); 1219 } 1220 Inst::XmmCmpRmRVex { src1, src2, .. } => { 1221 collector.reg_use(src1); 1222 src2.get_operands(collector); 1223 } 1224 Inst::MovFromPReg { dst, src } => { 1225 debug_assert!(dst.to_reg().to_reg().is_virtual()); 1226 collector.reg_fixed_nonallocatable(*src); 1227 collector.reg_def(dst); 1228 } 1229 Inst::MovToPReg { dst, src } => { 1230 debug_assert!(src.to_reg().is_virtual()); 1231 collector.reg_use(src); 1232 collector.reg_fixed_nonallocatable(*dst); 1233 } 1234 Inst::CvtUint64ToFloatSeq { 1235 src, 1236 dst, 1237 tmp_gpr1, 1238 tmp_gpr2, 1239 .. 1240 } => { 1241 collector.reg_use(src); 1242 collector.reg_early_def(dst); 1243 collector.reg_early_def(tmp_gpr1); 1244 collector.reg_early_def(tmp_gpr2); 1245 } 1246 Inst::CvtFloatToSintSeq { 1247 src, 1248 dst, 1249 tmp_xmm, 1250 tmp_gpr, 1251 .. 1252 } => { 1253 collector.reg_use(src); 1254 collector.reg_early_def(dst); 1255 collector.reg_early_def(tmp_gpr); 1256 collector.reg_early_def(tmp_xmm); 1257 } 1258 Inst::CvtFloatToUintSeq { 1259 src, 1260 dst, 1261 tmp_gpr, 1262 tmp_xmm, 1263 tmp_xmm2, 1264 .. 1265 } => { 1266 collector.reg_use(src); 1267 collector.reg_early_def(dst); 1268 collector.reg_early_def(tmp_gpr); 1269 collector.reg_early_def(tmp_xmm); 1270 collector.reg_early_def(tmp_xmm2); 1271 } 1272 1273 Inst::XmmCmove { 1274 consequent, 1275 alternative, 1276 dst, 1277 .. 1278 } => { 1279 collector.reg_use(alternative); 1280 collector.reg_reuse_def(dst, 0); 1281 collector.reg_use(consequent); 1282 } 1283 Inst::StackProbeLoop { tmp, .. } => { 1284 collector.reg_early_def(tmp); 1285 } 1286 1287 Inst::CallKnown { info } => { 1288 // Probestack is special and is only inserted after 1289 // regalloc, so we do not need to represent its ABI to the 1290 // register allocator. Assert that we don't alter that 1291 // arrangement. 1292 let CallInfo { 1293 uses, 1294 defs, 1295 clobbers, 1296 dest, 1297 .. 1298 } = &mut **info; 1299 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack)); 1300 for CallArgPair { vreg, preg } in uses { 1301 collector.reg_fixed_use(vreg, *preg); 1302 } 1303 for CallRetPair { vreg, location } in defs { 1304 match location { 1305 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg), 1306 RetLocation::Stack(..) => collector.any_def(vreg), 1307 } 1308 } 1309 collector.reg_clobbers(*clobbers); 1310 } 1311 1312 Inst::CallUnknown { info } => { 1313 let CallInfo { 1314 uses, 1315 defs, 1316 clobbers, 1317 callee_conv, 1318 dest, 1319 .. 1320 } = &mut **info; 1321 match dest { 1322 RegMem::Reg { reg } if *callee_conv == CallConv::Winch => { 1323 // TODO(https://github.com/bytecodealliance/regalloc2/issues/145): 1324 // This shouldn't be a fixed register constraint. r10 is caller-saved, so this 1325 // should be safe to use. 1326 collector.reg_fixed_use(reg, regs::r10()); 1327 } 1328 _ => dest.get_operands(collector), 1329 } 1330 for CallArgPair { vreg, preg } in uses { 1331 collector.reg_fixed_use(vreg, *preg); 1332 } 1333 for CallRetPair { vreg, location } in defs { 1334 match location { 1335 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg), 1336 RetLocation::Stack(..) => collector.any_def(vreg), 1337 } 1338 } 1339 collector.reg_clobbers(*clobbers); 1340 } 1341 Inst::StackSwitchBasic { 1342 store_context_ptr, 1343 load_context_ptr, 1344 in_payload0, 1345 out_payload0, 1346 } => { 1347 collector.reg_use(load_context_ptr); 1348 collector.reg_use(store_context_ptr); 1349 collector.reg_fixed_use(in_payload0, stack_switch::payload_register()); 1350 collector.reg_fixed_def(out_payload0, stack_switch::payload_register()); 1351 1352 let mut clobbers = crate::isa::x64::abi::ALL_CLOBBERS; 1353 // The return/payload reg must not be included in the clobber set 1354 clobbers.remove( 1355 stack_switch::payload_register() 1356 .to_real_reg() 1357 .unwrap() 1358 .into(), 1359 ); 1360 collector.reg_clobbers(clobbers); 1361 } 1362 1363 Inst::ReturnCallKnown { info } => { 1364 let ReturnCallInfo { 1365 dest, uses, tmp, .. 1366 } = &mut **info; 1367 collector.reg_fixed_def(tmp, regs::r11()); 1368 // Same as in the `Inst::CallKnown` branch. 1369 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack)); 1370 for CallArgPair { vreg, preg } in uses { 1371 collector.reg_fixed_use(vreg, *preg); 1372 } 1373 } 1374 1375 Inst::ReturnCallUnknown { info } => { 1376 let ReturnCallInfo { 1377 dest, uses, tmp, .. 1378 } = &mut **info; 1379 1380 // TODO(https://github.com/bytecodealliance/regalloc2/issues/145): 1381 // This shouldn't be a fixed register constraint, but it's not clear how to 1382 // pick a register that won't be clobbered by the callee-save restore code 1383 // emitted with a return_call_indirect. r10 is caller-saved, so this should be 1384 // safe to use. 1385 collector.reg_fixed_use(dest, regs::r10()); 1386 1387 collector.reg_fixed_def(tmp, regs::r11()); 1388 for CallArgPair { vreg, preg } in uses { 1389 collector.reg_fixed_use(vreg, *preg); 1390 } 1391 } 1392 1393 Inst::JmpTableSeq { 1394 idx, tmp1, tmp2, .. 1395 } => { 1396 collector.reg_use(idx); 1397 collector.reg_early_def(tmp1); 1398 // In the sequence emitted for this pseudoinstruction in emit.rs, 1399 // tmp2 is only written after idx is read, so it doesn't need to be 1400 // an early def. 1401 collector.reg_def(tmp2); 1402 } 1403 1404 Inst::JmpUnknown { target } => { 1405 target.get_operands(collector); 1406 } 1407 1408 Inst::LoadExtName { dst, .. } => { 1409 collector.reg_def(dst); 1410 } 1411 1412 Inst::AtomicRmwSeq { 1413 operand, 1414 temp, 1415 dst_old, 1416 mem, 1417 .. 1418 } => { 1419 collector.reg_late_use(operand); 1420 collector.reg_early_def(temp); 1421 // This `fixed_def` is needed because `CMPXCHG` always uses this 1422 // register implicitly. 1423 collector.reg_fixed_def(dst_old, regs::rax()); 1424 mem.get_operands_late(collector) 1425 } 1426 1427 Inst::Atomic128RmwSeq { 1428 operand_low, 1429 operand_high, 1430 temp_low, 1431 temp_high, 1432 dst_old_low, 1433 dst_old_high, 1434 mem, 1435 .. 1436 } => { 1437 // All registers are collected in the `Late` position so that they don't overlap. 1438 collector.reg_late_use(operand_low); 1439 collector.reg_late_use(operand_high); 1440 collector.reg_fixed_def(temp_low, regs::rbx()); 1441 collector.reg_fixed_def(temp_high, regs::rcx()); 1442 collector.reg_fixed_def(dst_old_low, regs::rax()); 1443 collector.reg_fixed_def(dst_old_high, regs::rdx()); 1444 mem.get_operands_late(collector) 1445 } 1446 1447 Inst::Atomic128XchgSeq { 1448 operand_low, 1449 operand_high, 1450 dst_old_low, 1451 dst_old_high, 1452 mem, 1453 .. 1454 } => { 1455 // All registers are collected in the `Late` position so that they don't overlap. 1456 collector.reg_fixed_late_use(operand_low, regs::rbx()); 1457 collector.reg_fixed_late_use(operand_high, regs::rcx()); 1458 collector.reg_fixed_def(dst_old_low, regs::rax()); 1459 collector.reg_fixed_def(dst_old_high, regs::rdx()); 1460 mem.get_operands_late(collector) 1461 } 1462 1463 Inst::Args { args } => { 1464 for ArgPair { vreg, preg } in args { 1465 collector.reg_fixed_def(vreg, *preg); 1466 } 1467 } 1468 1469 Inst::Rets { rets } => { 1470 // The return value(s) are live-out; we represent this 1471 // with register uses on the return instruction. 1472 for RetPair { vreg, preg } in rets { 1473 collector.reg_fixed_use(vreg, *preg); 1474 } 1475 } 1476 1477 Inst::JmpKnown { .. } 1478 | Inst::WinchJmpIf { .. } 1479 | Inst::JmpCond { .. } 1480 | Inst::JmpCondOr { .. } 1481 | Inst::TrapIf { .. } 1482 | Inst::TrapIfAnd { .. } 1483 | Inst::TrapIfOr { .. } => { 1484 // No registers are used. 1485 } 1486 1487 Inst::ElfTlsGetAddr { dst, .. } | Inst::MachOTlsGetAddr { dst, .. } => { 1488 collector.reg_fixed_def(dst, regs::rax()); 1489 // All caller-saves are clobbered. 1490 // 1491 // We use the SysV calling convention here because the 1492 // pseudoinstruction (and relocation that it emits) is specific to 1493 // ELF systems; other x86-64 targets with other conventions (i.e., 1494 // Windows) use different TLS strategies. 1495 let mut clobbers = 1496 X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV, false); 1497 clobbers.remove(regs::gpr_preg(regs::ENC_RAX)); 1498 collector.reg_clobbers(clobbers); 1499 } 1500 1501 Inst::CoffTlsGetAddr { dst, tmp, .. } => { 1502 // We also use the gs register. But that register is not allocatable by the 1503 // register allocator, so we don't need to mark it as used here. 1504 1505 // We use %rax to set the address 1506 collector.reg_fixed_def(dst, regs::rax()); 1507 1508 // We use %rcx as a temporary variable to load the _tls_index 1509 collector.reg_fixed_def(tmp, regs::rcx()); 1510 } 1511 1512 Inst::Unwind { .. } => {} 1513 1514 Inst::DummyUse { reg } => { 1515 collector.reg_use(reg); 1516 } 1517 1518 Inst::External { inst } => { 1519 inst.visit(&mut external::RegallocVisitor { collector }); 1520 } 1521 } 1522 } 1523 1524 //============================================================================= 1525 // Instructions: misc functions and external interface 1526 1527 impl MachInst for Inst { 1528 type ABIMachineSpec = X64ABIMachineSpec; 1529 1530 fn get_operands(&mut self, collector: &mut impl OperandVisitor) { 1531 x64_get_operands(self, collector) 1532 } 1533 1534 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> { 1535 use asm::inst::Inst as I; 1536 match self { 1537 // Note (carefully!) that a 32-bit mov *isn't* a no-op since it zeroes 1538 // out the upper 32 bits of the destination. For example, we could 1539 // conceivably use `movl %reg, %reg` to zero out the top 32 bits of 1540 // %reg. 1541 Self::External { 1542 inst: I::movq_mr(asm::inst::movq_mr { rm64, r64 }), 1543 } => match rm64 { 1544 asm::GprMem::Gpr(reg) => Some((reg.map(|r| r.to_reg()), r64.as_ref().to_reg())), 1545 asm::GprMem::Mem(_) => None, 1546 }, 1547 Self::External { 1548 inst: I::movq_rm(asm::inst::movq_rm { r64, rm64 }), 1549 } => match rm64 { 1550 asm::GprMem::Gpr(reg) => Some((r64.as_ref().map(|r| r.to_reg()), reg.to_reg())), 1551 asm::GprMem::Mem(_) => None, 1552 }, 1553 1554 // Note that `movss_a_r` and `movsd_a_r` are specifically omitted 1555 // here because they only overwrite the low bits in the destination 1556 // register, otherwise preserving the upper bits. That can be used 1557 // for lane-insertion instructions, for example, meaning it's not 1558 // classified as a register move. 1559 // 1560 // Otherwise though all register-to-register movement instructions 1561 // which move 128-bits are registered as moves. 1562 Self::External { 1563 inst: 1564 I::movaps_a(asm::inst::movaps_a { xmm1, xmm_m128 }) 1565 | I::movups_a(asm::inst::movups_a { xmm1, xmm_m128 }) 1566 | I::movapd_a(asm::inst::movapd_a { xmm1, xmm_m128 }) 1567 | I::movupd_a(asm::inst::movupd_a { xmm1, xmm_m128 }) 1568 | I::movdqa_a(asm::inst::movdqa_a { xmm1, xmm_m128 }) 1569 | I::movdqu_a(asm::inst::movdqu_a { xmm1, xmm_m128 }), 1570 } => match xmm_m128 { 1571 asm::XmmMem::Xmm(xmm2) => Some((xmm1.as_ref().map(|r| r.to_reg()), xmm2.to_reg())), 1572 asm::XmmMem::Mem(_) => None, 1573 }, 1574 // In addition to the "A" format of instructions above also 1575 // recognize the "B" format which while it can be used for stores it 1576 // can also be used for register moves. 1577 Self::External { 1578 inst: 1579 I::movaps_b(asm::inst::movaps_b { xmm_m128, xmm1 }) 1580 | I::movups_b(asm::inst::movups_b { xmm_m128, xmm1 }) 1581 | I::movapd_b(asm::inst::movapd_b { xmm_m128, xmm1 }) 1582 | I::movupd_b(asm::inst::movupd_b { xmm_m128, xmm1 }) 1583 | I::movdqa_b(asm::inst::movdqa_b { xmm_m128, xmm1 }) 1584 | I::movdqu_b(asm::inst::movdqu_b { xmm_m128, xmm1 }), 1585 } => match xmm_m128 { 1586 asm::XmmMem::Xmm(dst) => Some((dst.map(|r| r.to_reg()), xmm1.as_ref().to_reg())), 1587 asm::XmmMem::Mem(_) => None, 1588 }, 1589 _ => None, 1590 } 1591 } 1592 1593 fn is_included_in_clobbers(&self) -> bool { 1594 match self { 1595 &Inst::Args { .. } => false, 1596 _ => true, 1597 } 1598 } 1599 1600 fn is_trap(&self) -> bool { 1601 match self { 1602 Self::External { 1603 inst: asm::inst::Inst::ud2_zo(..), 1604 } => true, 1605 _ => false, 1606 } 1607 } 1608 1609 fn is_args(&self) -> bool { 1610 match self { 1611 Self::Args { .. } => true, 1612 _ => false, 1613 } 1614 } 1615 1616 fn is_term(&self) -> MachTerminator { 1617 match self { 1618 // Interesting cases. 1619 &Self::Rets { .. } => MachTerminator::Ret, 1620 &Self::ReturnCallKnown { .. } | &Self::ReturnCallUnknown { .. } => { 1621 MachTerminator::RetCall 1622 } 1623 &Self::JmpKnown { .. } => MachTerminator::Branch, 1624 &Self::JmpCond { .. } => MachTerminator::Branch, 1625 &Self::JmpCondOr { .. } => MachTerminator::Branch, 1626 &Self::JmpTableSeq { .. } => MachTerminator::Branch, 1627 &Self::CallKnown { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, 1628 &Self::CallUnknown { ref info } if info.try_call_info.is_some() => { 1629 MachTerminator::Branch 1630 } 1631 // All other cases are boring. 1632 _ => MachTerminator::None, 1633 } 1634 } 1635 1636 fn is_low_level_branch(&self) -> bool { 1637 match self { 1638 &Self::WinchJmpIf { .. } => true, 1639 _ => false, 1640 } 1641 } 1642 1643 fn is_mem_access(&self) -> bool { 1644 panic!("TODO FILL ME OUT") 1645 } 1646 1647 fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst { 1648 trace!( 1649 "Inst::gen_move {:?} -> {:?} (type: {:?})", 1650 src_reg, 1651 dst_reg.to_reg(), 1652 ty 1653 ); 1654 let rc_dst = dst_reg.to_reg().class(); 1655 let rc_src = src_reg.class(); 1656 // If this isn't true, we have gone way off the rails. 1657 debug_assert!(rc_dst == rc_src); 1658 let inst = match rc_dst { 1659 RegClass::Int => { 1660 asm::inst::movq_mr::new(dst_reg.map(Gpr::unwrap_new), Gpr::unwrap_new(src_reg)) 1661 .into() 1662 } 1663 RegClass::Float => { 1664 // The Intel optimization manual, in "3.5.1.13 Zero-Latency MOV Instructions", 1665 // doesn't include MOVSS/MOVSD as instructions with zero-latency. Use movaps for 1666 // those, which may write more lanes that we need, but are specified to have 1667 // zero-latency. 1668 let dst_reg = dst_reg.map(|r| Xmm::new(r).unwrap()); 1669 let src_reg = Xmm::new(src_reg).unwrap(); 1670 match ty { 1671 types::F16 | types::F32 | types::F64 | types::F32X4 => { 1672 asm::inst::movaps_a::new(dst_reg, src_reg).into() 1673 } 1674 types::F64X2 => asm::inst::movapd_a::new(dst_reg, src_reg).into(), 1675 _ if (ty.is_float() || ty.is_vector()) && ty.bits() <= 128 => { 1676 asm::inst::movdqa_a::new(dst_reg, src_reg).into() 1677 } 1678 _ => unimplemented!("unable to move type: {}", ty), 1679 } 1680 } 1681 RegClass::Vector => unreachable!(), 1682 }; 1683 Inst::External { inst } 1684 } 1685 1686 fn gen_nop(preferred_size: usize) -> Inst { 1687 Inst::nop(std::cmp::min(preferred_size, 9) as u8) 1688 } 1689 1690 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> { 1691 match ty { 1692 types::I8 => Ok((&[RegClass::Int], &[types::I8])), 1693 types::I16 => Ok((&[RegClass::Int], &[types::I16])), 1694 types::I32 => Ok((&[RegClass::Int], &[types::I32])), 1695 types::I64 => Ok((&[RegClass::Int], &[types::I64])), 1696 types::F16 => Ok((&[RegClass::Float], &[types::F16])), 1697 types::F32 => Ok((&[RegClass::Float], &[types::F32])), 1698 types::F64 => Ok((&[RegClass::Float], &[types::F64])), 1699 types::F128 => Ok((&[RegClass::Float], &[types::F128])), 1700 types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])), 1701 _ if ty.is_vector() && ty.bits() <= 128 => { 1702 let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16]; 1703 Ok(( 1704 &[RegClass::Float], 1705 slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]), 1706 )) 1707 } 1708 _ => Err(CodegenError::Unsupported(format!( 1709 "Unexpected SSA-value type: {ty}" 1710 ))), 1711 } 1712 } 1713 1714 fn canonical_type_for_rc(rc: RegClass) -> Type { 1715 match rc { 1716 RegClass::Float => types::I8X16, 1717 RegClass::Int => types::I64, 1718 RegClass::Vector => unreachable!(), 1719 } 1720 } 1721 1722 fn gen_jump(label: MachLabel) -> Inst { 1723 Inst::jmp_known(label) 1724 } 1725 1726 fn gen_imm_u64(value: u64, dst: Writable<Reg>) -> Option<Self> { 1727 Some(Inst::imm(OperandSize::Size64, value, dst)) 1728 } 1729 1730 fn gen_imm_f64(value: f64, tmp: Writable<Reg>, dst: Writable<Reg>) -> SmallVec<[Self; 2]> { 1731 let imm_to_gpr = Inst::imm(OperandSize::Size64, value.to_bits(), tmp); 1732 let gpr_to_xmm = Inst::External { 1733 inst: asm::inst::movq_a::new(dst.map(|r| Xmm::new(r).unwrap()), tmp.to_reg()).into(), 1734 }; 1735 smallvec![imm_to_gpr, gpr_to_xmm] 1736 } 1737 1738 fn gen_dummy_use(reg: Reg) -> Self { 1739 Inst::DummyUse { reg } 1740 } 1741 1742 fn worst_case_size() -> CodeOffset { 1743 15 1744 } 1745 1746 fn ref_type_regclass(_: &settings::Flags) -> RegClass { 1747 RegClass::Int 1748 } 1749 1750 fn is_safepoint(&self) -> bool { 1751 match self { 1752 Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true, 1753 _ => false, 1754 } 1755 } 1756 1757 fn function_alignment() -> FunctionAlignment { 1758 FunctionAlignment { 1759 minimum: 1, 1760 // Change the alignment from 16-bytes to 32-bytes for better performance. 1761 // fix-8573: https://github.com/bytecodealliance/wasmtime/issues/8573 1762 preferred: 32, 1763 } 1764 } 1765 1766 type LabelUse = LabelUse; 1767 1768 const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b]; 1769 } 1770 1771 /// Constant state used during emissions of a sequence of instructions. 1772 pub struct EmitInfo { 1773 pub(super) flags: settings::Flags, 1774 isa_flags: x64_settings::Flags, 1775 } 1776 1777 impl EmitInfo { 1778 /// Create a constant state for emission of instructions. 1779 pub fn new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self { 1780 Self { flags, isa_flags } 1781 } 1782 } 1783 1784 impl MachInstEmit for Inst { 1785 type State = EmitState; 1786 type Info = EmitInfo; 1787 1788 fn emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State) { 1789 emit::emit(self, sink, info, state); 1790 } 1791 1792 fn pretty_print_inst(&self, _: &mut Self::State) -> String { 1793 PrettyPrint::pretty_print(self, 0) 1794 } 1795 } 1796 1797 /// A label-use (internal relocation) in generated code. 1798 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 1799 pub enum LabelUse { 1800 /// A 32-bit offset from location of relocation itself, added to the existing value at that 1801 /// location. Used for control flow instructions which consider an offset from the start of the 1802 /// next instruction (so the size of the payload -- 4 bytes -- is subtracted from the payload). 1803 JmpRel32, 1804 1805 /// A 32-bit offset from location of relocation itself, added to the existing value at that 1806 /// location. 1807 PCRel32, 1808 } 1809 1810 impl MachInstLabelUse for LabelUse { 1811 const ALIGN: CodeOffset = 1; 1812 1813 fn max_pos_range(self) -> CodeOffset { 1814 match self { 1815 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x7fff_ffff, 1816 } 1817 } 1818 1819 fn max_neg_range(self) -> CodeOffset { 1820 match self { 1821 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x8000_0000, 1822 } 1823 } 1824 1825 fn patch_size(self) -> CodeOffset { 1826 match self { 1827 LabelUse::JmpRel32 | LabelUse::PCRel32 => 4, 1828 } 1829 } 1830 1831 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) { 1832 let pc_rel = (label_offset as i64) - (use_offset as i64); 1833 debug_assert!(pc_rel <= self.max_pos_range() as i64); 1834 debug_assert!(pc_rel >= -(self.max_neg_range() as i64)); 1835 let pc_rel = pc_rel as u32; 1836 match self { 1837 LabelUse::JmpRel32 => { 1838 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]); 1839 let value = pc_rel.wrapping_add(addend).wrapping_sub(4); 1840 buffer.copy_from_slice(&value.to_le_bytes()[..]); 1841 } 1842 LabelUse::PCRel32 => { 1843 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]); 1844 let value = pc_rel.wrapping_add(addend); 1845 buffer.copy_from_slice(&value.to_le_bytes()[..]); 1846 } 1847 } 1848 } 1849 1850 fn supports_veneer(self) -> bool { 1851 match self { 1852 LabelUse::JmpRel32 | LabelUse::PCRel32 => false, 1853 } 1854 } 1855 1856 fn veneer_size(self) -> CodeOffset { 1857 match self { 1858 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0, 1859 } 1860 } 1861 1862 fn worst_case_veneer_size() -> CodeOffset { 1863 0 1864 } 1865 1866 fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) { 1867 match self { 1868 LabelUse::JmpRel32 | LabelUse::PCRel32 => { 1869 panic!("Veneer not supported for JumpRel32 label-use."); 1870 } 1871 } 1872 } 1873 1874 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> { 1875 match (reloc, addend) { 1876 (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32), 1877 _ => None, 1878 } 1879 } 1880 } 1881