1 //! This module defines x86_64-specific machine instruction types.
2
3 pub use emit_state::EmitState;
4
5 use crate::binemit::{Addend, CodeOffset, Reloc};
6 use crate::ir::{ExternalName, LibCall, TrapCode, Type, types};
7 use crate::isa::x64::abi::X64ABIMachineSpec;
8 use crate::isa::x64::inst::regs::pretty_print_reg;
9 use crate::isa::x64::settings as x64_settings;
10 use crate::isa::{CallConv, FunctionAlignment};
11 use crate::{CodegenError, CodegenResult, settings};
12 use crate::{machinst::*, trace};
13 use alloc::boxed::Box;
14 use alloc::string::{String, ToString};
15 use alloc::vec;
16 use alloc::vec::Vec;
17 use core::fmt::{self, Write};
18 use core::slice;
19 use cranelift_assembler_x64 as asm;
20 use smallvec::{SmallVec, smallvec};
21
22 pub mod args;
23 mod emit;
24 mod emit_state;
25 #[cfg(test)]
26 mod emit_tests;
27 pub mod external;
28 pub mod regs;
29 mod stack_switch;
30 pub mod unwind;
31
32 use args::*;
33
34 //=============================================================================
35 // Instructions (top level): definition
36
37 // `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here.
38 pub use super::lower::isle::generated_code::AtomicRmwSeqOp;
39 pub use super::lower::isle::generated_code::MInst as Inst;
40
41 /// Out-of-line data for return-calls, to keep the size of `Inst` down.
42 #[derive(Clone, Debug)]
43 pub struct ReturnCallInfo<T> {
44 /// Where this call is going.
45 pub dest: T,
46
47 /// The size of the argument area for this return-call, potentially smaller than that of the
48 /// caller, but never larger.
49 pub new_stack_arg_size: u32,
50
51 /// The in-register arguments and their constraints.
52 pub uses: CallArgList,
53
54 /// A temporary for use when moving the return address.
55 pub tmp: WritableGpr,
56 }
57
58 #[test]
59 #[cfg(target_pointer_width = "64")]
inst_size_test()60 fn inst_size_test() {
61 // This test will help with unintentionally growing the size
62 // of the Inst enum.
63 assert_eq!(48, core::mem::size_of::<Inst>());
64 }
65
66 impl Inst {
67 /// Check if the instruction (or pseudo-instruction) can be emitted given
68 /// the current target architecture given by `emit_info`. For non-assembler
69 /// instructions, this assumes a baseline feature set (i.e., 64-bit AND SSE2
70 /// and below).
is_available(&self, emit_info: &EmitInfo) -> bool71 fn is_available(&self, emit_info: &EmitInfo) -> bool {
72 use asm::AvailableFeatures;
73
74 match self {
75 // These instructions are part of SSE2, which is a basic requirement
76 // in Cranelift, and don't have to be checked.
77 Inst::AtomicRmwSeq { .. }
78 | Inst::CallKnown { .. }
79 | Inst::CallUnknown { .. }
80 | Inst::ReturnCallKnown { .. }
81 | Inst::ReturnCallUnknown { .. }
82 | Inst::CheckedSRemSeq { .. }
83 | Inst::CheckedSRemSeq8 { .. }
84 | Inst::CvtFloatToSintSeq { .. }
85 | Inst::CvtFloatToUintSeq { .. }
86 | Inst::CvtUint64ToFloatSeq { .. }
87 | Inst::JmpCond { .. }
88 | Inst::JmpCondOr { .. }
89 | Inst::WinchJmpIf { .. }
90 | Inst::JmpKnown { .. }
91 | Inst::JmpTableSeq { .. }
92 | Inst::LoadExtName { .. }
93 | Inst::MovFromPReg { .. }
94 | Inst::MovToPReg { .. }
95 | Inst::StackProbeLoop { .. }
96 | Inst::Args { .. }
97 | Inst::Rets { .. }
98 | Inst::StackSwitchBasic { .. }
99 | Inst::TrapIf { .. }
100 | Inst::TrapIfAnd { .. }
101 | Inst::TrapIfOr { .. }
102 | Inst::XmmCmove { .. }
103 | Inst::XmmMinMaxSeq { .. }
104 | Inst::XmmUninitializedValue { .. }
105 | Inst::GprUninitializedValue { .. }
106 | Inst::ElfTlsGetAddr { .. }
107 | Inst::MachOTlsGetAddr { .. }
108 | Inst::CoffTlsGetAddr { .. }
109 | Inst::Unwind { .. }
110 | Inst::DummyUse { .. }
111 | Inst::LabelAddress { .. }
112 | Inst::SequencePoint => true,
113
114 Inst::Atomic128RmwSeq { .. } | Inst::Atomic128XchgSeq { .. } => emit_info.cmpxchg16b(),
115
116 Inst::External { inst } => inst.is_available(&emit_info),
117 }
118 }
119 }
120
121 // Handy constructors for Insts.
122
123 impl Inst {
nop(len: u8) -> Self124 pub(crate) fn nop(len: u8) -> Self {
125 assert!(len > 0 && len <= 9);
126 let inst = match len {
127 1 => asm::inst::nop_1b::new().into(),
128 2 => asm::inst::nop_2b::new().into(),
129 3 => asm::inst::nop_3b::new().into(),
130 4 => asm::inst::nop_4b::new().into(),
131 5 => asm::inst::nop_5b::new().into(),
132 6 => asm::inst::nop_6b::new().into(),
133 7 => asm::inst::nop_7b::new().into(),
134 8 => asm::inst::nop_8b::new().into(),
135 9 => asm::inst::nop_9b::new().into(),
136 _ => unreachable!("nop length must be between 1 and 9"),
137 };
138 Self::External { inst }
139 }
140
addq_mi(dst: Writable<Reg>, simm32: i32) -> Self141 pub(crate) fn addq_mi(dst: Writable<Reg>, simm32: i32) -> Self {
142 let inst = if let Ok(simm8) = i8::try_from(simm32) {
143 asm::inst::addq_mi_sxb::new(dst, simm8).into()
144 } else {
145 asm::inst::addq_mi_sxl::new(dst, simm32).into()
146 };
147 Inst::External { inst }
148 }
149
subq_mi(dst: Writable<Reg>, simm32: i32) -> Self150 pub(crate) fn subq_mi(dst: Writable<Reg>, simm32: i32) -> Self {
151 let inst = if let Ok(simm8) = i8::try_from(simm32) {
152 asm::inst::subq_mi_sxb::new(dst, simm8).into()
153 } else {
154 asm::inst::subq_mi_sxl::new(dst, simm32).into()
155 };
156 Inst::External { inst }
157 }
158
159 /// Writes the `simm64` immediate into `dst`.
160 ///
161 /// Note that if `dst_size` is less than 64-bits then the upper bits of
162 /// `simm64` will be converted to zero.
imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst163 pub fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
164 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
165 debug_assert!(dst.to_reg().class() == RegClass::Int);
166 let dst = WritableGpr::from_writable_reg(dst).unwrap();
167 let inst = match dst_size {
168 OperandSize::Size64 => match u32::try_from(simm64) {
169 // If `simm64` is zero-extended use `movl` which zeros the
170 // upper bits.
171 Ok(imm32) => asm::inst::movl_oi::new(dst, imm32).into(),
172 _ => match i32::try_from(simm64.cast_signed()) {
173 // If `simm64` is sign-extended use `movq` which sign the
174 // upper bits.
175 Ok(simm32) => asm::inst::movq_mi_sxl::new(dst, simm32).into(),
176 // fall back to embedding the entire immediate.
177 _ => asm::inst::movabsq_oi::new(dst, simm64).into(),
178 },
179 },
180 // FIXME: the input to this function is a logical `simm64` stored
181 // as `u64`. That means that ideally what we would do here is cast
182 // the `simm64` to an `i64`, perform a `i32::try_from()`, then cast
183 // that back to `u32`. That would ensure that the immediate loses
184 // no meaning and has the same logical value. Currently though
185 // Cranelift relies on discarding the upper bits because literals
186 // like `0x8000_0000_u64` fail to convert to an `i32`. In theory
187 // the input to this function should change to `i64`. In the
188 // meantime this is documented as discarding the upper bits,
189 // although this is an old function so that's unlikely to help
190 // much.
191 _ => asm::inst::movl_oi::new(dst, simm64 as u32).into(),
192 };
193 Inst::External { inst }
194 }
195
movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst196 pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
197 src.assert_regclass_is(RegClass::Int);
198 debug_assert!(dst.to_reg().class() == RegClass::Int);
199 let src = match src {
200 RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()),
201 RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
202 };
203 let inst = match ext_mode {
204 ExtMode::BL => asm::inst::movzbl_rm::new(dst, src).into(),
205 ExtMode::BQ => asm::inst::movzbq_rm::new(dst, src).into(),
206 ExtMode::WL => asm::inst::movzwl_rm::new(dst, src).into(),
207 ExtMode::WQ => asm::inst::movzwq_rm::new(dst, src).into(),
208 ExtMode::LQ => {
209 // This instruction selection may seem strange but is correct in
210 // 64-bit mode: section 3.4.1.1 of the Intel manual says that
211 // "32-bit operands generate a 32-bit result, zero-extended to a
212 // 64-bit result in the destination general-purpose register."
213 // This is applicable beyond `mov` but we use this fact to
214 // zero-extend `src` into `dst`.
215 asm::inst::movl_rm::new(dst, src).into()
216 }
217 };
218 Inst::External { inst }
219 }
220
movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst221 pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
222 src.assert_regclass_is(RegClass::Int);
223 debug_assert!(dst.to_reg().class() == RegClass::Int);
224 let src = match src {
225 RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()),
226 RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
227 };
228 let inst = match ext_mode {
229 ExtMode::BL => asm::inst::movsbl_rm::new(dst, src).into(),
230 ExtMode::BQ => asm::inst::movsbq_rm::new(dst, src).into(),
231 ExtMode::WL => asm::inst::movswl_rm::new(dst, src).into(),
232 ExtMode::WQ => asm::inst::movswq_rm::new(dst, src).into(),
233 ExtMode::LQ => asm::inst::movslq_rm::new(dst, src).into(),
234 };
235 Inst::External { inst }
236 }
237
238 /// Compares `src1` against `src2`
cmp_mi_sxb(size: OperandSize, src1: Gpr, src2: i8) -> Inst239 pub(crate) fn cmp_mi_sxb(size: OperandSize, src1: Gpr, src2: i8) -> Inst {
240 let inst = match size {
241 OperandSize::Size8 => asm::inst::cmpb_mi::new(src1, src2.cast_unsigned()).into(),
242 OperandSize::Size16 => asm::inst::cmpw_mi_sxb::new(src1, src2).into(),
243 OperandSize::Size32 => asm::inst::cmpl_mi_sxb::new(src1, src2).into(),
244 OperandSize::Size64 => asm::inst::cmpq_mi_sxb::new(src1, src2).into(),
245 };
246 Inst::External { inst }
247 }
248
trap_if(cc: CC, trap_code: TrapCode) -> Inst249 pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
250 Inst::TrapIf { cc, trap_code }
251 }
252
call_known(info: Box<CallInfo<ExternalName>>) -> Inst253 pub(crate) fn call_known(info: Box<CallInfo<ExternalName>>) -> Inst {
254 Inst::CallKnown { info }
255 }
256
call_unknown(info: Box<CallInfo<RegMem>>) -> Inst257 pub(crate) fn call_unknown(info: Box<CallInfo<RegMem>>) -> Inst {
258 info.dest.assert_regclass_is(RegClass::Int);
259 Inst::CallUnknown { info }
260 }
261
jmp_known(dst: MachLabel) -> Inst262 pub(crate) fn jmp_known(dst: MachLabel) -> Inst {
263 Inst::JmpKnown { dst }
264 }
265
266 /// Choose which instruction to use for loading a register value from memory. For loads smaller
267 /// than 64 bits, this method expects a way to extend the value (i.e. [ExtKind::SignExtend],
268 /// [ExtKind::ZeroExtend]); loads with no extension necessary will ignore this.
load( ty: Type, from_addr: impl Into<SyntheticAmode>, to_reg: Writable<Reg>, ext_kind: ExtKind, ) -> Inst269 pub(crate) fn load(
270 ty: Type,
271 from_addr: impl Into<SyntheticAmode>,
272 to_reg: Writable<Reg>,
273 ext_kind: ExtKind,
274 ) -> Inst {
275 let rc = to_reg.to_reg().class();
276 match rc {
277 RegClass::Int => {
278 let ext_mode = match ty.bytes() {
279 1 => Some(ExtMode::BQ),
280 2 => Some(ExtMode::WQ),
281 4 => Some(ExtMode::LQ),
282 8 => None,
283 _ => unreachable!("the type should never use a scalar load: {}", ty),
284 };
285 if let Some(ext_mode) = ext_mode {
286 // Values smaller than 64 bits must be extended in some way.
287 match ext_kind {
288 ExtKind::SignExtend => {
289 Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
290 }
291 ExtKind::ZeroExtend => {
292 Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
293 }
294 ExtKind::None => {
295 panic!("expected an extension kind for extension mode: {ext_mode:?}")
296 }
297 }
298 } else {
299 // 64-bit values can be moved directly.
300 let from_addr = asm::GprMem::from(from_addr.into());
301 Inst::External {
302 inst: asm::inst::movq_rm::new(to_reg, from_addr).into(),
303 }
304 }
305 }
306 RegClass::Float => {
307 let to_reg = to_reg.map(|r| Xmm::new(r).unwrap());
308 let from_addr = from_addr.into();
309 let inst = match ty {
310 types::F16 | types::I8X2 => {
311 panic!("loading a f16 or i8x2 requires multiple instructions")
312 }
313 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => {
314 asm::inst::movss_a_m::new(to_reg, from_addr).into()
315 }
316 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => {
317 asm::inst::movsd_a_m::new(to_reg, from_addr).into()
318 }
319 types::F32X4 => asm::inst::movups_a::new(to_reg, from_addr).into(),
320 types::F64X2 => asm::inst::movupd_a::new(to_reg, from_addr).into(),
321 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => {
322 asm::inst::movdqu_a::new(to_reg, from_addr).into()
323 }
324 _ => unimplemented!("unable to load type: {}", ty),
325 };
326 Inst::External { inst }
327 }
328 RegClass::Vector => unreachable!(),
329 }
330 }
331
332 /// Choose which instruction to use for storing a register value to memory.
store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst333 pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
334 let rc = from_reg.class();
335 let to_addr = to_addr.into();
336 let inst = match rc {
337 RegClass::Int => {
338 let from_reg = Gpr::unwrap_new(from_reg);
339 match ty {
340 types::I8 => asm::inst::movb_mr::new(to_addr, from_reg).into(),
341 types::I16 => asm::inst::movw_mr::new(to_addr, from_reg).into(),
342 types::I32 => asm::inst::movl_mr::new(to_addr, from_reg).into(),
343 types::I64 => asm::inst::movq_mr::new(to_addr, from_reg).into(),
344 _ => unreachable!(),
345 }
346 }
347 RegClass::Float => {
348 let from_reg = Xmm::new(from_reg).unwrap();
349 match ty {
350 types::F16 | types::I8X2 => {
351 panic!("storing a f16 or i8x2 requires multiple instructions")
352 }
353 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => {
354 asm::inst::movss_c_m::new(to_addr, from_reg).into()
355 }
356 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => {
357 asm::inst::movsd_c_m::new(to_addr, from_reg).into()
358 }
359 types::F32X4 => asm::inst::movups_b::new(to_addr, from_reg).into(),
360 types::F64X2 => asm::inst::movupd_b::new(to_addr, from_reg).into(),
361 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => {
362 asm::inst::movdqu_b::new(to_addr, from_reg).into()
363 }
364 _ => unimplemented!("unable to store type: {}", ty),
365 }
366 }
367 RegClass::Vector => unreachable!(),
368 };
369 Inst::External { inst }
370 }
371 }
372
373 //=============================================================================
374 // Instructions: printing
375
376 impl PrettyPrint for Inst {
pretty_print(&self, _size: u8) -> String377 fn pretty_print(&self, _size: u8) -> String {
378 fn ljustify(s: String) -> String {
379 let w = 7;
380 if s.len() >= w {
381 s
382 } else {
383 let need = usize::min(w, w - s.len());
384 s + &format!("{nil: <width$}", nil = "", width = need)
385 }
386 }
387
388 fn ljustify2(s1: String, s2: String) -> String {
389 ljustify(s1 + &s2)
390 }
391
392 match self {
393 Inst::CheckedSRemSeq {
394 size,
395 divisor,
396 dividend_lo,
397 dividend_hi,
398 dst_quotient,
399 dst_remainder,
400 } => {
401 let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes());
402 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes());
403 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes());
404 let dst_quotient =
405 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes());
406 let dst_remainder =
407 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes());
408 format!(
409 "checked_srem_seq {dividend_lo}, {dividend_hi}, \
410 {divisor}, {dst_quotient}, {dst_remainder}",
411 )
412 }
413
414 Inst::CheckedSRemSeq8 {
415 divisor,
416 dividend,
417 dst,
418 } => {
419 let divisor = pretty_print_reg(divisor.to_reg(), 1);
420 let dividend = pretty_print_reg(dividend.to_reg(), 1);
421 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
422 format!("checked_srem_seq {dividend}, {divisor}, {dst}")
423 }
424
425 Inst::XmmMinMaxSeq {
426 lhs,
427 rhs,
428 dst,
429 is_min,
430 size,
431 } => {
432 let rhs = pretty_print_reg(rhs.to_reg(), 8);
433 let lhs = pretty_print_reg(lhs.to_reg(), 8);
434 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
435 let op = ljustify2(
436 if *is_min {
437 "xmm min seq ".to_string()
438 } else {
439 "xmm max seq ".to_string()
440 },
441 format!("f{}", size.to_bits()),
442 );
443 format!("{op} {lhs}, {rhs}, {dst}")
444 }
445
446 Inst::XmmUninitializedValue { dst } => {
447 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
448 let op = ljustify("uninit".into());
449 format!("{op} {dst}")
450 }
451
452 Inst::GprUninitializedValue { dst } => {
453 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
454 let op = ljustify("uninit".into());
455 format!("{op} {dst}")
456 }
457
458 Inst::CvtUint64ToFloatSeq {
459 src,
460 dst,
461 dst_size,
462 tmp_gpr1,
463 tmp_gpr2,
464 ..
465 } => {
466 let src = pretty_print_reg(src.to_reg(), 8);
467 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
468 let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8);
469 let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8);
470 let op = ljustify(format!(
471 "u64_to_{}_seq",
472 if *dst_size == OperandSize::Size64 {
473 "f64"
474 } else {
475 "f32"
476 }
477 ));
478 format!("{op} {src}, {dst}, {tmp_gpr1}, {tmp_gpr2}")
479 }
480
481 Inst::CvtFloatToSintSeq {
482 src,
483 dst,
484 src_size,
485 dst_size,
486 tmp_xmm,
487 tmp_gpr,
488 is_saturating,
489 } => {
490 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
491 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
492 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
493 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
494 let op = ljustify(format!(
495 "cvt_float{}_to_sint{}{}_seq",
496 src_size.to_bits(),
497 dst_size.to_bits(),
498 if *is_saturating { "_sat" } else { "" },
499 ));
500 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}")
501 }
502
503 Inst::CvtFloatToUintSeq {
504 src,
505 dst,
506 src_size,
507 dst_size,
508 tmp_gpr,
509 tmp_xmm,
510 tmp_xmm2,
511 is_saturating,
512 } => {
513 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
514 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
515 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
516 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
517 let tmp_xmm2 = pretty_print_reg(tmp_xmm2.to_reg().to_reg(), 8);
518 let op = ljustify(format!(
519 "cvt_float{}_to_uint{}{}_seq",
520 src_size.to_bits(),
521 dst_size.to_bits(),
522 if *is_saturating { "_sat" } else { "" },
523 ));
524 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}, {tmp_xmm2}")
525 }
526
527 Inst::MovFromPReg { src, dst } => {
528 let src: Reg = (*src).into();
529 let src = pretty_print_reg(src, 8);
530 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
531 let op = ljustify("movq".to_string());
532 format!("{op} {src}, {dst}")
533 }
534
535 Inst::MovToPReg { src, dst } => {
536 let src = pretty_print_reg(src.to_reg(), 8);
537 let dst: Reg = (*dst).into();
538 let dst = pretty_print_reg(dst, 8);
539 let op = ljustify("movq".to_string());
540 format!("{op} {src}, {dst}")
541 }
542
543 Inst::XmmCmove {
544 ty,
545 cc,
546 consequent,
547 alternative,
548 dst,
549 ..
550 } => {
551 let size = u8::try_from(ty.bytes()).unwrap();
552 let alternative = pretty_print_reg(alternative.to_reg(), size);
553 let dst = pretty_print_reg(dst.to_reg().to_reg(), size);
554 let consequent = pretty_print_reg(consequent.to_reg(), size);
555 let suffix = match *ty {
556 types::F64 => "sd",
557 types::F32 => "ss",
558 types::F16 => "ss",
559 types::F32X4 => "aps",
560 types::F64X2 => "apd",
561 _ => "dqa",
562 };
563 let cc = cc.invert();
564 format!(
565 "mov{suffix} {alternative}, {dst}; \
566 j{cc} $next; \
567 mov{suffix} {consequent}, {dst}; \
568 $next:"
569 )
570 }
571
572 Inst::StackProbeLoop {
573 tmp,
574 frame_size,
575 guard_size,
576 } => {
577 let tmp = pretty_print_reg(tmp.to_reg(), 8);
578 let op = ljustify("stack_probe_loop".to_string());
579 format!("{op} {tmp}, frame_size={frame_size}, guard_size={guard_size}")
580 }
581
582 Inst::CallKnown { info } => {
583 let op = ljustify("call".to_string());
584 let try_call = info
585 .try_call_info
586 .as_ref()
587 .map(|tci| pretty_print_try_call(tci))
588 .unwrap_or_default();
589 format!("{op} {:?}{try_call}", info.dest)
590 }
591
592 Inst::CallUnknown { info } => {
593 let dest = info.dest.pretty_print(8);
594 let op = ljustify("call".to_string());
595 let try_call = info
596 .try_call_info
597 .as_ref()
598 .map(|tci| pretty_print_try_call(tci))
599 .unwrap_or_default();
600 format!("{op} *{dest}{try_call}")
601 }
602
603 Inst::ReturnCallKnown { info } => {
604 let ReturnCallInfo {
605 uses,
606 new_stack_arg_size,
607 tmp,
608 dest,
609 } = &**info;
610 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
611 let mut s = format!("return_call_known {dest:?} ({new_stack_arg_size}) tmp={tmp}");
612 for ret in uses {
613 let preg = pretty_print_reg(ret.preg, 8);
614 let vreg = pretty_print_reg(ret.vreg, 8);
615 write!(&mut s, " {vreg}={preg}").unwrap();
616 }
617 s
618 }
619
620 Inst::ReturnCallUnknown { info } => {
621 let ReturnCallInfo {
622 uses,
623 new_stack_arg_size,
624 tmp,
625 dest,
626 } = &**info;
627 let callee = pretty_print_reg(*dest, 8);
628 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
629 let mut s =
630 format!("return_call_unknown {callee} ({new_stack_arg_size}) tmp={tmp}");
631 for ret in uses {
632 let preg = pretty_print_reg(ret.preg, 8);
633 let vreg = pretty_print_reg(ret.vreg, 8);
634 write!(&mut s, " {vreg}={preg}").unwrap();
635 }
636 s
637 }
638
639 Inst::Args { args } => {
640 let mut s = "args".to_string();
641 for arg in args {
642 let preg = pretty_print_reg(arg.preg, 8);
643 let def = pretty_print_reg(arg.vreg.to_reg(), 8);
644 write!(&mut s, " {def}={preg}").unwrap();
645 }
646 s
647 }
648
649 Inst::Rets { rets } => {
650 let mut s = "rets".to_string();
651 for ret in rets {
652 let preg = pretty_print_reg(ret.preg, 8);
653 let vreg = pretty_print_reg(ret.vreg, 8);
654 write!(&mut s, " {vreg}={preg}").unwrap();
655 }
656 s
657 }
658
659 Inst::StackSwitchBasic {
660 store_context_ptr,
661 load_context_ptr,
662 in_payload0,
663 out_payload0,
664 } => {
665 let store_context_ptr = pretty_print_reg(**store_context_ptr, 8);
666 let load_context_ptr = pretty_print_reg(**load_context_ptr, 8);
667 let in_payload0 = pretty_print_reg(**in_payload0, 8);
668 let out_payload0 = pretty_print_reg(*out_payload0.to_reg(), 8);
669 format!(
670 "{out_payload0} = stack_switch_basic {store_context_ptr}, {load_context_ptr}, {in_payload0}"
671 )
672 }
673
674 Inst::JmpKnown { dst } => {
675 let op = ljustify("jmp".to_string());
676 let dst = dst.to_string();
677 format!("{op} {dst}")
678 }
679
680 Inst::WinchJmpIf { cc, taken } => {
681 let taken = taken.to_string();
682 let op = ljustify2("j".to_string(), cc.to_string());
683 format!("{op} {taken}")
684 }
685
686 Inst::JmpCondOr {
687 cc1,
688 cc2,
689 taken,
690 not_taken,
691 } => {
692 let taken = taken.to_string();
693 let not_taken = not_taken.to_string();
694 let op = ljustify(format!("j{cc1},{cc2}"));
695 format!("{op} {taken}; j {not_taken}")
696 }
697
698 Inst::JmpCond {
699 cc,
700 taken,
701 not_taken,
702 } => {
703 let taken = taken.to_string();
704 let not_taken = not_taken.to_string();
705 let op = ljustify2("j".to_string(), cc.to_string());
706 format!("{op} {taken}; j {not_taken}")
707 }
708
709 Inst::JmpTableSeq {
710 idx, tmp1, tmp2, ..
711 } => {
712 let idx = pretty_print_reg(*idx, 8);
713 let tmp1 = pretty_print_reg(tmp1.to_reg(), 8);
714 let tmp2 = pretty_print_reg(tmp2.to_reg(), 8);
715 let op = ljustify("br_table".into());
716 format!("{op} {idx}, {tmp1}, {tmp2}")
717 }
718
719 Inst::TrapIf { cc, trap_code, .. } => {
720 format!("j{cc} #trap={trap_code}")
721 }
722
723 Inst::TrapIfAnd {
724 cc1,
725 cc2,
726 trap_code,
727 ..
728 } => {
729 let cc1 = cc1.invert();
730 let cc2 = cc2.invert();
731 format!("trap_if_and {cc1}, {cc2}, {trap_code}")
732 }
733
734 Inst::TrapIfOr {
735 cc1,
736 cc2,
737 trap_code,
738 ..
739 } => {
740 let cc2 = cc2.invert();
741 format!("trap_if_or {cc1}, {cc2}, {trap_code}")
742 }
743
744 Inst::LoadExtName {
745 dst, name, offset, ..
746 } => {
747 let dst = pretty_print_reg(*dst.to_reg(), 8);
748 let name = name.display(None);
749 let op = ljustify("load_ext_name".into());
750 format!("{op} {name}+{offset}, {dst}")
751 }
752
753 Inst::AtomicRmwSeq { ty, op, .. } => {
754 let ty = ty.bits();
755 format!(
756 "atomically {{ {ty}_bits_at_[%r9] {op:?}= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }}"
757 )
758 }
759
760 Inst::Atomic128RmwSeq { args } => {
761 let Atomic128RmwSeqArgs {
762 op,
763 mem_low,
764 mem_high,
765 operand_low,
766 operand_high,
767 temp_low,
768 temp_high,
769 dst_old_low,
770 dst_old_high,
771 } = &**args;
772 let operand_low = pretty_print_reg(**operand_low, 8);
773 let operand_high = pretty_print_reg(**operand_high, 8);
774 let temp_low = pretty_print_reg(*temp_low.to_reg(), 8);
775 let temp_high = pretty_print_reg(*temp_high.to_reg(), 8);
776 let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);
777 let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);
778 let mem_low = mem_low.pretty_print(16);
779 let mem_high = mem_high.pretty_print(16);
780 format!(
781 "atomically {{ {dst_old_high}:{dst_old_low} = {mem_low}:{mem_high}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem_low}:{mem_high} = {temp_high}:{temp_low} }}"
782 )
783 }
784
785 Inst::Atomic128XchgSeq { args } => {
786 let Atomic128XchgSeqArgs {
787 mem_low,
788 mem_high,
789 operand_low,
790 operand_high,
791 dst_old_low,
792 dst_old_high,
793 } = &**args;
794 let operand_low = pretty_print_reg(**operand_low, 8);
795 let operand_high = pretty_print_reg(**operand_high, 8);
796 let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);
797 let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);
798 let mem_low = mem_low.pretty_print(16);
799 let mem_high = mem_high.pretty_print(16);
800 format!(
801 "atomically {{ {dst_old_high}:{dst_old_low} = {mem_low}:{mem_high}; {mem_low}:{mem_high} = {operand_high}:{operand_low} }}"
802 )
803 }
804
805 Inst::ElfTlsGetAddr { symbol, dst } => {
806 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
807 format!("{dst} = elf_tls_get_addr {symbol:?}")
808 }
809
810 Inst::MachOTlsGetAddr { symbol, dst } => {
811 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
812 format!("{dst} = macho_tls_get_addr {symbol:?}")
813 }
814
815 Inst::CoffTlsGetAddr { symbol, dst, tmp } => {
816 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
817 let tmp = tmp.to_reg().to_reg();
818
819 let mut s = format!("{dst} = coff_tls_get_addr {symbol:?}");
820 if tmp.is_virtual() {
821 let tmp = pretty_print_reg(tmp, 8);
822 write!(&mut s, ", {tmp}").unwrap();
823 };
824
825 s
826 }
827
828 Inst::Unwind { inst } => format!("unwind {inst:?}"),
829
830 Inst::DummyUse { reg } => {
831 let reg = pretty_print_reg(*reg, 8);
832 format!("dummy_use {reg}")
833 }
834
835 Inst::LabelAddress { dst, label } => {
836 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
837 format!("label_address {dst}, {label:?}")
838 }
839
840 Inst::SequencePoint {} => {
841 format!("sequence_point")
842 }
843
844 Inst::External { inst } => {
845 format!("{inst}")
846 }
847 }
848 }
849 }
850
pretty_print_try_call(info: &TryCallInfo) -> String851 fn pretty_print_try_call(info: &TryCallInfo) -> String {
852 format!(
853 "; jmp {:?}; catch [{}]",
854 info.continuation,
855 info.pretty_print_dests()
856 )
857 }
858
859 impl fmt::Debug for Inst {
fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result860 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
861 write!(fmt, "{}", self.pretty_print_inst(&mut Default::default()))
862 }
863 }
864
x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor)865 fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
866 // Note: because we need to statically know the indices of each
867 // reg in the operands list in order to fetch its allocation
868 // later, we put the variable-operand-count bits (the RegMem,
869 // RegMemImm, etc args) last. regalloc2 doesn't care what order
870 // the operands come in; they can be freely reordered.
871
872 // N.B.: we MUST keep the below in careful sync with (i) emission,
873 // in `emit.rs`, and (ii) pretty-printing, in the `pretty_print`
874 // method above.
875 match inst {
876 Inst::CheckedSRemSeq {
877 divisor,
878 dividend_lo,
879 dividend_hi,
880 dst_quotient,
881 dst_remainder,
882 ..
883 } => {
884 collector.reg_use(divisor);
885 collector.reg_fixed_use(dividend_lo, regs::rax());
886 collector.reg_fixed_use(dividend_hi, regs::rdx());
887 collector.reg_fixed_def(dst_quotient, regs::rax());
888 collector.reg_fixed_def(dst_remainder, regs::rdx());
889 }
890 Inst::CheckedSRemSeq8 {
891 divisor,
892 dividend,
893 dst,
894 ..
895 } => {
896 collector.reg_use(divisor);
897 collector.reg_fixed_use(dividend, regs::rax());
898 collector.reg_fixed_def(dst, regs::rax());
899 }
900 Inst::XmmUninitializedValue { dst } => collector.reg_def(dst),
901 Inst::GprUninitializedValue { dst } => collector.reg_def(dst),
902 Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => {
903 collector.reg_use(rhs);
904 collector.reg_use(lhs);
905 collector.reg_reuse_def(dst, 0); // Reuse RHS.
906 }
907 Inst::MovFromPReg { dst, src } => {
908 debug_assert!(dst.to_reg().to_reg().is_virtual());
909 collector.reg_fixed_nonallocatable(*src);
910 collector.reg_def(dst);
911 }
912 Inst::MovToPReg { dst, src } => {
913 debug_assert!(src.to_reg().is_virtual());
914 collector.reg_use(src);
915 collector.reg_fixed_nonallocatable(*dst);
916 }
917 Inst::CvtUint64ToFloatSeq {
918 src,
919 dst,
920 tmp_gpr1,
921 tmp_gpr2,
922 ..
923 } => {
924 collector.reg_use(src);
925 collector.reg_early_def(dst);
926 collector.reg_early_def(tmp_gpr1);
927 collector.reg_early_def(tmp_gpr2);
928 }
929 Inst::CvtFloatToSintSeq {
930 src,
931 dst,
932 tmp_xmm,
933 tmp_gpr,
934 ..
935 } => {
936 collector.reg_use(src);
937 collector.reg_early_def(dst);
938 collector.reg_early_def(tmp_gpr);
939 collector.reg_early_def(tmp_xmm);
940 }
941 Inst::CvtFloatToUintSeq {
942 src,
943 dst,
944 tmp_gpr,
945 tmp_xmm,
946 tmp_xmm2,
947 ..
948 } => {
949 collector.reg_use(src);
950 collector.reg_early_def(dst);
951 collector.reg_early_def(tmp_gpr);
952 collector.reg_early_def(tmp_xmm);
953 collector.reg_early_def(tmp_xmm2);
954 }
955
956 Inst::XmmCmove {
957 consequent,
958 alternative,
959 dst,
960 ..
961 } => {
962 collector.reg_use(alternative);
963 collector.reg_reuse_def(dst, 0);
964 collector.reg_use(consequent);
965 }
966 Inst::StackProbeLoop { tmp, .. } => {
967 collector.reg_early_def(tmp);
968 }
969
970 Inst::CallKnown { info } => {
971 // Probestack is special and is only inserted after
972 // regalloc, so we do not need to represent its ABI to the
973 // register allocator. Assert that we don't alter that
974 // arrangement.
975 let CallInfo {
976 uses,
977 defs,
978 clobbers,
979 dest,
980 try_call_info,
981 ..
982 } = &mut **info;
983 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
984 for CallArgPair { vreg, preg } in uses {
985 collector.reg_fixed_use(vreg, *preg);
986 }
987 for CallRetPair { vreg, location } in defs {
988 match location {
989 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
990 RetLocation::Stack(..) => collector.any_def(vreg),
991 }
992 }
993 collector.reg_clobbers(*clobbers);
994 if let Some(try_call_info) = try_call_info {
995 try_call_info.collect_operands(collector);
996 }
997 }
998
999 Inst::CallUnknown { info } => {
1000 let CallInfo {
1001 uses,
1002 defs,
1003 clobbers,
1004 callee_conv,
1005 dest,
1006 try_call_info,
1007 ..
1008 } = &mut **info;
1009 match dest {
1010 RegMem::Reg { reg } if *callee_conv == CallConv::Winch => {
1011 // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
1012 // This shouldn't be a fixed register constraint. r10 is caller-saved, so this
1013 // should be safe to use.
1014 collector.reg_fixed_use(reg, regs::r10());
1015 }
1016 _ => dest.get_operands(collector),
1017 }
1018 for CallArgPair { vreg, preg } in uses {
1019 collector.reg_fixed_use(vreg, *preg);
1020 }
1021 for CallRetPair { vreg, location } in defs {
1022 match location {
1023 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
1024 RetLocation::Stack(..) => collector.any_def(vreg),
1025 }
1026 }
1027 collector.reg_clobbers(*clobbers);
1028 if let Some(try_call_info) = try_call_info {
1029 try_call_info.collect_operands(collector);
1030 }
1031 }
1032 Inst::StackSwitchBasic {
1033 store_context_ptr,
1034 load_context_ptr,
1035 in_payload0,
1036 out_payload0,
1037 } => {
1038 collector.reg_use(load_context_ptr);
1039 collector.reg_use(store_context_ptr);
1040 collector.reg_fixed_use(in_payload0, stack_switch::payload_register());
1041 collector.reg_fixed_def(out_payload0, stack_switch::payload_register());
1042
1043 let mut clobbers = crate::isa::x64::abi::ALL_CLOBBERS;
1044 // The return/payload reg must not be included in the clobber set
1045 clobbers.remove(
1046 stack_switch::payload_register()
1047 .to_real_reg()
1048 .unwrap()
1049 .into(),
1050 );
1051 collector.reg_clobbers(clobbers);
1052 }
1053
1054 Inst::ReturnCallKnown { info } => {
1055 let ReturnCallInfo {
1056 dest, uses, tmp, ..
1057 } = &mut **info;
1058 collector.reg_fixed_def(tmp, regs::r11());
1059 // Same as in the `Inst::CallKnown` branch.
1060 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
1061 for CallArgPair { vreg, preg } in uses {
1062 collector.reg_fixed_use(vreg, *preg);
1063 }
1064 }
1065
1066 Inst::ReturnCallUnknown { info } => {
1067 let ReturnCallInfo {
1068 dest, uses, tmp, ..
1069 } = &mut **info;
1070
1071 // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
1072 // This shouldn't be a fixed register constraint, but it's not clear how to
1073 // pick a register that won't be clobbered by the callee-save restore code
1074 // emitted with a return_call_indirect. r10 is caller-saved, so this should be
1075 // safe to use.
1076 collector.reg_fixed_use(dest, regs::r10());
1077
1078 collector.reg_fixed_def(tmp, regs::r11());
1079 for CallArgPair { vreg, preg } in uses {
1080 collector.reg_fixed_use(vreg, *preg);
1081 }
1082 }
1083
1084 Inst::JmpTableSeq {
1085 idx, tmp1, tmp2, ..
1086 } => {
1087 collector.reg_use(idx);
1088 collector.reg_early_def(tmp1);
1089 // In the sequence emitted for this pseudoinstruction in emit.rs,
1090 // tmp2 is only written after idx is read, so it doesn't need to be
1091 // an early def.
1092 collector.reg_def(tmp2);
1093 }
1094
1095 Inst::LoadExtName { dst, .. } => {
1096 collector.reg_def(dst);
1097 }
1098
1099 Inst::AtomicRmwSeq {
1100 operand,
1101 temp,
1102 dst_old,
1103 mem,
1104 ..
1105 } => {
1106 collector.reg_late_use(operand);
1107 collector.reg_early_def(temp);
1108 // This `fixed_def` is needed because `CMPXCHG` always uses this
1109 // register implicitly.
1110 collector.reg_fixed_def(dst_old, regs::rax());
1111 mem.get_operands_late(collector)
1112 }
1113
1114 Inst::Atomic128RmwSeq { args } => {
1115 let Atomic128RmwSeqArgs {
1116 mem_low,
1117 mem_high,
1118 operand_low,
1119 operand_high,
1120 temp_low,
1121 temp_high,
1122 dst_old_low,
1123 dst_old_high,
1124 op: _,
1125 } = &mut **args;
1126 // All registers are collected in the `Late` position so that they don't overlap.
1127 collector.reg_late_use(operand_low);
1128 collector.reg_late_use(operand_high);
1129 collector.reg_fixed_def(temp_low, regs::rbx());
1130 collector.reg_fixed_def(temp_high, regs::rcx());
1131 collector.reg_fixed_def(dst_old_low, regs::rax());
1132 collector.reg_fixed_def(dst_old_high, regs::rdx());
1133 mem_low.get_operands_late(collector);
1134 mem_high.get_operands_late(collector);
1135 }
1136
1137 Inst::Atomic128XchgSeq { args } => {
1138 let Atomic128XchgSeqArgs {
1139 mem_low,
1140 mem_high,
1141 operand_low,
1142 operand_high,
1143 dst_old_low,
1144 dst_old_high,
1145 } = &mut **args;
1146 // All registers are collected in the `Late` position so that they don't overlap.
1147 collector.reg_fixed_late_use(operand_low, regs::rbx());
1148 collector.reg_fixed_late_use(operand_high, regs::rcx());
1149 collector.reg_fixed_def(dst_old_low, regs::rax());
1150 collector.reg_fixed_def(dst_old_high, regs::rdx());
1151 mem_low.get_operands_late(collector);
1152 mem_high.get_operands_late(collector);
1153 }
1154
1155 Inst::Args { args } => {
1156 for ArgPair { vreg, preg } in args {
1157 collector.reg_fixed_def(vreg, *preg);
1158 }
1159 }
1160
1161 Inst::Rets { rets } => {
1162 // The return value(s) are live-out; we represent this
1163 // with register uses on the return instruction.
1164 for RetPair { vreg, preg } in rets {
1165 collector.reg_fixed_use(vreg, *preg);
1166 }
1167 }
1168
1169 Inst::JmpKnown { .. }
1170 | Inst::WinchJmpIf { .. }
1171 | Inst::JmpCond { .. }
1172 | Inst::JmpCondOr { .. }
1173 | Inst::TrapIf { .. }
1174 | Inst::TrapIfAnd { .. }
1175 | Inst::TrapIfOr { .. } => {
1176 // No registers are used.
1177 }
1178
1179 Inst::ElfTlsGetAddr { dst, .. } | Inst::MachOTlsGetAddr { dst, .. } => {
1180 collector.reg_fixed_def(dst, regs::rax());
1181 // All caller-saves are clobbered.
1182 //
1183 // We use the SysV calling convention here because the
1184 // pseudoinstruction (and relocation that it emits) is specific to
1185 // ELF systems; other x86-64 targets with other conventions (i.e.,
1186 // Windows) use different TLS strategies.
1187 let mut clobbers =
1188 X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV, false);
1189 clobbers.remove(regs::gpr_preg(asm::gpr::enc::RAX));
1190 collector.reg_clobbers(clobbers);
1191 }
1192
1193 Inst::CoffTlsGetAddr { dst, tmp, .. } => {
1194 // We also use the gs register. But that register is not allocatable by the
1195 // register allocator, so we don't need to mark it as used here.
1196
1197 // We use %rax to set the address
1198 collector.reg_fixed_def(dst, regs::rax());
1199
1200 // We use %rcx as a temporary variable to load the _tls_index
1201 collector.reg_fixed_def(tmp, regs::rcx());
1202 }
1203
1204 Inst::Unwind { .. } => {}
1205
1206 Inst::DummyUse { reg } => {
1207 collector.reg_use(reg);
1208 }
1209
1210 Inst::LabelAddress { dst, .. } => {
1211 collector.reg_def(dst);
1212 }
1213
1214 Inst::SequencePoint { .. } => {}
1215
1216 Inst::External { inst } => {
1217 inst.visit(&mut external::RegallocVisitor { collector });
1218 }
1219 }
1220 }
1221
1222 //=============================================================================
1223 // Instructions: misc functions and external interface
1224
1225 impl MachInst for Inst {
1226 type ABIMachineSpec = X64ABIMachineSpec;
1227
get_operands(&mut self, collector: &mut impl OperandVisitor)1228 fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
1229 x64_get_operands(self, collector)
1230 }
1231
is_move(&self) -> Option<(Writable<Reg>, Reg)>1232 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
1233 use asm::inst::Inst as I;
1234 match self {
1235 // Note (carefully!) that a 32-bit mov *isn't* a no-op since it zeroes
1236 // out the upper 32 bits of the destination. For example, we could
1237 // conceivably use `movl %reg, %reg` to zero out the top 32 bits of
1238 // %reg.
1239 Self::External {
1240 inst: I::movq_mr(asm::inst::movq_mr { rm64, r64 }),
1241 } => match rm64 {
1242 asm::GprMem::Gpr(reg) => Some((reg.map(|r| r.to_reg()), r64.as_ref().to_reg())),
1243 asm::GprMem::Mem(_) => None,
1244 },
1245 Self::External {
1246 inst: I::movq_rm(asm::inst::movq_rm { r64, rm64 }),
1247 } => match rm64 {
1248 asm::GprMem::Gpr(reg) => Some((r64.as_ref().map(|r| r.to_reg()), reg.to_reg())),
1249 asm::GprMem::Mem(_) => None,
1250 },
1251
1252 // Note that `movss_a_r` and `movsd_a_r` are specifically omitted
1253 // here because they only overwrite the low bits in the destination
1254 // register, otherwise preserving the upper bits. That can be used
1255 // for lane-insertion instructions, for example, meaning it's not
1256 // classified as a register move.
1257 //
1258 // Otherwise though all register-to-register movement instructions
1259 // which move 128-bits are registered as moves.
1260 Self::External {
1261 inst:
1262 I::movaps_a(asm::inst::movaps_a { xmm1, xmm_m128 })
1263 | I::movups_a(asm::inst::movups_a { xmm1, xmm_m128 })
1264 | I::movapd_a(asm::inst::movapd_a { xmm1, xmm_m128 })
1265 | I::movupd_a(asm::inst::movupd_a { xmm1, xmm_m128 })
1266 | I::movdqa_a(asm::inst::movdqa_a { xmm1, xmm_m128 })
1267 | I::movdqu_a(asm::inst::movdqu_a { xmm1, xmm_m128 }),
1268 } => match xmm_m128 {
1269 asm::XmmMem::Xmm(xmm2) => Some((xmm1.as_ref().map(|r| r.to_reg()), xmm2.to_reg())),
1270 asm::XmmMem::Mem(_) => None,
1271 },
1272 // In addition to the "A" format of instructions above also
1273 // recognize the "B" format which while it can be used for stores it
1274 // can also be used for register moves.
1275 Self::External {
1276 inst:
1277 I::movaps_b(asm::inst::movaps_b { xmm_m128, xmm1 })
1278 | I::movups_b(asm::inst::movups_b { xmm_m128, xmm1 })
1279 | I::movapd_b(asm::inst::movapd_b { xmm_m128, xmm1 })
1280 | I::movupd_b(asm::inst::movupd_b { xmm_m128, xmm1 })
1281 | I::movdqa_b(asm::inst::movdqa_b { xmm_m128, xmm1 })
1282 | I::movdqu_b(asm::inst::movdqu_b { xmm_m128, xmm1 }),
1283 } => match xmm_m128 {
1284 asm::XmmMem::Xmm(dst) => Some((dst.map(|r| r.to_reg()), xmm1.as_ref().to_reg())),
1285 asm::XmmMem::Mem(_) => None,
1286 },
1287 _ => None,
1288 }
1289 }
1290
is_included_in_clobbers(&self) -> bool1291 fn is_included_in_clobbers(&self) -> bool {
1292 match self {
1293 &Inst::Args { .. } => false,
1294 _ => true,
1295 }
1296 }
1297
is_trap(&self) -> bool1298 fn is_trap(&self) -> bool {
1299 match self {
1300 Self::External {
1301 inst: asm::inst::Inst::ud2_zo(..),
1302 } => true,
1303 _ => false,
1304 }
1305 }
1306
is_args(&self) -> bool1307 fn is_args(&self) -> bool {
1308 match self {
1309 Self::Args { .. } => true,
1310 _ => false,
1311 }
1312 }
1313
call_type(&self) -> CallType1314 fn call_type(&self) -> CallType {
1315 match self {
1316 Inst::CallKnown { .. }
1317 | Inst::CallUnknown { .. }
1318 | Inst::ElfTlsGetAddr { .. }
1319 | Inst::MachOTlsGetAddr { .. } => CallType::Regular,
1320
1321 Inst::ReturnCallKnown { .. } | Inst::ReturnCallUnknown { .. } => CallType::TailCall,
1322
1323 _ => CallType::None,
1324 }
1325 }
1326
is_term(&self) -> MachTerminator1327 fn is_term(&self) -> MachTerminator {
1328 match self {
1329 // Interesting cases.
1330 &Self::Rets { .. } => MachTerminator::Ret,
1331 &Self::ReturnCallKnown { .. } | &Self::ReturnCallUnknown { .. } => {
1332 MachTerminator::RetCall
1333 }
1334 &Self::JmpKnown { .. } => MachTerminator::Branch,
1335 &Self::JmpCond { .. } => MachTerminator::Branch,
1336 &Self::JmpCondOr { .. } => MachTerminator::Branch,
1337 &Self::JmpTableSeq { .. } => MachTerminator::Branch,
1338 &Self::CallKnown { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1339 &Self::CallUnknown { ref info } if info.try_call_info.is_some() => {
1340 MachTerminator::Branch
1341 }
1342 // All other cases are boring.
1343 _ => MachTerminator::None,
1344 }
1345 }
1346
is_low_level_branch(&self) -> bool1347 fn is_low_level_branch(&self) -> bool {
1348 match self {
1349 &Self::WinchJmpIf { .. } => true,
1350 _ => false,
1351 }
1352 }
1353
is_mem_access(&self) -> bool1354 fn is_mem_access(&self) -> bool {
1355 panic!("TODO FILL ME OUT")
1356 }
1357
gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst1358 fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
1359 trace!(
1360 "Inst::gen_move {:?} -> {:?} (type: {:?})",
1361 src_reg,
1362 dst_reg.to_reg(),
1363 ty
1364 );
1365 let rc_dst = dst_reg.to_reg().class();
1366 let rc_src = src_reg.class();
1367 // If this isn't true, we have gone way off the rails.
1368 debug_assert!(rc_dst == rc_src);
1369 let inst = match rc_dst {
1370 RegClass::Int => {
1371 asm::inst::movq_mr::new(dst_reg.map(Gpr::unwrap_new), Gpr::unwrap_new(src_reg))
1372 .into()
1373 }
1374 RegClass::Float => {
1375 // The Intel optimization manual, in "3.5.1.13 Zero-Latency MOV Instructions",
1376 // doesn't include MOVSS/MOVSD as instructions with zero-latency. Use movaps for
1377 // those, which may write more lanes that we need, but are specified to have
1378 // zero-latency.
1379 let dst_reg = dst_reg.map(|r| Xmm::new(r).unwrap());
1380 let src_reg = Xmm::new(src_reg).unwrap();
1381 match ty {
1382 types::F16 | types::F32 | types::F64 | types::F32X4 => {
1383 asm::inst::movaps_a::new(dst_reg, src_reg).into()
1384 }
1385 types::F64X2 => asm::inst::movapd_a::new(dst_reg, src_reg).into(),
1386 _ if (ty.is_float() || ty.is_vector()) && ty.bits() <= 128 => {
1387 asm::inst::movdqa_a::new(dst_reg, src_reg).into()
1388 }
1389 _ => unimplemented!("unable to move type: {}", ty),
1390 }
1391 }
1392 RegClass::Vector => unreachable!(),
1393 };
1394 Inst::External { inst }
1395 }
1396
gen_nop(preferred_size: usize) -> Inst1397 fn gen_nop(preferred_size: usize) -> Inst {
1398 Inst::nop(core::cmp::min(preferred_size, 9) as u8)
1399 }
1400
gen_nop_units() -> Vec<Vec<u8>>1401 fn gen_nop_units() -> Vec<Vec<u8>> {
1402 vec![
1403 // Standard 1-byte NOP.
1404 vec![0x90],
1405 // 5-byte NOP useful for patching out patchable calls.
1406 vec![0x0f, 0x1f, 0x44, 0x00, 0x00],
1407 ]
1408 }
1409
rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])>1410 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
1411 match ty {
1412 types::I8 => Ok((&[RegClass::Int], &[types::I8])),
1413 types::I16 => Ok((&[RegClass::Int], &[types::I16])),
1414 types::I32 => Ok((&[RegClass::Int], &[types::I32])),
1415 types::I64 => Ok((&[RegClass::Int], &[types::I64])),
1416 types::F16 => Ok((&[RegClass::Float], &[types::F16])),
1417 types::F32 => Ok((&[RegClass::Float], &[types::F32])),
1418 types::F64 => Ok((&[RegClass::Float], &[types::F64])),
1419 types::F128 => Ok((&[RegClass::Float], &[types::F128])),
1420 types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
1421 _ if ty.is_vector() && ty.bits() <= 128 => {
1422 let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16];
1423 Ok((
1424 &[RegClass::Float],
1425 slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]),
1426 ))
1427 }
1428 _ => Err(CodegenError::Unsupported(format!(
1429 "Unexpected SSA-value type: {ty}"
1430 ))),
1431 }
1432 }
1433
canonical_type_for_rc(rc: RegClass) -> Type1434 fn canonical_type_for_rc(rc: RegClass) -> Type {
1435 match rc {
1436 RegClass::Float => types::I8X16,
1437 RegClass::Int => types::I64,
1438 RegClass::Vector => unreachable!(),
1439 }
1440 }
1441
gen_jump(label: MachLabel) -> Inst1442 fn gen_jump(label: MachLabel) -> Inst {
1443 Inst::jmp_known(label)
1444 }
1445
gen_imm_u64(value: u64, dst: Writable<Reg>) -> Option<Self>1446 fn gen_imm_u64(value: u64, dst: Writable<Reg>) -> Option<Self> {
1447 Some(Inst::imm(OperandSize::Size64, value, dst))
1448 }
1449
gen_imm_f64(value: f64, tmp: Writable<Reg>, dst: Writable<Reg>) -> SmallVec<[Self; 2]>1450 fn gen_imm_f64(value: f64, tmp: Writable<Reg>, dst: Writable<Reg>) -> SmallVec<[Self; 2]> {
1451 let imm_to_gpr = Inst::imm(OperandSize::Size64, value.to_bits(), tmp);
1452 let gpr_to_xmm = Inst::External {
1453 inst: asm::inst::movq_a::new(dst.map(|r| Xmm::new(r).unwrap()), tmp.to_reg()).into(),
1454 };
1455 smallvec![imm_to_gpr, gpr_to_xmm]
1456 }
1457
gen_dummy_use(reg: Reg) -> Self1458 fn gen_dummy_use(reg: Reg) -> Self {
1459 Inst::DummyUse { reg }
1460 }
1461
worst_case_size() -> CodeOffset1462 fn worst_case_size() -> CodeOffset {
1463 15
1464 }
1465
ref_type_regclass(_: &settings::Flags) -> RegClass1466 fn ref_type_regclass(_: &settings::Flags) -> RegClass {
1467 RegClass::Int
1468 }
1469
is_safepoint(&self) -> bool1470 fn is_safepoint(&self) -> bool {
1471 match self {
1472 Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true,
1473 _ => false,
1474 }
1475 }
1476
function_alignment() -> FunctionAlignment1477 fn function_alignment() -> FunctionAlignment {
1478 FunctionAlignment {
1479 minimum: 1,
1480 // Change the alignment from 16-bytes to 32-bytes for better performance.
1481 // fix-8573: https://github.com/bytecodealliance/wasmtime/issues/8573
1482 preferred: 32,
1483 }
1484 }
1485
1486 type LabelUse = LabelUse;
1487
1488 const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b];
1489 }
1490
1491 /// Constant state used during emissions of a sequence of instructions.
1492 pub struct EmitInfo {
1493 pub(super) flags: settings::Flags,
1494 isa_flags: x64_settings::Flags,
1495 }
1496
1497 impl EmitInfo {
1498 /// Create a constant state for emission of instructions.
new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self1499 pub fn new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self {
1500 Self { flags, isa_flags }
1501 }
1502 }
1503
1504 impl asm::AvailableFeatures for &EmitInfo {
_64b(&self) -> bool1505 fn _64b(&self) -> bool {
1506 // Currently, this x64 backend always assumes 64-bit mode.
1507 true
1508 }
1509
compat(&self) -> bool1510 fn compat(&self) -> bool {
1511 // For 32-bit compatibility mode, see
1512 // https://github.com/bytecodealliance/wasmtime/issues/1980 (TODO).
1513 false
1514 }
1515
sse(&self) -> bool1516 fn sse(&self) -> bool {
1517 // Currently, this x64 backend always assumes SSE.
1518 true
1519 }
1520
sse2(&self) -> bool1521 fn sse2(&self) -> bool {
1522 // Currently, this x64 backend always assumes SSE2.
1523 true
1524 }
1525
sse3(&self) -> bool1526 fn sse3(&self) -> bool {
1527 self.isa_flags.has_sse3()
1528 }
1529
ssse3(&self) -> bool1530 fn ssse3(&self) -> bool {
1531 self.isa_flags.has_ssse3()
1532 }
1533
sse41(&self) -> bool1534 fn sse41(&self) -> bool {
1535 self.isa_flags.has_sse41()
1536 }
1537
sse42(&self) -> bool1538 fn sse42(&self) -> bool {
1539 self.isa_flags.has_sse42()
1540 }
1541
bmi1(&self) -> bool1542 fn bmi1(&self) -> bool {
1543 self.isa_flags.has_bmi1()
1544 }
1545
bmi2(&self) -> bool1546 fn bmi2(&self) -> bool {
1547 self.isa_flags.has_bmi2()
1548 }
1549
lzcnt(&self) -> bool1550 fn lzcnt(&self) -> bool {
1551 self.isa_flags.has_lzcnt()
1552 }
1553
popcnt(&self) -> bool1554 fn popcnt(&self) -> bool {
1555 self.isa_flags.has_popcnt()
1556 }
1557
avx(&self) -> bool1558 fn avx(&self) -> bool {
1559 self.isa_flags.has_avx()
1560 }
1561
avx2(&self) -> bool1562 fn avx2(&self) -> bool {
1563 self.isa_flags.has_avx2()
1564 }
1565
avx512f(&self) -> bool1566 fn avx512f(&self) -> bool {
1567 self.isa_flags.has_avx512f()
1568 }
1569
avx512vl(&self) -> bool1570 fn avx512vl(&self) -> bool {
1571 self.isa_flags.has_avx512vl()
1572 }
1573
cmpxchg16b(&self) -> bool1574 fn cmpxchg16b(&self) -> bool {
1575 self.isa_flags.has_cmpxchg16b()
1576 }
1577
fma(&self) -> bool1578 fn fma(&self) -> bool {
1579 self.isa_flags.has_fma()
1580 }
1581
avx512dq(&self) -> bool1582 fn avx512dq(&self) -> bool {
1583 self.isa_flags.has_avx512dq()
1584 }
1585
avx512bitalg(&self) -> bool1586 fn avx512bitalg(&self) -> bool {
1587 self.isa_flags.has_avx512bitalg()
1588 }
1589
avx512vbmi(&self) -> bool1590 fn avx512vbmi(&self) -> bool {
1591 self.isa_flags.has_avx512vbmi()
1592 }
1593 }
1594
1595 impl MachInstEmit for Inst {
1596 type State = EmitState;
1597 type Info = EmitInfo;
1598
emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State)1599 fn emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State) {
1600 emit::emit(self, sink, info, state);
1601 }
1602
pretty_print_inst(&self, _: &mut Self::State) -> String1603 fn pretty_print_inst(&self, _: &mut Self::State) -> String {
1604 PrettyPrint::pretty_print(self, 0)
1605 }
1606 }
1607
1608 /// A label-use (internal relocation) in generated code.
1609 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
1610 pub enum LabelUse {
1611 /// A 32-bit offset from location of relocation itself, added to the existing value at that
1612 /// location. Used for control flow instructions which consider an offset from the start of the
1613 /// next instruction (so the size of the payload -- 4 bytes -- is subtracted from the payload).
1614 JmpRel32,
1615
1616 /// A 32-bit offset from location of relocation itself, added to the existing value at that
1617 /// location.
1618 PCRel32,
1619 }
1620
1621 impl MachInstLabelUse for LabelUse {
1622 const ALIGN: CodeOffset = 1;
1623
max_pos_range(self) -> CodeOffset1624 fn max_pos_range(self) -> CodeOffset {
1625 match self {
1626 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x7fff_ffff,
1627 }
1628 }
1629
max_neg_range(self) -> CodeOffset1630 fn max_neg_range(self) -> CodeOffset {
1631 match self {
1632 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x8000_0000,
1633 }
1634 }
1635
patch_size(self) -> CodeOffset1636 fn patch_size(self) -> CodeOffset {
1637 match self {
1638 LabelUse::JmpRel32 | LabelUse::PCRel32 => 4,
1639 }
1640 }
1641
patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset)1642 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
1643 let pc_rel = (label_offset as i64) - (use_offset as i64);
1644 debug_assert!(pc_rel <= self.max_pos_range() as i64);
1645 debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
1646 let pc_rel = pc_rel as u32;
1647 match self {
1648 LabelUse::JmpRel32 => {
1649 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1650 let value = pc_rel.wrapping_add(addend).wrapping_sub(4);
1651 buffer.copy_from_slice(&value.to_le_bytes()[..]);
1652 }
1653 LabelUse::PCRel32 => {
1654 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1655 let value = pc_rel.wrapping_add(addend);
1656 buffer.copy_from_slice(&value.to_le_bytes()[..]);
1657 }
1658 }
1659 }
1660
supports_veneer(self) -> bool1661 fn supports_veneer(self) -> bool {
1662 match self {
1663 LabelUse::JmpRel32 | LabelUse::PCRel32 => false,
1664 }
1665 }
1666
veneer_size(self) -> CodeOffset1667 fn veneer_size(self) -> CodeOffset {
1668 match self {
1669 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0,
1670 }
1671 }
1672
worst_case_veneer_size() -> CodeOffset1673 fn worst_case_veneer_size() -> CodeOffset {
1674 0
1675 }
1676
generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse)1677 fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) {
1678 match self {
1679 LabelUse::JmpRel32 | LabelUse::PCRel32 => {
1680 panic!("Veneer not supported for JumpRel32 label-use.");
1681 }
1682 }
1683 }
1684
from_reloc(reloc: Reloc, addend: Addend) -> Option<Self>1685 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {
1686 match (reloc, addend) {
1687 (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
1688 _ => None,
1689 }
1690 }
1691 }
1692