//! Interface with the external assembler crate. use super::{ Amode, Gpr, Inst, LabelUse, MachBuffer, MachLabel, OperandVisitor, OperandVisitorImpl, SyntheticAmode, VCodeConstant, WritableGpr, WritableXmm, Xmm, args::FromWritableReg, }; use crate::{Reg, Writable, ir::TrapCode}; use alloc::string::String; use cranelift_assembler_x64 as asm; use regalloc2::{PReg, RegClass}; /// Define the types of registers Cranelift will use. #[derive(Clone, Debug)] pub struct CraneliftRegisters; impl asm::Registers for CraneliftRegisters { type ReadGpr = Gpr; type ReadWriteGpr = PairedGpr; type WriteGpr = WritableGpr; type ReadXmm = Xmm; type ReadWriteXmm = PairedXmm; type WriteXmm = WritableXmm; } /// Convenience type alias of `asm::inst::Inst` with `R = CraneliftRegisters` /// filled in. pub type AsmInst = asm::inst::Inst; /// A pair of registers, one for reading and one for writing. /// /// Due to how Cranelift's SSA form, we must track the read and write registers /// separately prior to register allocation. Once register allocation is /// complete, we expect the hardware encoding for both `read` and `write` to be /// the same. #[derive(Clone, Copy, Debug, PartialEq)] #[expect(missing_docs, reason = "self-describing variants")] pub struct PairedGpr { pub read: Gpr, pub write: WritableGpr, } impl From for PairedGpr { fn from(wgpr: WritableGpr) -> Self { let read = wgpr.to_reg(); let write = wgpr; Self { read, write } } } /// For ABI ergonomics. impl From for asm::Gpr { fn from(wgpr: WritableGpr) -> Self { asm::Gpr::new(wgpr.into()) } } // For ABI ergonomics. impl From> for asm::GprMem { fn from(wgpr: Writable) -> Self { assert!(wgpr.to_reg().class() == RegClass::Int); let wgpr = WritableGpr::from_writable_reg(wgpr).unwrap(); Self::Gpr(wgpr.into()) } } // For ABI ergonomics. impl From for asm::GprMem { fn from(gpr: Reg) -> Self { assert!(gpr.class() == RegClass::Int); let gpr = Gpr::unwrap_new(gpr); Self::Gpr(gpr) } } // For ABI ergonomics. impl From> for asm::GprMem { fn from(wgpr: Writable) -> Self { wgpr.to_reg().into() } } // For ABI ergonomics. impl From> for asm::Gpr { fn from(wgpr: Writable) -> Self { assert!(wgpr.to_reg().class() == RegClass::Int); let wgpr = WritableGpr::from_writable_reg(wgpr).unwrap(); Self::new(wgpr.into()) } } impl From> for asm::Gpr { fn from(wgpr: Writable) -> Self { assert!(wgpr.to_reg().class() == RegClass::Int); let wgpr = WritableGpr::from_writable_reg(wgpr).unwrap(); Self::new(wgpr) } } impl asm::AsReg for PairedGpr { fn enc(&self) -> u8 { let PairedGpr { read, write } = self; let read = enc_gpr(read); let write = enc_gpr(&write.to_reg()); assert_eq!(read, write); write } fn to_string(&self, size: Option) -> String { if self.read.is_real() { asm::gpr::enc::to_string(self.enc(), size.unwrap()).into() } else { let read = self.read.to_reg(); let write = self.write.to_reg().to_reg(); format!("(%{write:?} <- %{read:?})") } } fn new(_: u8) -> Self { panic!("disallow creation of new assembler registers") } } /// A pair of XMM registers, one for reading and one for writing. #[derive(Clone, Copy, Debug, PartialEq)] #[expect(missing_docs, reason = "self-describing variants")] pub struct PairedXmm { pub read: Xmm, pub write: WritableXmm, } impl From for PairedXmm { fn from(wxmm: WritableXmm) -> Self { let read = wxmm.to_reg(); let write = wxmm; Self { read, write } } } /// For ABI ergonomics. impl From for asm::Xmm { fn from(wgpr: WritableXmm) -> Self { asm::Xmm::new(wgpr.into()) } } // For emission ergonomics. impl From> for asm::Xmm { fn from(wxmm: Writable) -> Self { assert!(wxmm.to_reg().class() == RegClass::Float); let wxmm = WritableXmm::from_writable_reg(wxmm).unwrap(); Self::new(wxmm.into()) } } // For emission ergonomics. impl From for asm::Xmm { fn from(xmm: Reg) -> Self { assert!(xmm.class() == RegClass::Float); let xmm = Xmm::unwrap_new(xmm); Self::new(xmm) } } // For emission ergonomics. impl From for asm::XmmMem { fn from(xmm: Reg) -> Self { assert!(xmm.class() == RegClass::Float); let xmm = Xmm::unwrap_new(xmm); Self::Xmm(xmm) } } impl asm::AsReg for PairedXmm { fn enc(&self) -> u8 { let PairedXmm { read, write } = self; let read = enc_xmm(read); let write = enc_xmm(&write.to_reg()); assert_eq!(read, write); write } fn to_string(&self, size: Option) -> String { assert!(size.is_none(), "XMM registers do not have size variants"); if self.read.is_real() { asm::xmm::enc::to_string(self.enc()).into() } else { let read = self.read.to_reg(); let write = self.write.to_reg().to_reg(); format!("(%{write:?} <- %{read:?})") } } fn new(_: u8) -> Self { panic!("disallow creation of new assembler registers") } } /// This bridges the gap between codegen and assembler for general purpose register types. impl asm::AsReg for Gpr { fn enc(&self) -> u8 { enc_gpr(self) } fn to_string(&self, size: Option) -> String { if self.is_real() { asm::gpr::enc::to_string(self.enc(), size.unwrap()).into() } else { format!("%{:?}", self.to_reg()) } } fn new(_: u8) -> Self { panic!("disallow creation of new assembler registers") } } /// This bridges the gap between codegen and assembler for xmm register types. impl asm::AsReg for Xmm { fn enc(&self) -> u8 { enc_xmm(self) } fn to_string(&self, size: Option) -> String { assert!(size.is_none(), "XMM registers do not have size variants"); if self.is_real() { asm::xmm::enc::to_string(self.enc()).into() } else { format!("%{:?}", self.to_reg()) } } fn new(_: u8) -> Self { panic!("disallow creation of new assembler registers") } } /// A helper method for extracting the hardware encoding of a general purpose register. #[inline] fn enc_gpr(gpr: &Gpr) -> u8 { if let Some(real) = gpr.to_reg().to_real_reg() { real.hw_enc() } else { unreachable!() } } /// A helper method for extracting the hardware encoding of an xmm register. #[inline] fn enc_xmm(xmm: &Xmm) -> u8 { if let Some(real) = xmm.to_reg().to_real_reg() { real.hw_enc() } else { unreachable!() } } /// A wrapper to implement the `cranelift-assembler-x64` register allocation trait, /// `RegallocVisitor`, in terms of the trait used in Cranelift, /// `OperandVisitor`. pub(crate) struct RegallocVisitor<'a, T> where T: OperandVisitorImpl, { pub collector: &'a mut T, } impl<'a, T: OperandVisitor> asm::RegisterVisitor for RegallocVisitor<'a, T> { fn read_gpr(&mut self, reg: &mut Gpr) { self.collector.reg_use(reg); } fn read_write_gpr(&mut self, reg: &mut PairedGpr) { let PairedGpr { read, write } = reg; self.collector.reg_use(read); self.collector.reg_reuse_def(write, 0); } fn write_gpr(&mut self, reg: &mut WritableGpr) { self.collector.reg_def(reg); } fn fixed_read_gpr(&mut self, reg: &mut Gpr, enc: u8) { self.collector .reg_fixed_use(reg, fixed_reg(enc, RegClass::Int)); } fn fixed_read_write_gpr(&mut self, reg: &mut PairedGpr, enc: u8) { let PairedGpr { read, write } = reg; self.collector .reg_fixed_use(read, fixed_reg(enc, RegClass::Int)); self.collector .reg_fixed_def(write, fixed_reg(enc, RegClass::Int)); } fn fixed_write_gpr(&mut self, reg: &mut WritableGpr, enc: u8) { self.collector .reg_fixed_def(reg, fixed_reg(enc, RegClass::Int)); } fn read_xmm(&mut self, reg: &mut Xmm) { self.collector.reg_use(reg); } fn read_write_xmm(&mut self, reg: &mut PairedXmm) { let PairedXmm { read, write } = reg; self.collector.reg_use(read); self.collector.reg_reuse_def(write, 0); } fn write_xmm(&mut self, reg: &mut WritableXmm) { self.collector.reg_def(reg); } fn fixed_read_xmm(&mut self, reg: &mut Xmm, enc: u8) { self.collector .reg_fixed_use(reg, fixed_reg(enc, RegClass::Float)); } fn fixed_read_write_xmm(&mut self, reg: &mut PairedXmm, enc: u8) { let PairedXmm { read, write } = reg; self.collector .reg_fixed_use(read, fixed_reg(enc, RegClass::Float)); self.collector .reg_fixed_def(write, fixed_reg(enc, RegClass::Float)); } fn fixed_write_xmm(&mut self, reg: &mut WritableXmm, enc: u8) { self.collector .reg_fixed_def(reg, fixed_reg(enc, RegClass::Float)); } } /// A helper for building a fixed register from its hardware encoding. fn fixed_reg(enc: u8, class: RegClass) -> Reg { let preg = PReg::new(usize::from(enc), class); Reg::from_real_reg(preg) } impl From for asm::Amode { fn from(amode: SyntheticAmode) -> asm::Amode { match amode { SyntheticAmode::Real(amode) => amode.into(), SyntheticAmode::IncomingArg { offset } => asm::Amode::ImmReg { base: Gpr::RBP, simm32: asm::AmodeOffsetPlusKnownOffset { simm32: (-i32::try_from(offset).unwrap()).into(), offset: Some(offsets::KEY_INCOMING_ARG), }, trap: None, }, SyntheticAmode::SlotOffset { simm32 } => asm::Amode::ImmReg { base: Gpr::RSP, simm32: asm::AmodeOffsetPlusKnownOffset { simm32: simm32.into(), offset: Some(offsets::KEY_SLOT_OFFSET), }, trap: None, }, SyntheticAmode::ConstantOffset(vcode_constant) => asm::Amode::RipRelative { target: asm::DeferredTarget::Constant(asm::Constant(vcode_constant.as_u32())), }, } } } impl From for asm::Amode { fn from(amode: Amode) -> asm::Amode { match amode { Amode::ImmReg { simm32, base, flags, } => asm::Amode::ImmReg { simm32: asm::AmodeOffsetPlusKnownOffset { simm32: simm32.into(), offset: None, }, base: Gpr::unwrap_new(base), trap: flags.trap_code().map(Into::into), }, Amode::ImmRegRegShift { simm32, base, index, shift, flags, } => asm::Amode::ImmRegRegShift { base, index: asm::NonRspGpr::new(index), scale: asm::Scale::new(shift), simm32: simm32.into(), trap: flags.trap_code().map(Into::into), }, Amode::RipRelative { target } => asm::Amode::RipRelative { target: asm::DeferredTarget::Label(asm::Label(target.as_u32())), }, } } } impl From for asm::XmmMem { fn from(amode: SyntheticAmode) -> Self { asm::XmmMem::Mem(amode.into()) } } impl From for asm::GprMem { fn from(amode: SyntheticAmode) -> Self { asm::GprMem::Mem(amode.into()) } } impl From for asm::XmmMem { fn from(amode: Amode) -> Self { asm::XmmMem::Mem(amode.into()) } } impl From for asm::GprMem { fn from(amode: Amode) -> Self { asm::GprMem::Mem(amode.into()) } } /// Keep track of the offset slots to fill in during emission; see /// `KnownOffsetTable`. #[expect(missing_docs, reason = "self-describing keys")] pub mod offsets { pub const KEY_INCOMING_ARG: u8 = 0; pub const KEY_SLOT_OFFSET: u8 = 1; } /// Implementor of the [`asm::CodeSink`] trait. pub struct AsmCodeSink<'a> { /// The buffer this is emitting into. pub sink: &'a mut MachBuffer, /// The value of `KEY_INCOMING_ARG`. pub incoming_arg_offset: i32, /// The value of `KEY_SLOT_OFFSET`. pub slot_offset: i32, } impl asm::CodeSink for AsmCodeSink<'_> { fn put1(&mut self, value: u8) { self.sink.put1(value) } fn put2(&mut self, value: u16) { self.sink.put2(value) } fn put4(&mut self, value: u32) { self.sink.put4(value) } fn put8(&mut self, value: u64) { self.sink.put8(value) } fn add_trap(&mut self, code: asm::TrapCode) { self.sink.add_trap(code.into()); } fn use_target(&mut self, target: asm::DeferredTarget) { let offset = self.sink.cur_offset(); match target { asm::DeferredTarget::Label(label) => { self.sink .use_label_at_offset(offset, label.into(), LabelUse::JmpRel32); } asm::DeferredTarget::Constant(constant) => { let label = self.sink.get_label_for_constant(constant.into()); self.sink .use_label_at_offset(offset, label, LabelUse::JmpRel32); } asm::DeferredTarget::None => {} } } fn known_offset(&self, offset: asm::KnownOffset) -> i32 { match offset { offsets::KEY_INCOMING_ARG => self.incoming_arg_offset, offsets::KEY_SLOT_OFFSET => self.slot_offset, other => panic!("unknown \"known\" offset {other}"), } } } impl From for TrapCode { fn from(value: asm::TrapCode) -> Self { Self::from_raw(value.0) } } impl From for asm::TrapCode { fn from(value: TrapCode) -> Self { Self(value.as_raw()) } } impl From for MachLabel { fn from(value: asm::Label) -> Self { Self::from_u32(value.0) } } impl From for asm::Label { fn from(value: MachLabel) -> Self { Self(value.as_u32()) } } impl From for VCodeConstant { fn from(value: asm::Constant) -> Self { Self::from_u32(value.0) } } // Include code generated by `cranelift-codegen/meta/src/gen_asm.rs`. This file // contains a `isle_assembler_methods!` macro with Rust implementations of all // the assembler instructions exposed to ISLE. include!(concat!(env!("OUT_DIR"), "/assembler-isle-macro.rs")); pub(crate) use isle_assembler_methods; #[cfg(test)] mod tests { use super::PairedGpr; use super::asm::{AsReg, Size}; use crate::isa::x64::args::{FromWritableReg, Gpr, WritableGpr, WritableXmm, Xmm}; use crate::isa::x64::inst::external::PairedXmm; use crate::{Reg, Writable}; use regalloc2::{RegClass, VReg}; #[test] fn pretty_print_registers() { // For logging, we need to be able to pretty-print the virtual registers // that Cranelift uses before register allocation. This test ensures // that these remain printable using the `AsReg::to_string` interface // (see issue #10631). let v200: Reg = VReg::new(200, RegClass::Int).into(); let gpr200 = Gpr::new(v200).unwrap(); assert_eq!(gpr200.to_string(Some(Size::Quadword)), "%v200"); let v300: Reg = VReg::new(300, RegClass::Int).into(); let wgpr300 = WritableGpr::from_writable_reg(Writable::from_reg(v300)).unwrap(); let pair = PairedGpr { read: gpr200, write: wgpr300, }; assert_eq!(pair.to_string(Some(Size::Quadword)), "(%v300 <- %v200)"); let v400: Reg = VReg::new(400, RegClass::Float).into(); let xmm400 = Xmm::new(v400).unwrap(); assert_eq!(xmm400.to_string(None), "%v400"); let v500: Reg = VReg::new(500, RegClass::Float).into(); let wxmm500 = WritableXmm::from_writable_reg(Writable::from_reg(v500)).unwrap(); let pair = PairedXmm { read: xmm400, write: wxmm500, }; assert_eq!(pair.to_string(None), "(%v500 <- %v400)"); } }