1 //! Implementation of a standard Pulley ABI. 2 3 use super::{PulleyFlags, PulleyTargetKind, inst::*}; 4 use crate::isa::pulley_shared::PointerWidth; 5 use crate::{ 6 CodegenResult, 7 ir::{self, MemFlags, Signature, types::*}, 8 isa, 9 machinst::*, 10 settings, 11 }; 12 use alloc::borrow::ToOwned; 13 use alloc::vec::Vec; 14 use core::marker::PhantomData; 15 use cranelift_bitset::ScalarBitSet; 16 use regalloc2::{MachineEnv, PReg, PRegSet}; 17 use smallvec::{SmallVec, smallvec}; 18 use std::sync::OnceLock; 19 20 /// Support for the Pulley ABI from the callee side (within a function body). 21 pub(crate) type PulleyCallee<P> = Callee<PulleyMachineDeps<P>>; 22 23 /// Pulley-specific ABI behavior. This struct just serves as an implementation 24 /// point for the trait; it is never actually instantiated. 25 pub struct PulleyMachineDeps<P> 26 where 27 P: PulleyTargetKind, 28 { 29 _phantom: PhantomData<P>, 30 } 31 32 impl<P> ABIMachineSpec for PulleyMachineDeps<P> 33 where 34 P: PulleyTargetKind, 35 { 36 type I = InstAndKind<P>; 37 type F = PulleyFlags; 38 39 /// This is the limit for the size of argument and return-value areas on the 40 /// stack. We place a reasonable limit here to avoid integer overflow issues 41 /// with 32-bit arithmetic: for now, 128 MB. 42 const STACK_ARG_RET_SIZE_LIMIT: u32 = 128 * 1024 * 1024; 43 44 fn word_bits() -> u32 { 45 P::pointer_width().bits().into() 46 } 47 48 /// Return required stack alignment in bytes. 49 fn stack_align(_call_conv: isa::CallConv) -> u32 { 50 16 51 } 52 53 fn compute_arg_locs( 54 call_conv: isa::CallConv, 55 flags: &settings::Flags, 56 params: &[ir::AbiParam], 57 args_or_rets: ArgsOrRets, 58 add_ret_area_ptr: bool, 59 mut args: ArgsAccumulator, 60 ) -> CodegenResult<(u32, Option<usize>)> { 61 // NB: make sure this method stays in sync with 62 // `cranelift_pulley::interp::Vm::call`. 63 // 64 // In general we use the first half of all register banks as argument 65 // passing registers because, well, why not for now. Currently the only 66 // exception is x15 which is reserved as a single caller-saved register 67 // not used for arguments. This is used in `ReturnCallIndirect` to hold 68 // the location of where we're jumping to. 69 70 let x_end = 14; 71 let f_end = 15; 72 let v_end = 15; 73 74 let mut next_x_reg = 0; 75 let mut next_f_reg = 0; 76 let mut next_v_reg = 0; 77 let mut next_stack: u32 = 0; 78 79 let ret_area_ptr = if add_ret_area_ptr { 80 debug_assert_eq!(args_or_rets, ArgsOrRets::Args); 81 next_x_reg += 1; 82 Some(ABIArg::reg( 83 x_reg(next_x_reg - 1).to_real_reg().unwrap(), 84 I64, 85 ir::ArgumentExtension::None, 86 ir::ArgumentPurpose::Normal, 87 )) 88 } else { 89 None 90 }; 91 92 for param in params { 93 // Find the regclass(es) of the register(s) used to store a value of 94 // this type. 95 let (rcs, reg_tys) = Self::I::rc_for_type(param.value_type)?; 96 97 let mut slots = ABIArgSlotVec::new(); 98 for (rc, reg_ty) in rcs.iter().zip(reg_tys.iter()) { 99 let next_reg = if (next_x_reg <= x_end) && *rc == RegClass::Int { 100 let x = Some(x_reg(next_x_reg)); 101 next_x_reg += 1; 102 x 103 } else if (next_f_reg <= f_end) && *rc == RegClass::Float { 104 let f = Some(f_reg(next_f_reg)); 105 next_f_reg += 1; 106 f 107 } else if (next_v_reg <= v_end) && *rc == RegClass::Vector { 108 let v = Some(v_reg(next_v_reg)); 109 next_v_reg += 1; 110 v 111 } else { 112 None 113 }; 114 115 if let Some(reg) = next_reg { 116 slots.push(ABIArgSlot::Reg { 117 reg: reg.to_real_reg().unwrap(), 118 ty: *reg_ty, 119 extension: param.extension, 120 }); 121 } else { 122 if args_or_rets == ArgsOrRets::Rets && !flags.enable_multi_ret_implicit_sret() { 123 return Err(crate::CodegenError::Unsupported( 124 "Too many return values to fit in registers. \ 125 Use a StructReturn argument instead. (#9510)" 126 .to_owned(), 127 )); 128 } 129 130 // Compute size and 16-byte stack alignment happens 131 // separately after all args. 132 let size = reg_ty.bits() / 8; 133 let size = core::cmp::max(size, 8); 134 135 // Align. 136 debug_assert!(size.is_power_of_two()); 137 next_stack = align_to(next_stack, size); 138 139 slots.push(ABIArgSlot::Stack { 140 offset: i64::from(next_stack), 141 ty: *reg_ty, 142 extension: param.extension, 143 }); 144 145 next_stack += size; 146 } 147 } 148 149 args.push(ABIArg::Slots { 150 slots, 151 purpose: param.purpose, 152 }); 153 } 154 155 let pos = if let Some(ret_area_ptr) = ret_area_ptr { 156 args.push_non_formal(ret_area_ptr); 157 Some(args.args().len() - 1) 158 } else { 159 None 160 }; 161 162 next_stack = align_to(next_stack, Self::stack_align(call_conv)); 163 164 Ok((next_stack, pos)) 165 } 166 167 fn gen_load_stack(mem: StackAMode, into_reg: Writable<Reg>, ty: Type) -> Self::I { 168 let mut flags = MemFlags::trusted(); 169 // Stack loads/stores of vectors always use little-endianess to avoid 170 // implementing a byte-swap of vectors on big-endian platforms. 171 if ty.is_vector() { 172 flags.set_endianness(ir::Endianness::Little); 173 } 174 Inst::gen_load(into_reg, mem.into(), ty, flags).into() 175 } 176 177 fn gen_store_stack(mem: StackAMode, from_reg: Reg, ty: Type) -> Self::I { 178 let mut flags = MemFlags::trusted(); 179 // Stack loads/stores of vectors always use little-endianess to avoid 180 // implementing a byte-swap of vectors on big-endian platforms. 181 if ty.is_vector() { 182 flags.set_endianness(ir::Endianness::Little); 183 } 184 Inst::gen_store(mem.into(), from_reg, ty, flags).into() 185 } 186 187 fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self::I { 188 Self::I::gen_move(to_reg, from_reg, ty) 189 } 190 191 fn gen_extend( 192 dst: Writable<Reg>, 193 src: Reg, 194 signed: bool, 195 from_bits: u8, 196 to_bits: u8, 197 ) -> Self::I { 198 assert!(from_bits < to_bits); 199 let src = XReg::new(src).unwrap(); 200 let dst = dst.try_into().unwrap(); 201 match (signed, from_bits) { 202 (true, 8) => RawInst::Sext8 { dst, src }.into(), 203 (true, 16) => RawInst::Sext16 { dst, src }.into(), 204 (true, 32) => RawInst::Sext32 { dst, src }.into(), 205 (false, 8) => RawInst::Zext8 { dst, src }.into(), 206 (false, 16) => RawInst::Zext16 { dst, src }.into(), 207 (false, 32) => RawInst::Zext32 { dst, src }.into(), 208 _ => unimplemented!("extend {from_bits} to {to_bits} as signed? {signed}"), 209 } 210 } 211 212 fn get_ext_mode( 213 _call_conv: isa::CallConv, 214 specified: ir::ArgumentExtension, 215 ) -> ir::ArgumentExtension { 216 specified 217 } 218 219 fn gen_args(args: Vec<ArgPair>) -> Self::I { 220 Inst::Args { args }.into() 221 } 222 223 fn gen_rets(rets: Vec<RetPair>) -> Self::I { 224 Inst::Rets { rets }.into() 225 } 226 227 fn get_stacklimit_reg(_call_conv: isa::CallConv) -> Reg { 228 spilltmp_reg() 229 } 230 231 fn gen_add_imm( 232 _call_conv: isa::CallConv, 233 into_reg: Writable<Reg>, 234 from_reg: Reg, 235 imm: u32, 236 ) -> SmallInstVec<Self::I> { 237 let dst = into_reg.try_into().unwrap(); 238 let imm = imm as i32; 239 smallvec![ 240 RawInst::Xconst32 { dst, imm }.into(), 241 RawInst::Xadd32 { 242 dst, 243 src1: from_reg.try_into().unwrap(), 244 src2: dst.to_reg(), 245 } 246 .into() 247 ] 248 } 249 250 fn gen_stack_lower_bound_trap(_limit_reg: Reg) -> SmallInstVec<Self::I> { 251 unimplemented!("pulley shouldn't need stack bound checks") 252 } 253 254 fn gen_get_stack_addr(mem: StackAMode, dst: Writable<Reg>) -> Self::I { 255 let dst = dst.to_reg(); 256 let dst = XReg::new(dst).unwrap(); 257 let dst = WritableXReg::from_reg(dst); 258 let mem = mem.into(); 259 Inst::LoadAddr { dst, mem }.into() 260 } 261 262 fn gen_load_base_offset(into_reg: Writable<Reg>, base: Reg, offset: i32, ty: Type) -> Self::I { 263 let base = XReg::try_from(base).unwrap(); 264 let mem = Amode::RegOffset { base, offset }; 265 Inst::gen_load(into_reg, mem, ty, MemFlags::trusted()).into() 266 } 267 268 fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Self::I { 269 let base = XReg::try_from(base).unwrap(); 270 let mem = Amode::RegOffset { base, offset }; 271 Inst::gen_store(mem, from_reg, ty, MemFlags::trusted()).into() 272 } 273 274 fn gen_sp_reg_adjust(amount: i32) -> SmallInstVec<Self::I> { 275 if amount == 0 { 276 return smallvec![]; 277 } 278 279 let inst = if amount < 0 { 280 let amount = amount.checked_neg().unwrap(); 281 if let Ok(amt) = u32::try_from(amount) { 282 RawInst::StackAlloc32 { amt } 283 } else { 284 unreachable!() 285 } 286 } else { 287 if let Ok(amt) = u32::try_from(amount) { 288 RawInst::StackFree32 { amt } 289 } else { 290 unreachable!() 291 } 292 }; 293 smallvec![inst.into()] 294 } 295 296 /// Generates the entire prologue for the function. 297 /// 298 /// Note that this is different from other backends where it's not spread 299 /// out among a few individual functions. That's because the goal here is to 300 /// generate a single macro-instruction for the entire prologue in the most 301 /// common cases and we don't want to spread the logic over multiple 302 /// functions. 303 /// 304 /// The general machinst methods are split to accommodate stack checks and 305 /// things like stack probes, all of which are empty on Pulley because 306 /// Pulley has its own stack check mechanism. 307 fn gen_prologue_frame_setup( 308 _call_conv: isa::CallConv, 309 _flags: &settings::Flags, 310 _isa_flags: &PulleyFlags, 311 frame_layout: &FrameLayout, 312 ) -> SmallInstVec<Self::I> { 313 let mut insts = SmallVec::new(); 314 315 let incoming_args_diff = frame_layout.tail_args_size - frame_layout.incoming_args_size; 316 if incoming_args_diff > 0 { 317 // Decrement SP by the amount of additional incoming argument space 318 // we need 319 insts.extend(Self::gen_sp_reg_adjust(-(incoming_args_diff as i32))); 320 } 321 322 let style = frame_layout.pulley_frame_style(); 323 324 match &style { 325 FrameStyle::None => {} 326 FrameStyle::PulleyBasicSetup { frame_size } => { 327 insts.push(RawInst::PushFrame.into()); 328 insts.extend(Self::gen_sp_reg_adjust( 329 -i32::try_from(*frame_size).unwrap(), 330 )); 331 } 332 FrameStyle::PulleySetupAndSaveClobbers { 333 frame_size, 334 saved_by_pulley, 335 } => insts.push( 336 RawInst::PushFrameSave { 337 amt: *frame_size, 338 regs: pulley_interpreter::UpperRegSet::from_bitset(*saved_by_pulley), 339 } 340 .into(), 341 ), 342 FrameStyle::Manual { frame_size } => insts.extend(Self::gen_sp_reg_adjust( 343 -i32::try_from(*frame_size).unwrap(), 344 )), 345 } 346 347 for (offset, ty, reg) in frame_layout.manually_managed_clobbers(&style) { 348 let mut flags = MemFlags::trusted(); 349 if ty.is_vector() { 350 flags.set_endianness(ir::Endianness::Little); 351 } 352 insts.push(Inst::gen_store(Amode::SpOffset { offset }, reg, ty, flags).into()); 353 } 354 355 insts 356 } 357 358 /// Reverse of `gen_prologue_frame_setup`. 359 fn gen_epilogue_frame_restore( 360 _call_conv: isa::CallConv, 361 _flags: &settings::Flags, 362 _isa_flags: &PulleyFlags, 363 frame_layout: &FrameLayout, 364 ) -> SmallInstVec<Self::I> { 365 let mut insts = SmallVec::new(); 366 367 let style = frame_layout.pulley_frame_style(); 368 369 // Restore clobbered registers that are manually managed in Cranelift. 370 for (offset, ty, reg) in frame_layout.manually_managed_clobbers(&style) { 371 let mut flags = MemFlags::trusted(); 372 if ty.is_vector() { 373 flags.set_endianness(ir::Endianness::Little); 374 } 375 insts.push( 376 Inst::gen_load( 377 Writable::from_reg(reg), 378 Amode::SpOffset { offset }, 379 ty, 380 flags, 381 ) 382 .into(), 383 ); 384 } 385 386 // Perform the inverse of `gen_prologue_frame_setup`. 387 match &style { 388 FrameStyle::None => {} 389 FrameStyle::PulleyBasicSetup { frame_size } => { 390 insts.extend(Self::gen_sp_reg_adjust(i32::try_from(*frame_size).unwrap())); 391 insts.push(RawInst::PopFrame.into()); 392 } 393 FrameStyle::PulleySetupAndSaveClobbers { 394 frame_size, 395 saved_by_pulley, 396 } => insts.push( 397 RawInst::PopFrameRestore { 398 amt: *frame_size, 399 regs: pulley_interpreter::UpperRegSet::from_bitset(*saved_by_pulley), 400 } 401 .into(), 402 ), 403 FrameStyle::Manual { frame_size } => { 404 insts.extend(Self::gen_sp_reg_adjust(i32::try_from(*frame_size).unwrap())) 405 } 406 } 407 408 insts 409 } 410 411 fn gen_return( 412 call_conv: isa::CallConv, 413 _isa_flags: &PulleyFlags, 414 frame_layout: &FrameLayout, 415 ) -> SmallInstVec<Self::I> { 416 let mut insts = SmallVec::new(); 417 418 // Handle final stack adjustments for the tail-call ABI. 419 if call_conv == isa::CallConv::Tail && frame_layout.tail_args_size > 0 { 420 insts.extend(Self::gen_sp_reg_adjust( 421 frame_layout.tail_args_size.try_into().unwrap(), 422 )); 423 } 424 insts.push(RawInst::Ret {}.into()); 425 426 insts 427 } 428 429 fn gen_probestack(_insts: &mut SmallInstVec<Self::I>, _frame_size: u32) { 430 // Pulley doesn't implement stack probes since all stack pointer 431 // decrements are checked already. 432 } 433 434 fn gen_clobber_save( 435 _call_conv: isa::CallConv, 436 _flags: &settings::Flags, 437 _frame_layout: &FrameLayout, 438 ) -> SmallVec<[Self::I; 16]> { 439 // Note that this is intentionally empty because everything necessary 440 // was already done in `gen_prologue_frame_setup`. 441 SmallVec::new() 442 } 443 444 fn gen_clobber_restore( 445 _call_conv: isa::CallConv, 446 _flags: &settings::Flags, 447 _frame_layout: &FrameLayout, 448 ) -> SmallVec<[Self::I; 16]> { 449 // Intentionally empty as restores happen for Pulley in `gen_return`. 450 SmallVec::new() 451 } 452 453 fn gen_memcpy<F: FnMut(Type) -> Writable<Reg>>( 454 _call_conv: isa::CallConv, 455 _dst: Reg, 456 _src: Reg, 457 _size: usize, 458 _alloc_tmp: F, 459 ) -> SmallVec<[Self::I; 8]> { 460 todo!() 461 } 462 463 fn get_number_of_spillslots_for_value( 464 rc: RegClass, 465 _target_vector_bytes: u32, 466 _isa_flags: &PulleyFlags, 467 ) -> u32 { 468 // Spill slots are the size of a "word" or a pointer, but Pulley 469 // registers are 8-byte for integers/floats regardless of pointer size. 470 // Calculate the number of slots necessary to store 8 bytes. 471 let slots_for_8bytes = match P::pointer_width() { 472 PointerWidth::PointerWidth32 => 2, 473 PointerWidth::PointerWidth64 => 1, 474 }; 475 match rc { 476 // Int/float registers are 8-bytes 477 RegClass::Int | RegClass::Float => slots_for_8bytes, 478 // Vector registers are 16 bytes 479 RegClass::Vector => 2 * slots_for_8bytes, 480 } 481 } 482 483 fn get_machine_env(_flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { 484 static MACHINE_ENV: OnceLock<MachineEnv> = OnceLock::new(); 485 MACHINE_ENV.get_or_init(create_reg_environment) 486 } 487 488 fn get_regs_clobbered_by_call( 489 call_conv_of_callee: isa::CallConv, 490 is_exception: bool, 491 ) -> PRegSet { 492 if is_exception { 493 ALL_CLOBBERS 494 } else if call_conv_of_callee == isa::CallConv::PreserveAll { 495 NO_CLOBBERS 496 } else { 497 DEFAULT_CLOBBERS 498 } 499 } 500 501 fn compute_frame_layout( 502 call_conv: isa::CallConv, 503 flags: &settings::Flags, 504 _sig: &Signature, 505 regs: &[Writable<RealReg>], 506 function_calls: FunctionCalls, 507 incoming_args_size: u32, 508 tail_args_size: u32, 509 stackslots_size: u32, 510 fixed_frame_storage_size: u32, 511 outgoing_args_size: u32, 512 ) -> FrameLayout { 513 let is_callee_save = |reg: &Writable<RealReg>| match call_conv { 514 isa::CallConv::PreserveAll => true, 515 _ => DEFAULT_CALLEE_SAVES.contains(reg.to_reg().into()), 516 }; 517 let mut regs: Vec<Writable<RealReg>> = 518 regs.iter().cloned().filter(is_callee_save).collect(); 519 520 regs.sort_unstable(); 521 522 // Compute clobber size. 523 let clobber_size = compute_clobber_size(®s); 524 525 // Compute linkage frame size. 526 let setup_area_size = if flags.preserve_frame_pointers() 527 || function_calls != FunctionCalls::None 528 // The function arguments that are passed on the stack are addressed 529 // relative to the Frame Pointer. 530 || incoming_args_size > 0 531 || clobber_size > 0 532 || fixed_frame_storage_size > 0 533 { 534 P::pointer_width().bytes() * 2 // FP, LR 535 } else { 536 0 537 }; 538 539 FrameLayout { 540 word_bytes: u32::from(P::pointer_width().bytes()), 541 incoming_args_size, 542 tail_args_size, 543 setup_area_size: setup_area_size.into(), 544 clobber_size, 545 fixed_frame_storage_size, 546 stackslots_size, 547 outgoing_args_size, 548 clobbered_callee_saves: regs, 549 function_calls, 550 } 551 } 552 553 fn gen_inline_probestack( 554 _insts: &mut SmallInstVec<Self::I>, 555 _call_conv: isa::CallConv, 556 _frame_size: u32, 557 _guard_size: u32, 558 ) { 559 // Pulley doesn't need inline probestacks because it always checks stack 560 // decrements. 561 } 562 563 fn retval_temp_reg(_call_conv_of_callee: isa::CallConv) -> Writable<Reg> { 564 // Use x15 as a temp if needed: clobbered, not a 565 // retval. 566 Writable::from_reg(regs::x_reg(15)) 567 } 568 569 fn exception_payload_regs(call_conv: isa::CallConv) -> &'static [Reg] { 570 const PAYLOAD_REGS: &'static [Reg] = &[ 571 Reg::from_real_reg(regs::px_reg(0)), 572 Reg::from_real_reg(regs::px_reg(1)), 573 ]; 574 match call_conv { 575 isa::CallConv::SystemV | isa::CallConv::Tail | isa::CallConv::PreserveAll => { 576 PAYLOAD_REGS 577 } 578 isa::CallConv::Fast 579 | isa::CallConv::WindowsFastcall 580 | isa::CallConv::AppleAarch64 581 | isa::CallConv::Probestack 582 | isa::CallConv::Winch => &[], 583 } 584 } 585 } 586 587 /// Different styles of management of fp/lr and clobbered registers. 588 /// 589 /// This helps decide, depending on Cranelift settings and frame layout, what 590 /// macro instruction is used to setup the pulley frame. 591 enum FrameStyle { 592 /// No management is happening, fp/lr aren't saved by Pulley or Cranelift. 593 /// No stack is being allocated either. 594 None, 595 596 /// Pulley saves the fp/lr combo and then stack adjustments/clobbers are 597 /// handled manually. 598 PulleyBasicSetup { frame_size: u32 }, 599 600 /// Pulley is managing the fp/lr combo, the stack size, and clobbered 601 /// X-class registers. 602 /// 603 /// Note that `saved_by_pulley` is not the exhaustive set of clobbered 604 /// registers. It's only those that are part of the `PushFrameSave` 605 /// instruction. 606 PulleySetupAndSaveClobbers { 607 /// The size of the frame, including clobbers, that's being allocated. 608 frame_size: u16, 609 /// Registers that pulley is saving/restoring. 610 saved_by_pulley: ScalarBitSet<u16>, 611 }, 612 613 /// Cranelift is manually managing everything, both clobbers and stack 614 /// increments/decrements. 615 /// 616 /// Note that fp/lr are not saved in this mode. 617 Manual { 618 /// The size of the stack being allocated. 619 frame_size: u32, 620 }, 621 } 622 623 /// Pulley-specific helpers when dealing with ABI code. 624 impl FrameLayout { 625 /// Whether or not this frame saves fp/lr. 626 fn setup_frame(&self) -> bool { 627 self.setup_area_size > 0 628 } 629 630 /// Returns the stack size allocated by this function, excluding incoming 631 /// tail args or the optional "setup area" of fp/lr. 632 fn stack_size(&self) -> u32 { 633 self.clobber_size + self.fixed_frame_storage_size + self.outgoing_args_size 634 } 635 636 /// Returns the style of frame being used for this function. 637 /// 638 /// See `FrameStyle` for more information. 639 fn pulley_frame_style(&self) -> FrameStyle { 640 let saved_by_pulley = self.clobbered_xregs_saved_by_pulley(); 641 match ( 642 self.stack_size(), 643 self.setup_frame(), 644 saved_by_pulley.is_empty(), 645 ) { 646 // No stack allocated, not saving fp/lr, no clobbers, nothing to do 647 (0, false, true) => FrameStyle::None, 648 649 // No stack allocated, saving fp/lr, no clobbers, so this is 650 // pulley-managed via push/pop_frame. 651 (0, true, true) => FrameStyle::PulleyBasicSetup { frame_size: 0 }, 652 653 // Some stack is being allocated and pulley is managing fp/lr. Let 654 // pulley manage clobbered registers as well, regardless if they're 655 // present or not. 656 // 657 // If the stack is too large then `PulleyBasicSetup` is used 658 // otherwise we'll be pushing `PushFrameSave` and `PopFrameRestore`. 659 (frame_size, true, _) => match frame_size.try_into() { 660 Ok(frame_size) => FrameStyle::PulleySetupAndSaveClobbers { 661 frame_size, 662 saved_by_pulley, 663 }, 664 Err(_) => FrameStyle::PulleyBasicSetup { frame_size }, 665 }, 666 667 // Some stack is being allocated, but pulley isn't managing fp/lr, 668 // so we're manually doing everything. 669 (frame_size, false, true) => FrameStyle::Manual { frame_size }, 670 671 // If there's no frame setup and there's clobbered registers this 672 // technically should have already hit a case above, so panic here. 673 (_, false, false) => unreachable!(), 674 } 675 } 676 677 /// Returns the set of clobbered registers that Pulley is managing via its 678 /// macro instructions rather than the generated code. 679 fn clobbered_xregs_saved_by_pulley(&self) -> ScalarBitSet<u16> { 680 let mut clobbered: ScalarBitSet<u16> = ScalarBitSet::new(); 681 // Pulley only manages clobbers if it's also managing fp/lr. 682 if !self.setup_frame() { 683 return clobbered; 684 } 685 let mut found_manual_clobber = false; 686 for reg in self.clobbered_callee_saves.iter() { 687 let r_reg = reg.to_reg(); 688 // Pulley can only manage clobbers of integer registers at this 689 // time, float registers are managed manually. 690 // 691 // Also assert that all pulley-managed clobbers come first, 692 // otherwise the loop below in `manually_managed_clobbers` is 693 // incorrect. 694 if r_reg.class() == RegClass::Int { 695 assert!(!found_manual_clobber); 696 if let Some(offset) = r_reg.hw_enc().checked_sub(16) { 697 clobbered.insert(offset); 698 } 699 } else { 700 found_manual_clobber = true; 701 } 702 } 703 clobbered 704 } 705 706 /// Returns an iterator over the clobbers that Cranelift is managing, not 707 /// Pulley. 708 /// 709 /// If this frame has clobbers then they're either saved by Pulley with 710 /// `FrameStyle::PulleySetupAndSaveClobbers`. Cranelift might need to manage 711 /// these registers depending on Cranelift settings. Cranelift also always 712 /// manages floating-point registers. 713 fn manually_managed_clobbers<'a>( 714 &'a self, 715 style: &'a FrameStyle, 716 ) -> impl Iterator<Item = (i32, Type, Reg)> + 'a { 717 let mut offset = self.stack_size(); 718 self.clobbered_callee_saves.iter().filter_map(move |reg| { 719 // Allocate space for this clobber no matter what. If pulley is 720 // managing this then we're just accounting for the pulley-saved 721 // registers as well. Note that all pulley-managed registers come 722 // first in the list here. 723 offset -= 8; 724 let r_reg = reg.to_reg(); 725 let ty = match r_reg.class() { 726 RegClass::Int => { 727 // If this register is saved by pulley, skip this clobber. 728 if let FrameStyle::PulleySetupAndSaveClobbers { 729 saved_by_pulley, .. 730 } = style 731 { 732 if let Some(reg) = r_reg.hw_enc().checked_sub(16) { 733 if saved_by_pulley.contains(reg) { 734 return None; 735 } 736 } 737 } 738 I64 739 } 740 RegClass::Float => F64, 741 RegClass::Vector => I8X16, 742 }; 743 let offset = i32::try_from(offset).unwrap(); 744 Some((offset, ty, Reg::from(reg.to_reg()))) 745 }) 746 } 747 } 748 749 const DEFAULT_CALLEE_SAVES: PRegSet = PRegSet::empty() 750 // Integer registers. 751 .with(px_reg(16)) 752 .with(px_reg(17)) 753 .with(px_reg(18)) 754 .with(px_reg(19)) 755 .with(px_reg(20)) 756 .with(px_reg(21)) 757 .with(px_reg(22)) 758 .with(px_reg(23)) 759 .with(px_reg(24)) 760 .with(px_reg(25)) 761 .with(px_reg(26)) 762 .with(px_reg(27)) 763 .with(px_reg(28)) 764 .with(px_reg(29)) 765 .with(px_reg(30)) 766 .with(px_reg(31)) 767 // Note: no float/vector registers are callee-saved. 768 ; 769 770 fn compute_clobber_size(clobbers: &[Writable<RealReg>]) -> u32 { 771 let mut clobbered_size = 0; 772 for reg in clobbers { 773 match reg.to_reg().class() { 774 RegClass::Int => { 775 clobbered_size += 8; 776 } 777 RegClass::Float => { 778 clobbered_size += 8; 779 } 780 RegClass::Vector => { 781 // No alignment concerns: the Pulley virtual CPU 782 // supports unaligned vector load/stores. 783 clobbered_size += 16; 784 } 785 } 786 } 787 align_to(clobbered_size, 16) 788 } 789 790 const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() 791 // Integer registers: the first 16 get clobbered. 792 .with(px_reg(0)) 793 .with(px_reg(1)) 794 .with(px_reg(2)) 795 .with(px_reg(3)) 796 .with(px_reg(4)) 797 .with(px_reg(5)) 798 .with(px_reg(6)) 799 .with(px_reg(7)) 800 .with(px_reg(8)) 801 .with(px_reg(9)) 802 .with(px_reg(10)) 803 .with(px_reg(11)) 804 .with(px_reg(12)) 805 .with(px_reg(13)) 806 .with(px_reg(14)) 807 .with(px_reg(15)) 808 // All float registers get clobbered. 809 .with(pf_reg(0)) 810 .with(pf_reg(1)) 811 .with(pf_reg(2)) 812 .with(pf_reg(3)) 813 .with(pf_reg(4)) 814 .with(pf_reg(5)) 815 .with(pf_reg(6)) 816 .with(pf_reg(7)) 817 .with(pf_reg(8)) 818 .with(pf_reg(9)) 819 .with(pf_reg(10)) 820 .with(pf_reg(11)) 821 .with(pf_reg(12)) 822 .with(pf_reg(13)) 823 .with(pf_reg(14)) 824 .with(pf_reg(15)) 825 .with(pf_reg(16)) 826 .with(pf_reg(17)) 827 .with(pf_reg(18)) 828 .with(pf_reg(19)) 829 .with(pf_reg(20)) 830 .with(pf_reg(21)) 831 .with(pf_reg(22)) 832 .with(pf_reg(23)) 833 .with(pf_reg(24)) 834 .with(pf_reg(25)) 835 .with(pf_reg(26)) 836 .with(pf_reg(27)) 837 .with(pf_reg(28)) 838 .with(pf_reg(29)) 839 .with(pf_reg(30)) 840 .with(pf_reg(31)) 841 // All vector registers get clobbered. 842 .with(pv_reg(0)) 843 .with(pv_reg(1)) 844 .with(pv_reg(2)) 845 .with(pv_reg(3)) 846 .with(pv_reg(4)) 847 .with(pv_reg(5)) 848 .with(pv_reg(6)) 849 .with(pv_reg(7)) 850 .with(pv_reg(8)) 851 .with(pv_reg(9)) 852 .with(pv_reg(10)) 853 .with(pv_reg(11)) 854 .with(pv_reg(12)) 855 .with(pv_reg(13)) 856 .with(pv_reg(14)) 857 .with(pv_reg(15)) 858 .with(pv_reg(16)) 859 .with(pv_reg(17)) 860 .with(pv_reg(18)) 861 .with(pv_reg(19)) 862 .with(pv_reg(20)) 863 .with(pv_reg(21)) 864 .with(pv_reg(22)) 865 .with(pv_reg(23)) 866 .with(pv_reg(24)) 867 .with(pv_reg(25)) 868 .with(pv_reg(26)) 869 .with(pv_reg(27)) 870 .with(pv_reg(28)) 871 .with(pv_reg(29)) 872 .with(pv_reg(30)) 873 .with(pv_reg(31)); 874 875 const ALL_CLOBBERS: PRegSet = PRegSet::empty() 876 .with(px_reg(0)) 877 .with(px_reg(1)) 878 .with(px_reg(2)) 879 .with(px_reg(3)) 880 .with(px_reg(4)) 881 .with(px_reg(5)) 882 .with(px_reg(6)) 883 .with(px_reg(7)) 884 .with(px_reg(8)) 885 .with(px_reg(9)) 886 .with(px_reg(10)) 887 .with(px_reg(11)) 888 .with(px_reg(12)) 889 .with(px_reg(13)) 890 .with(px_reg(14)) 891 .with(px_reg(15)) 892 .with(px_reg(16)) 893 .with(px_reg(17)) 894 .with(px_reg(18)) 895 .with(px_reg(19)) 896 .with(px_reg(20)) 897 .with(px_reg(21)) 898 .with(px_reg(22)) 899 .with(px_reg(23)) 900 .with(px_reg(24)) 901 .with(px_reg(25)) 902 .with(px_reg(26)) 903 .with(px_reg(27)) 904 .with(px_reg(28)) 905 .with(px_reg(29)) 906 .with(px_reg(30)) 907 .with(px_reg(31)) 908 .with(pf_reg(0)) 909 .with(pf_reg(1)) 910 .with(pf_reg(2)) 911 .with(pf_reg(3)) 912 .with(pf_reg(4)) 913 .with(pf_reg(5)) 914 .with(pf_reg(6)) 915 .with(pf_reg(7)) 916 .with(pf_reg(8)) 917 .with(pf_reg(9)) 918 .with(pf_reg(10)) 919 .with(pf_reg(11)) 920 .with(pf_reg(12)) 921 .with(pf_reg(13)) 922 .with(pf_reg(14)) 923 .with(pf_reg(15)) 924 .with(pf_reg(16)) 925 .with(pf_reg(17)) 926 .with(pf_reg(18)) 927 .with(pf_reg(19)) 928 .with(pf_reg(20)) 929 .with(pf_reg(21)) 930 .with(pf_reg(22)) 931 .with(pf_reg(23)) 932 .with(pf_reg(24)) 933 .with(pf_reg(25)) 934 .with(pf_reg(26)) 935 .with(pf_reg(27)) 936 .with(pf_reg(28)) 937 .with(pf_reg(29)) 938 .with(pf_reg(30)) 939 .with(pf_reg(31)) 940 .with(pv_reg(0)) 941 .with(pv_reg(1)) 942 .with(pv_reg(2)) 943 .with(pv_reg(3)) 944 .with(pv_reg(4)) 945 .with(pv_reg(5)) 946 .with(pv_reg(6)) 947 .with(pv_reg(7)) 948 .with(pv_reg(8)) 949 .with(pv_reg(9)) 950 .with(pv_reg(10)) 951 .with(pv_reg(11)) 952 .with(pv_reg(12)) 953 .with(pv_reg(13)) 954 .with(pv_reg(14)) 955 .with(pv_reg(15)) 956 .with(pv_reg(16)) 957 .with(pv_reg(17)) 958 .with(pv_reg(18)) 959 .with(pv_reg(19)) 960 .with(pv_reg(20)) 961 .with(pv_reg(21)) 962 .with(pv_reg(22)) 963 .with(pv_reg(23)) 964 .with(pv_reg(24)) 965 .with(pv_reg(25)) 966 .with(pv_reg(26)) 967 .with(pv_reg(27)) 968 .with(pv_reg(28)) 969 .with(pv_reg(29)) 970 .with(pv_reg(30)) 971 .with(pv_reg(31)); 972 973 const NO_CLOBBERS: PRegSet = PRegSet::empty(); 974 975 fn create_reg_environment() -> MachineEnv { 976 // Prefer caller-saved registers over callee-saved registers, because that 977 // way we don't need to emit code to save and restore them if we don't 978 // mutate them. 979 980 let preferred_regs_by_class: [Vec<PReg>; 3] = { 981 let x_registers: Vec<PReg> = (0..16).map(|x| px_reg(x)).collect(); 982 let f_registers: Vec<PReg> = (0..32).map(|x| pf_reg(x)).collect(); 983 let v_registers: Vec<PReg> = (0..32).map(|x| pv_reg(x)).collect(); 984 [x_registers, f_registers, v_registers] 985 }; 986 987 let non_preferred_regs_by_class: [Vec<PReg>; 3] = { 988 let x_registers: Vec<PReg> = (16..XReg::SPECIAL_START) 989 .map(|x| px_reg(x.into())) 990 .collect(); 991 let f_registers: Vec<PReg> = vec![]; 992 let v_registers: Vec<PReg> = vec![]; 993 [x_registers, f_registers, v_registers] 994 }; 995 996 MachineEnv { 997 preferred_regs_by_class, 998 non_preferred_regs_by_class, 999 fixed_stack_slots: vec![], 1000 scratch_by_class: [None, None, None], 1001 } 1002 } 1003