1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI implementation of the TargetRegisterInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIRegisterInfo.h" 15 #include "AMDGPURegisterBankInfo.h" 16 #include "AMDGPUSubtarget.h" 17 #include "SIInstrInfo.h" 18 #include "SIMachineFunctionInfo.h" 19 #include "MCTargetDesc/AMDGPUInstPrinter.h" 20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include "llvm/CodeGen/MachineDominators.h" 23 #include "llvm/CodeGen/MachineFrameInfo.h" 24 #include "llvm/CodeGen/MachineInstrBuilder.h" 25 #include "llvm/CodeGen/RegisterScavenging.h" 26 #include "llvm/CodeGen/SlotIndexes.h" 27 #include "llvm/IR/Function.h" 28 #include "llvm/IR/LLVMContext.h" 29 30 using namespace llvm; 31 32 #define GET_REGINFO_TARGET_DESC 33 #include "AMDGPUGenRegisterInfo.inc" 34 35 static cl::opt<bool> EnableSpillSGPRToVGPR( 36 "amdgpu-spill-sgpr-to-vgpr", 37 cl::desc("Enable spilling VGPRs to SGPRs"), 38 cl::ReallyHidden, 39 cl::init(true)); 40 41 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) 42 : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST), 43 SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) {} 44 45 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, 46 unsigned Reg) const { 47 MCRegAliasIterator R(Reg, this, true); 48 49 for (; R.isValid(); ++R) 50 Reserved.set(*R); 51 } 52 53 // Forced to be here by one .inc 54 const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs( 55 const MachineFunction *MF) const { 56 CallingConv::ID CC = MF->getFunction().getCallingConv(); 57 switch (CC) { 58 case CallingConv::C: 59 case CallingConv::Fast: 60 case CallingConv::Cold: 61 return CSR_AMDGPU_HighRegs_SaveList; 62 default: { 63 // Dummy to not crash RegisterClassInfo. 64 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister; 65 return &NoCalleeSavedReg; 66 } 67 } 68 } 69 70 const MCPhysReg * 71 SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const { 72 return nullptr; 73 } 74 75 const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF, 76 CallingConv::ID CC) const { 77 switch (CC) { 78 case CallingConv::C: 79 case CallingConv::Fast: 80 case CallingConv::Cold: 81 return CSR_AMDGPU_HighRegs_RegMask; 82 default: 83 return nullptr; 84 } 85 } 86 87 Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const { 88 const SIFrameLowering *TFI = 89 MF.getSubtarget<GCNSubtarget>().getFrameLowering(); 90 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); 91 // During ISel lowering we always reserve the stack pointer in entry 92 // functions, but never actually want to reference it when accessing our own 93 // frame. If we need a frame pointer we use it, but otherwise we can just use 94 // an immediate "0" which we represent by returning NoRegister. 95 if (FuncInfo->isEntryFunction()) { 96 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register(); 97 } 98 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() 99 : FuncInfo->getStackPtrOffsetReg(); 100 } 101 102 const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const { 103 return CSR_AMDGPU_AllVGPRs_RegMask; 104 } 105 106 const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const { 107 return CSR_AMDGPU_AllAllocatableSRegs_RegMask; 108 } 109 110 // FIXME: TableGen should generate something to make this manageable for all 111 // register classes. At a minimum we could use the opposite of 112 // composeSubRegIndices and go up from the base 32-bit subreg. 113 unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel, 114 unsigned NumRegs) { 115 // Table of NumRegs sized pieces at every 32-bit offset. 116 static const uint16_t SubRegFromChannelTable[][32] = { 117 {AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 118 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 119 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 120 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 121 AMDGPU::sub16, AMDGPU::sub17, AMDGPU::sub18, AMDGPU::sub19, 122 AMDGPU::sub20, AMDGPU::sub21, AMDGPU::sub22, AMDGPU::sub23, 123 AMDGPU::sub24, AMDGPU::sub25, AMDGPU::sub26, AMDGPU::sub27, 124 AMDGPU::sub28, AMDGPU::sub29, AMDGPU::sub30, AMDGPU::sub31}, 125 {AMDGPU::sub0_sub1, AMDGPU::sub1_sub2, AMDGPU::sub2_sub3, 126 AMDGPU::sub3_sub4, AMDGPU::sub4_sub5, AMDGPU::sub5_sub6, 127 AMDGPU::sub6_sub7, AMDGPU::sub7_sub8, AMDGPU::sub8_sub9, 128 AMDGPU::sub9_sub10, AMDGPU::sub10_sub11, AMDGPU::sub11_sub12, 129 AMDGPU::sub12_sub13, AMDGPU::sub13_sub14, AMDGPU::sub14_sub15, 130 AMDGPU::sub15_sub16, AMDGPU::sub16_sub17, AMDGPU::sub17_sub18, 131 AMDGPU::sub18_sub19, AMDGPU::sub19_sub20, AMDGPU::sub20_sub21, 132 AMDGPU::sub21_sub22, AMDGPU::sub22_sub23, AMDGPU::sub23_sub24, 133 AMDGPU::sub24_sub25, AMDGPU::sub25_sub26, AMDGPU::sub26_sub27, 134 AMDGPU::sub27_sub28, AMDGPU::sub28_sub29, AMDGPU::sub29_sub30, 135 AMDGPU::sub30_sub31, AMDGPU::NoSubRegister}, 136 {AMDGPU::sub0_sub1_sub2, AMDGPU::sub1_sub2_sub3, 137 AMDGPU::sub2_sub3_sub4, AMDGPU::sub3_sub4_sub5, 138 AMDGPU::sub4_sub5_sub6, AMDGPU::sub5_sub6_sub7, 139 AMDGPU::sub6_sub7_sub8, AMDGPU::sub7_sub8_sub9, 140 AMDGPU::sub8_sub9_sub10, AMDGPU::sub9_sub10_sub11, 141 AMDGPU::sub10_sub11_sub12, AMDGPU::sub11_sub12_sub13, 142 AMDGPU::sub12_sub13_sub14, AMDGPU::sub13_sub14_sub15, 143 AMDGPU::sub14_sub15_sub16, AMDGPU::sub15_sub16_sub17, 144 AMDGPU::sub16_sub17_sub18, AMDGPU::sub17_sub18_sub19, 145 AMDGPU::sub18_sub19_sub20, AMDGPU::sub19_sub20_sub21, 146 AMDGPU::sub20_sub21_sub22, AMDGPU::sub21_sub22_sub23, 147 AMDGPU::sub22_sub23_sub24, AMDGPU::sub23_sub24_sub25, 148 AMDGPU::sub24_sub25_sub26, AMDGPU::sub25_sub26_sub27, 149 AMDGPU::sub26_sub27_sub28, AMDGPU::sub27_sub28_sub29, 150 AMDGPU::sub28_sub29_sub30, AMDGPU::sub29_sub30_sub31, 151 AMDGPU::NoSubRegister, AMDGPU::NoSubRegister}, 152 {AMDGPU::sub0_sub1_sub2_sub3, AMDGPU::sub1_sub2_sub3_sub4, 153 AMDGPU::sub2_sub3_sub4_sub5, AMDGPU::sub3_sub4_sub5_sub6, 154 AMDGPU::sub4_sub5_sub6_sub7, AMDGPU::sub5_sub6_sub7_sub8, 155 AMDGPU::sub6_sub7_sub8_sub9, AMDGPU::sub7_sub8_sub9_sub10, 156 AMDGPU::sub8_sub9_sub10_sub11, AMDGPU::sub9_sub10_sub11_sub12, 157 AMDGPU::sub10_sub11_sub12_sub13, AMDGPU::sub11_sub12_sub13_sub14, 158 AMDGPU::sub12_sub13_sub14_sub15, AMDGPU::sub13_sub14_sub15_sub16, 159 AMDGPU::sub14_sub15_sub16_sub17, AMDGPU::sub15_sub16_sub17_sub18, 160 AMDGPU::sub16_sub17_sub18_sub19, AMDGPU::sub17_sub18_sub19_sub20, 161 AMDGPU::sub18_sub19_sub20_sub21, AMDGPU::sub19_sub20_sub21_sub22, 162 AMDGPU::sub20_sub21_sub22_sub23, AMDGPU::sub21_sub22_sub23_sub24, 163 AMDGPU::sub22_sub23_sub24_sub25, AMDGPU::sub23_sub24_sub25_sub26, 164 AMDGPU::sub24_sub25_sub26_sub27, AMDGPU::sub25_sub26_sub27_sub28, 165 AMDGPU::sub26_sub27_sub28_sub29, AMDGPU::sub27_sub28_sub29_sub30, 166 AMDGPU::sub28_sub29_sub30_sub31, AMDGPU::NoSubRegister, 167 AMDGPU::NoSubRegister, AMDGPU::NoSubRegister}}; 168 169 const unsigned NumRegIndex = NumRegs - 1; 170 171 assert(NumRegIndex < array_lengthof(SubRegFromChannelTable) && 172 "Not implemented"); 173 assert(Channel < array_lengthof(SubRegFromChannelTable[0])); 174 return SubRegFromChannelTable[NumRegIndex][Channel]; 175 } 176 177 unsigned SIRegisterInfo::reservedPrivateSegmentBufferReg( 178 const MachineFunction &MF) const { 179 unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4; 180 unsigned BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); 181 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass); 182 } 183 184 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 185 BitVector Reserved(getNumRegs()); 186 187 // EXEC_LO and EXEC_HI could be allocated and used as regular register, but 188 // this seems likely to result in bugs, so I'm marking them as reserved. 189 reserveRegisterTuples(Reserved, AMDGPU::EXEC); 190 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); 191 192 // M0 has to be reserved so that llvm accepts it as a live-in into a block. 193 reserveRegisterTuples(Reserved, AMDGPU::M0); 194 195 // Reserve src_vccz, src_execz, src_scc. 196 reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ); 197 reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ); 198 reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC); 199 200 // Reserve the memory aperture registers. 201 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); 202 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); 203 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); 204 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); 205 206 // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen. 207 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); 208 209 // Reserve xnack_mask registers - support is not implemented in Codegen. 210 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); 211 212 // Reserve lds_direct register - support is not implemented in Codegen. 213 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); 214 215 // Reserve Trap Handler registers - support is not implemented in Codegen. 216 reserveRegisterTuples(Reserved, AMDGPU::TBA); 217 reserveRegisterTuples(Reserved, AMDGPU::TMA); 218 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); 219 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); 220 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); 221 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); 222 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); 223 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); 224 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); 225 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); 226 227 // Reserve null register - it shall never be allocated 228 reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL); 229 230 // Disallow vcc_hi allocation in wave32. It may be allocated but most likely 231 // will result in bugs. 232 if (isWave32) { 233 Reserved.set(AMDGPU::VCC); 234 Reserved.set(AMDGPU::VCC_HI); 235 } 236 237 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); 238 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 239 for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { 240 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 241 reserveRegisterTuples(Reserved, Reg); 242 } 243 244 unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF); 245 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); 246 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) { 247 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); 248 reserveRegisterTuples(Reserved, Reg); 249 Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 250 reserveRegisterTuples(Reserved, Reg); 251 } 252 253 // Reserve all the rest AGPRs if there are no instructions to use it. 254 if (!ST.hasMAIInsts()) { 255 for (unsigned i = 0; i < MaxNumVGPRs; ++i) { 256 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 257 reserveRegisterTuples(Reserved, Reg); 258 } 259 } 260 261 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 262 263 unsigned ScratchRSrcReg = MFI->getScratchRSrcReg(); 264 if (ScratchRSrcReg != AMDGPU::NoRegister) { 265 // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need 266 // to spill. 267 // TODO: May need to reserve a VGPR if doing LDS spilling. 268 reserveRegisterTuples(Reserved, ScratchRSrcReg); 269 } 270 271 // We have to assume the SP is needed in case there are calls in the function, 272 // which is detected after the function is lowered. If we aren't really going 273 // to need SP, don't bother reserving it. 274 unsigned StackPtrReg = MFI->getStackPtrOffsetReg(); 275 276 if (StackPtrReg != AMDGPU::NoRegister) { 277 reserveRegisterTuples(Reserved, StackPtrReg); 278 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg)); 279 } 280 281 unsigned FrameReg = MFI->getFrameOffsetReg(); 282 if (FrameReg != AMDGPU::NoRegister) { 283 reserveRegisterTuples(Reserved, FrameReg); 284 assert(!isSubRegister(ScratchRSrcReg, FrameReg)); 285 } 286 287 for (unsigned Reg : MFI->WWMReservedRegs) { 288 reserveRegisterTuples(Reserved, Reg); 289 } 290 291 // FIXME: Stop using reserved registers for this. 292 for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs()) 293 reserveRegisterTuples(Reserved, Reg); 294 295 for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs()) 296 reserveRegisterTuples(Reserved, Reg); 297 298 return Reserved; 299 } 300 301 bool SIRegisterInfo::canRealignStack(const MachineFunction &MF) const { 302 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 303 // On entry, the base address is 0, so it can't possibly need any more 304 // alignment. 305 306 // FIXME: Should be able to specify the entry frame alignment per calling 307 // convention instead. 308 if (Info->isEntryFunction()) 309 return false; 310 311 return TargetRegisterInfo::canRealignStack(MF); 312 } 313 314 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 315 const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>(); 316 if (Info->isEntryFunction()) { 317 const MachineFrameInfo &MFI = Fn.getFrameInfo(); 318 return MFI.hasStackObjects() || MFI.hasCalls(); 319 } 320 321 // May need scavenger for dealing with callee saved registers. 322 return true; 323 } 324 325 bool SIRegisterInfo::requiresFrameIndexScavenging( 326 const MachineFunction &MF) const { 327 // Do not use frame virtual registers. They used to be used for SGPRs, but 328 // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the 329 // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a 330 // spill. 331 return false; 332 } 333 334 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( 335 const MachineFunction &MF) const { 336 const MachineFrameInfo &MFI = MF.getFrameInfo(); 337 return MFI.hasStackObjects(); 338 } 339 340 bool SIRegisterInfo::requiresVirtualBaseRegisters( 341 const MachineFunction &) const { 342 // There are no special dedicated stack or frame pointers. 343 return true; 344 } 345 346 int64_t SIRegisterInfo::getMUBUFInstrOffset(const MachineInstr *MI) const { 347 assert(SIInstrInfo::isMUBUF(*MI)); 348 349 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 350 AMDGPU::OpName::offset); 351 return MI->getOperand(OffIdx).getImm(); 352 } 353 354 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI, 355 int Idx) const { 356 if (!SIInstrInfo::isMUBUF(*MI)) 357 return 0; 358 359 assert(Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 360 AMDGPU::OpName::vaddr) && 361 "Should never see frame index on non-address operand"); 362 363 return getMUBUFInstrOffset(MI); 364 } 365 366 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { 367 if (!MI->mayLoadOrStore()) 368 return false; 369 370 int64_t FullOffset = Offset + getMUBUFInstrOffset(MI); 371 372 return !isUInt<12>(FullOffset); 373 } 374 375 void SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 376 unsigned BaseReg, 377 int FrameIdx, 378 int64_t Offset) const { 379 MachineBasicBlock::iterator Ins = MBB->begin(); 380 DebugLoc DL; // Defaults to "unknown" 381 382 if (Ins != MBB->end()) 383 DL = Ins->getDebugLoc(); 384 385 MachineFunction *MF = MBB->getParent(); 386 const SIInstrInfo *TII = ST.getInstrInfo(); 387 388 if (Offset == 0) { 389 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg) 390 .addFrameIndex(FrameIdx); 391 return; 392 } 393 394 MachineRegisterInfo &MRI = MF->getRegInfo(); 395 Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 396 397 Register FIReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 398 399 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 400 .addImm(Offset); 401 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), FIReg) 402 .addFrameIndex(FrameIdx); 403 404 TII->getAddNoCarry(*MBB, Ins, DL, BaseReg) 405 .addReg(OffsetReg, RegState::Kill) 406 .addReg(FIReg) 407 .addImm(0); // clamp bit 408 } 409 410 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, unsigned BaseReg, 411 int64_t Offset) const { 412 const SIInstrInfo *TII = ST.getInstrInfo(); 413 414 #ifndef NDEBUG 415 // FIXME: Is it possible to be storing a frame index to itself? 416 bool SeenFI = false; 417 for (const MachineOperand &MO: MI.operands()) { 418 if (MO.isFI()) { 419 if (SeenFI) 420 llvm_unreachable("should not see multiple frame indices"); 421 422 SeenFI = true; 423 } 424 } 425 #endif 426 427 MachineOperand *FIOp = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr); 428 #ifndef NDEBUG 429 MachineBasicBlock *MBB = MI.getParent(); 430 MachineFunction *MF = MBB->getParent(); 431 #endif 432 assert(FIOp && FIOp->isFI() && "frame index must be address operand"); 433 assert(TII->isMUBUF(MI)); 434 assert(TII->getNamedOperand(MI, AMDGPU::OpName::soffset)->getReg() == 435 MF->getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg() && 436 "should only be seeing stack pointer offset relative FrameIndex"); 437 438 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); 439 int64_t NewOffset = OffsetOp->getImm() + Offset; 440 assert(isUInt<12>(NewOffset) && "offset should be legal"); 441 442 FIOp->ChangeToRegister(BaseReg, false); 443 OffsetOp->setImm(NewOffset); 444 } 445 446 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 447 unsigned BaseReg, 448 int64_t Offset) const { 449 if (!SIInstrInfo::isMUBUF(*MI)) 450 return false; 451 452 int64_t NewOffset = Offset + getMUBUFInstrOffset(MI); 453 454 return isUInt<12>(NewOffset); 455 } 456 457 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass( 458 const MachineFunction &MF, unsigned Kind) const { 459 // This is inaccurate. It depends on the instruction and address space. The 460 // only place where we should hit this is for dealing with frame indexes / 461 // private accesses, so this is correct in that case. 462 return &AMDGPU::VGPR_32RegClass; 463 } 464 465 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 466 467 switch (Op) { 468 case AMDGPU::SI_SPILL_S1024_SAVE: 469 case AMDGPU::SI_SPILL_S1024_RESTORE: 470 case AMDGPU::SI_SPILL_V1024_SAVE: 471 case AMDGPU::SI_SPILL_V1024_RESTORE: 472 case AMDGPU::SI_SPILL_A1024_SAVE: 473 case AMDGPU::SI_SPILL_A1024_RESTORE: 474 return 32; 475 case AMDGPU::SI_SPILL_S512_SAVE: 476 case AMDGPU::SI_SPILL_S512_RESTORE: 477 case AMDGPU::SI_SPILL_V512_SAVE: 478 case AMDGPU::SI_SPILL_V512_RESTORE: 479 case AMDGPU::SI_SPILL_A512_SAVE: 480 case AMDGPU::SI_SPILL_A512_RESTORE: 481 return 16; 482 case AMDGPU::SI_SPILL_S256_SAVE: 483 case AMDGPU::SI_SPILL_S256_RESTORE: 484 case AMDGPU::SI_SPILL_V256_SAVE: 485 case AMDGPU::SI_SPILL_V256_RESTORE: 486 return 8; 487 case AMDGPU::SI_SPILL_S160_SAVE: 488 case AMDGPU::SI_SPILL_S160_RESTORE: 489 case AMDGPU::SI_SPILL_V160_SAVE: 490 case AMDGPU::SI_SPILL_V160_RESTORE: 491 return 5; 492 case AMDGPU::SI_SPILL_S128_SAVE: 493 case AMDGPU::SI_SPILL_S128_RESTORE: 494 case AMDGPU::SI_SPILL_V128_SAVE: 495 case AMDGPU::SI_SPILL_V128_RESTORE: 496 case AMDGPU::SI_SPILL_A128_SAVE: 497 case AMDGPU::SI_SPILL_A128_RESTORE: 498 return 4; 499 case AMDGPU::SI_SPILL_S96_SAVE: 500 case AMDGPU::SI_SPILL_S96_RESTORE: 501 case AMDGPU::SI_SPILL_V96_SAVE: 502 case AMDGPU::SI_SPILL_V96_RESTORE: 503 return 3; 504 case AMDGPU::SI_SPILL_S64_SAVE: 505 case AMDGPU::SI_SPILL_S64_RESTORE: 506 case AMDGPU::SI_SPILL_V64_SAVE: 507 case AMDGPU::SI_SPILL_V64_RESTORE: 508 case AMDGPU::SI_SPILL_A64_SAVE: 509 case AMDGPU::SI_SPILL_A64_RESTORE: 510 return 2; 511 case AMDGPU::SI_SPILL_S32_SAVE: 512 case AMDGPU::SI_SPILL_S32_RESTORE: 513 case AMDGPU::SI_SPILL_V32_SAVE: 514 case AMDGPU::SI_SPILL_V32_RESTORE: 515 case AMDGPU::SI_SPILL_A32_SAVE: 516 case AMDGPU::SI_SPILL_A32_RESTORE: 517 return 1; 518 default: llvm_unreachable("Invalid spill opcode"); 519 } 520 } 521 522 static int getOffsetMUBUFStore(unsigned Opc) { 523 switch (Opc) { 524 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 525 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; 526 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: 527 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; 528 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: 529 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; 530 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: 531 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; 532 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: 533 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; 534 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: 535 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; 536 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: 537 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; 538 default: 539 return -1; 540 } 541 } 542 543 static int getOffsetMUBUFLoad(unsigned Opc) { 544 switch (Opc) { 545 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 546 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 547 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: 548 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; 549 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: 550 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; 551 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: 552 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; 553 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: 554 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; 555 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: 556 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; 557 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: 558 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; 559 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: 560 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; 561 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: 562 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; 563 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: 564 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; 565 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: 566 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; 567 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: 568 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; 569 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: 570 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; 571 default: 572 return -1; 573 } 574 } 575 576 static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST, 577 MachineBasicBlock::iterator MI, 578 int Index, 579 unsigned Lane, 580 unsigned ValueReg, 581 bool IsKill) { 582 MachineBasicBlock *MBB = MI->getParent(); 583 MachineFunction *MF = MI->getParent()->getParent(); 584 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 585 const SIInstrInfo *TII = ST.getInstrInfo(); 586 587 MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane); 588 589 if (Reg == AMDGPU::NoRegister) 590 return MachineInstrBuilder(); 591 592 bool IsStore = MI->mayStore(); 593 MachineRegisterInfo &MRI = MF->getRegInfo(); 594 auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo()); 595 596 unsigned Dst = IsStore ? Reg : ValueReg; 597 unsigned Src = IsStore ? ValueReg : Reg; 598 unsigned Opc = (IsStore ^ TRI->isVGPR(MRI, Reg)) ? AMDGPU::V_ACCVGPR_WRITE_B32 599 : AMDGPU::V_ACCVGPR_READ_B32; 600 601 return BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(Opc), Dst) 602 .addReg(Src, getKillRegState(IsKill)); 603 } 604 605 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not 606 // need to handle the case where an SGPR may need to be spilled while spilling. 607 static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST, 608 MachineFrameInfo &MFI, 609 MachineBasicBlock::iterator MI, 610 int Index, 611 int64_t Offset) { 612 const SIInstrInfo *TII = ST.getInstrInfo(); 613 MachineBasicBlock *MBB = MI->getParent(); 614 const DebugLoc &DL = MI->getDebugLoc(); 615 bool IsStore = MI->mayStore(); 616 617 unsigned Opc = MI->getOpcode(); 618 int LoadStoreOp = IsStore ? 619 getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc); 620 if (LoadStoreOp == -1) 621 return false; 622 623 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); 624 if (spillVGPRtoAGPR(ST, MI, Index, 0, Reg->getReg(), false).getInstr()) 625 return true; 626 627 MachineInstrBuilder NewMI = 628 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 629 .add(*Reg) 630 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) 631 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) 632 .addImm(Offset) 633 .addImm(0) // glc 634 .addImm(0) // slc 635 .addImm(0) // tfe 636 .addImm(0) // dlc 637 .addImm(0) // swz 638 .cloneMemRefs(*MI); 639 640 const MachineOperand *VDataIn = TII->getNamedOperand(*MI, 641 AMDGPU::OpName::vdata_in); 642 if (VDataIn) 643 NewMI.add(*VDataIn); 644 return true; 645 } 646 647 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI, 648 unsigned LoadStoreOp, 649 int Index, 650 unsigned ValueReg, 651 bool IsKill, 652 unsigned ScratchRsrcReg, 653 unsigned ScratchOffsetReg, 654 int64_t InstOffset, 655 MachineMemOperand *MMO, 656 RegScavenger *RS) const { 657 MachineBasicBlock *MBB = MI->getParent(); 658 MachineFunction *MF = MI->getParent()->getParent(); 659 const SIInstrInfo *TII = ST.getInstrInfo(); 660 const MachineFrameInfo &MFI = MF->getFrameInfo(); 661 662 const MCInstrDesc &Desc = TII->get(LoadStoreOp); 663 const DebugLoc &DL = MI->getDebugLoc(); 664 bool IsStore = Desc.mayStore(); 665 666 bool Scavenged = false; 667 unsigned SOffset = ScratchOffsetReg; 668 669 const unsigned EltSize = 4; 670 const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg); 671 unsigned NumSubRegs = AMDGPU::getRegBitWidth(RC->getID()) / (EltSize * CHAR_BIT); 672 unsigned Size = NumSubRegs * EltSize; 673 int64_t Offset = InstOffset + MFI.getObjectOffset(Index); 674 int64_t ScratchOffsetRegDelta = 0; 675 676 unsigned Align = MFI.getObjectAlignment(Index); 677 const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo(); 678 679 Register TmpReg = 680 hasAGPRs(RC) ? TII->getNamedOperand(*MI, AMDGPU::OpName::tmp)->getReg() 681 : Register(); 682 683 assert((Offset % EltSize) == 0 && "unexpected VGPR spill offset"); 684 685 if (!isUInt<12>(Offset + Size - EltSize)) { 686 SOffset = AMDGPU::NoRegister; 687 688 // We currently only support spilling VGPRs to EltSize boundaries, meaning 689 // we can simplify the adjustment of Offset here to just scale with 690 // WavefrontSize. 691 Offset *= ST.getWavefrontSize(); 692 693 // We don't have access to the register scavenger if this function is called 694 // during PEI::scavengeFrameVirtualRegs(). 695 if (RS) 696 SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false); 697 698 if (SOffset == AMDGPU::NoRegister) { 699 if (ScratchOffsetReg == AMDGPU::NoRegister) { 700 report_fatal_error("could not scavenge SGPR to spill in entry function"); 701 } 702 // There are no free SGPRs, and since we are in the process of spilling 703 // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true 704 // on SI/CI and on VI it is true until we implement spilling using scalar 705 // stores), we have no way to free up an SGPR. Our solution here is to 706 // add the offset directly to the ScratchOffset register, and then 707 // subtract the offset after the spill to return ScratchOffset to it's 708 // original value. 709 SOffset = ScratchOffsetReg; 710 ScratchOffsetRegDelta = Offset; 711 } else { 712 Scavenged = true; 713 } 714 715 if (ScratchOffsetReg == AMDGPU::NoRegister) { 716 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset) 717 .addImm(Offset); 718 } else { 719 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset) 720 .addReg(ScratchOffsetReg) 721 .addImm(Offset); 722 } 723 724 Offset = 0; 725 } 726 727 for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += EltSize) { 728 Register SubReg = NumSubRegs == 1 729 ? Register(ValueReg) 730 : getSubReg(ValueReg, getSubRegFromChannel(i)); 731 732 unsigned SOffsetRegState = 0; 733 unsigned SrcDstRegState = getDefRegState(!IsStore); 734 if (i + 1 == e) { 735 SOffsetRegState |= getKillRegState(Scavenged); 736 // The last implicit use carries the "Kill" flag. 737 SrcDstRegState |= getKillRegState(IsKill); 738 } 739 740 auto MIB = spillVGPRtoAGPR(ST, MI, Index, i, SubReg, IsKill); 741 742 if (!MIB.getInstr()) { 743 unsigned FinalReg = SubReg; 744 if (TmpReg != AMDGPU::NoRegister) { 745 if (IsStore) 746 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_READ_B32), TmpReg) 747 .addReg(SubReg, getKillRegState(IsKill)); 748 SubReg = TmpReg; 749 } 750 751 MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(EltSize * i); 752 MachineMemOperand *NewMMO 753 = MF->getMachineMemOperand(PInfo, MMO->getFlags(), 754 EltSize, MinAlign(Align, EltSize * i)); 755 756 MIB = BuildMI(*MBB, MI, DL, Desc) 757 .addReg(SubReg, 758 getDefRegState(!IsStore) | getKillRegState(IsKill)) 759 .addReg(ScratchRsrcReg); 760 if (SOffset == AMDGPU::NoRegister) { 761 MIB.addImm(0); 762 } else { 763 MIB.addReg(SOffset, SOffsetRegState); 764 } 765 MIB.addImm(Offset) 766 .addImm(0) // glc 767 .addImm(0) // slc 768 .addImm(0) // tfe 769 .addImm(0) // dlc 770 .addImm(0) // swz 771 .addMemOperand(NewMMO); 772 773 if (!IsStore && TmpReg != AMDGPU::NoRegister) 774 MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32), 775 FinalReg) 776 .addReg(TmpReg, RegState::Kill); 777 } 778 779 if (NumSubRegs > 1) 780 MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState); 781 } 782 783 if (ScratchOffsetRegDelta != 0) { 784 // Subtract the offset we added to the ScratchOffset register. 785 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScratchOffsetReg) 786 .addReg(ScratchOffsetReg) 787 .addImm(ScratchOffsetRegDelta); 788 } 789 } 790 791 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, 792 int Index, 793 RegScavenger *RS, 794 bool OnlyToVGPR) const { 795 MachineBasicBlock *MBB = MI->getParent(); 796 MachineFunction *MF = MBB->getParent(); 797 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 798 DenseSet<unsigned> SGPRSpillVGPRDefinedSet; 799 800 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 801 = MFI->getSGPRToVGPRSpills(Index); 802 bool SpillToVGPR = !VGPRSpills.empty(); 803 if (OnlyToVGPR && !SpillToVGPR) 804 return false; 805 806 const SIInstrInfo *TII = ST.getInstrInfo(); 807 808 Register SuperReg = MI->getOperand(0).getReg(); 809 bool IsKill = MI->getOperand(0).isKill(); 810 const DebugLoc &DL = MI->getDebugLoc(); 811 812 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 813 814 assert(SpillToVGPR || (SuperReg != MFI->getStackPtrOffsetReg() && 815 SuperReg != MFI->getFrameOffsetReg())); 816 817 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 818 819 unsigned EltSize = 4; 820 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 821 822 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 823 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 824 825 // Scavenged temporary VGPR to use. It must be scavenged once for any number 826 // of spilled subregs. 827 Register TmpVGPR; 828 829 // SubReg carries the "Kill" flag when SubReg == SuperReg. 830 unsigned SubKillState = getKillRegState((NumSubRegs == 1) && IsKill); 831 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 832 Register SubReg = 833 NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]); 834 835 if (SpillToVGPR) { 836 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 837 838 // During SGPR spilling to VGPR, determine if the VGPR is defined. The 839 // only circumstance in which we say it is undefined is when it is the 840 // first spill to this VGPR in the first basic block. 841 bool VGPRDefined = true; 842 if (MBB == &MF->front()) 843 VGPRDefined = !SGPRSpillVGPRDefinedSet.insert(Spill.VGPR).second; 844 845 // Mark the "old value of vgpr" input undef only if this is the first sgpr 846 // spill to this specific vgpr in the first basic block. 847 BuildMI(*MBB, MI, DL, 848 TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32), 849 Spill.VGPR) 850 .addReg(SubReg, getKillRegState(IsKill)) 851 .addImm(Spill.Lane) 852 .addReg(Spill.VGPR, VGPRDefined ? 0 : RegState::Undef); 853 854 // FIXME: Since this spills to another register instead of an actual 855 // frame index, we should delete the frame index when all references to 856 // it are fixed. 857 } else { 858 // XXX - Can to VGPR spill fail for some subregisters but not others? 859 if (OnlyToVGPR) 860 return false; 861 862 // Spill SGPR to a frame index. 863 if (!TmpVGPR.isValid()) 864 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 865 866 MachineInstrBuilder Mov 867 = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 868 .addReg(SubReg, SubKillState); 869 870 // There could be undef components of a spilled super register. 871 // TODO: Can we detect this and skip the spill? 872 if (NumSubRegs > 1) { 873 // The last implicit use of the SuperReg carries the "Kill" flag. 874 unsigned SuperKillState = 0; 875 if (i + 1 == e) 876 SuperKillState |= getKillRegState(IsKill); 877 Mov.addReg(SuperReg, RegState::Implicit | SuperKillState); 878 } 879 880 unsigned Align = FrameInfo.getObjectAlignment(Index); 881 MachinePointerInfo PtrInfo 882 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 883 MachineMemOperand *MMO 884 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 885 EltSize, MinAlign(Align, EltSize * i)); 886 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_SAVE)) 887 .addReg(TmpVGPR, RegState::Kill) // src 888 .addFrameIndex(Index) // vaddr 889 .addReg(MFI->getScratchRSrcReg()) // srrsrc 890 .addReg(MFI->getStackPtrOffsetReg()) // soffset 891 .addImm(i * 4) // offset 892 .addMemOperand(MMO); 893 } 894 } 895 896 MI->eraseFromParent(); 897 MFI->addToSpilledSGPRs(NumSubRegs); 898 return true; 899 } 900 901 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, 902 int Index, 903 RegScavenger *RS, 904 bool OnlyToVGPR) const { 905 MachineFunction *MF = MI->getParent()->getParent(); 906 MachineBasicBlock *MBB = MI->getParent(); 907 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 908 909 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 910 = MFI->getSGPRToVGPRSpills(Index); 911 bool SpillToVGPR = !VGPRSpills.empty(); 912 if (OnlyToVGPR && !SpillToVGPR) 913 return false; 914 915 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 916 const SIInstrInfo *TII = ST.getInstrInfo(); 917 const DebugLoc &DL = MI->getDebugLoc(); 918 919 Register SuperReg = MI->getOperand(0).getReg(); 920 921 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 922 923 unsigned EltSize = 4; 924 925 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 926 927 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 928 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 929 930 Register TmpVGPR; 931 932 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 933 Register SubReg = 934 NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]); 935 936 if (SpillToVGPR) { 937 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 938 auto MIB = 939 BuildMI(*MBB, MI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32), 940 SubReg) 941 .addReg(Spill.VGPR) 942 .addImm(Spill.Lane); 943 944 if (NumSubRegs > 1 && i == 0) 945 MIB.addReg(SuperReg, RegState::ImplicitDefine); 946 } else { 947 if (OnlyToVGPR) 948 return false; 949 950 // Restore SGPR from a stack slot. 951 // FIXME: We should use S_LOAD_DWORD here for VI. 952 if (!TmpVGPR.isValid()) 953 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 954 unsigned Align = FrameInfo.getObjectAlignment(Index); 955 956 MachinePointerInfo PtrInfo 957 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 958 959 MachineMemOperand *MMO = MF->getMachineMemOperand(PtrInfo, 960 MachineMemOperand::MOLoad, EltSize, 961 MinAlign(Align, EltSize * i)); 962 963 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_RESTORE), TmpVGPR) 964 .addFrameIndex(Index) // vaddr 965 .addReg(MFI->getScratchRSrcReg()) // srsrc 966 .addReg(MFI->getStackPtrOffsetReg()) // soffset 967 .addImm(i * 4) // offset 968 .addMemOperand(MMO); 969 970 auto MIB = 971 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg) 972 .addReg(TmpVGPR, RegState::Kill); 973 974 if (NumSubRegs > 1) 975 MIB.addReg(MI->getOperand(0).getReg(), RegState::ImplicitDefine); 976 } 977 } 978 979 MI->eraseFromParent(); 980 return true; 981 } 982 983 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to 984 /// a VGPR and the stack slot can be safely eliminated when all other users are 985 /// handled. 986 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex( 987 MachineBasicBlock::iterator MI, 988 int FI, 989 RegScavenger *RS) const { 990 switch (MI->getOpcode()) { 991 case AMDGPU::SI_SPILL_S1024_SAVE: 992 case AMDGPU::SI_SPILL_S512_SAVE: 993 case AMDGPU::SI_SPILL_S256_SAVE: 994 case AMDGPU::SI_SPILL_S160_SAVE: 995 case AMDGPU::SI_SPILL_S128_SAVE: 996 case AMDGPU::SI_SPILL_S96_SAVE: 997 case AMDGPU::SI_SPILL_S64_SAVE: 998 case AMDGPU::SI_SPILL_S32_SAVE: 999 return spillSGPR(MI, FI, RS, true); 1000 case AMDGPU::SI_SPILL_S1024_RESTORE: 1001 case AMDGPU::SI_SPILL_S512_RESTORE: 1002 case AMDGPU::SI_SPILL_S256_RESTORE: 1003 case AMDGPU::SI_SPILL_S160_RESTORE: 1004 case AMDGPU::SI_SPILL_S128_RESTORE: 1005 case AMDGPU::SI_SPILL_S96_RESTORE: 1006 case AMDGPU::SI_SPILL_S64_RESTORE: 1007 case AMDGPU::SI_SPILL_S32_RESTORE: 1008 return restoreSGPR(MI, FI, RS, true); 1009 default: 1010 llvm_unreachable("not an SGPR spill instruction"); 1011 } 1012 } 1013 1014 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 1015 int SPAdj, unsigned FIOperandNum, 1016 RegScavenger *RS) const { 1017 MachineFunction *MF = MI->getParent()->getParent(); 1018 MachineBasicBlock *MBB = MI->getParent(); 1019 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1020 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1021 const SIInstrInfo *TII = ST.getInstrInfo(); 1022 DebugLoc DL = MI->getDebugLoc(); 1023 1024 assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?"); 1025 1026 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 1027 int Index = MI->getOperand(FIOperandNum).getIndex(); 1028 1029 Register FrameReg = getFrameRegister(*MF); 1030 1031 switch (MI->getOpcode()) { 1032 // SGPR register spill 1033 case AMDGPU::SI_SPILL_S1024_SAVE: 1034 case AMDGPU::SI_SPILL_S512_SAVE: 1035 case AMDGPU::SI_SPILL_S256_SAVE: 1036 case AMDGPU::SI_SPILL_S160_SAVE: 1037 case AMDGPU::SI_SPILL_S128_SAVE: 1038 case AMDGPU::SI_SPILL_S96_SAVE: 1039 case AMDGPU::SI_SPILL_S64_SAVE: 1040 case AMDGPU::SI_SPILL_S32_SAVE: { 1041 spillSGPR(MI, Index, RS); 1042 break; 1043 } 1044 1045 // SGPR register restore 1046 case AMDGPU::SI_SPILL_S1024_RESTORE: 1047 case AMDGPU::SI_SPILL_S512_RESTORE: 1048 case AMDGPU::SI_SPILL_S256_RESTORE: 1049 case AMDGPU::SI_SPILL_S160_RESTORE: 1050 case AMDGPU::SI_SPILL_S128_RESTORE: 1051 case AMDGPU::SI_SPILL_S96_RESTORE: 1052 case AMDGPU::SI_SPILL_S64_RESTORE: 1053 case AMDGPU::SI_SPILL_S32_RESTORE: { 1054 restoreSGPR(MI, Index, RS); 1055 break; 1056 } 1057 1058 // VGPR register spill 1059 case AMDGPU::SI_SPILL_V1024_SAVE: 1060 case AMDGPU::SI_SPILL_V512_SAVE: 1061 case AMDGPU::SI_SPILL_V256_SAVE: 1062 case AMDGPU::SI_SPILL_V160_SAVE: 1063 case AMDGPU::SI_SPILL_V128_SAVE: 1064 case AMDGPU::SI_SPILL_V96_SAVE: 1065 case AMDGPU::SI_SPILL_V64_SAVE: 1066 case AMDGPU::SI_SPILL_V32_SAVE: 1067 case AMDGPU::SI_SPILL_A1024_SAVE: 1068 case AMDGPU::SI_SPILL_A512_SAVE: 1069 case AMDGPU::SI_SPILL_A128_SAVE: 1070 case AMDGPU::SI_SPILL_A64_SAVE: 1071 case AMDGPU::SI_SPILL_A32_SAVE: { 1072 const MachineOperand *VData = TII->getNamedOperand(*MI, 1073 AMDGPU::OpName::vdata); 1074 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 1075 MFI->getStackPtrOffsetReg()); 1076 1077 buildSpillLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET, 1078 Index, 1079 VData->getReg(), VData->isKill(), 1080 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1081 FrameReg, 1082 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1083 *MI->memoperands_begin(), 1084 RS); 1085 MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode())); 1086 MI->eraseFromParent(); 1087 break; 1088 } 1089 case AMDGPU::SI_SPILL_V32_RESTORE: 1090 case AMDGPU::SI_SPILL_V64_RESTORE: 1091 case AMDGPU::SI_SPILL_V96_RESTORE: 1092 case AMDGPU::SI_SPILL_V128_RESTORE: 1093 case AMDGPU::SI_SPILL_V160_RESTORE: 1094 case AMDGPU::SI_SPILL_V256_RESTORE: 1095 case AMDGPU::SI_SPILL_V512_RESTORE: 1096 case AMDGPU::SI_SPILL_V1024_RESTORE: 1097 case AMDGPU::SI_SPILL_A32_RESTORE: 1098 case AMDGPU::SI_SPILL_A64_RESTORE: 1099 case AMDGPU::SI_SPILL_A128_RESTORE: 1100 case AMDGPU::SI_SPILL_A512_RESTORE: 1101 case AMDGPU::SI_SPILL_A1024_RESTORE: { 1102 const MachineOperand *VData = TII->getNamedOperand(*MI, 1103 AMDGPU::OpName::vdata); 1104 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 1105 MFI->getStackPtrOffsetReg()); 1106 1107 buildSpillLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET, 1108 Index, 1109 VData->getReg(), VData->isKill(), 1110 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1111 FrameReg, 1112 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1113 *MI->memoperands_begin(), 1114 RS); 1115 MI->eraseFromParent(); 1116 break; 1117 } 1118 1119 default: { 1120 const DebugLoc &DL = MI->getDebugLoc(); 1121 bool IsMUBUF = TII->isMUBUF(*MI); 1122 1123 if (!IsMUBUF && !MFI->isEntryFunction()) { 1124 // Convert to a swizzled stack address by scaling by the wave size. 1125 // 1126 // In an entry function/kernel the offset is already swizzled. 1127 1128 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32; 1129 Register ResultReg = 1130 IsCopy ? MI->getOperand(0).getReg() 1131 : RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1132 1133 int64_t Offset = FrameInfo.getObjectOffset(Index); 1134 if (Offset == 0) { 1135 // XXX - This never happens because of emergency scavenging slot at 0? 1136 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ResultReg) 1137 .addImm(ST.getWavefrontSizeLog2()) 1138 .addReg(FrameReg); 1139 } else { 1140 if (auto MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) { 1141 // Reuse ResultReg in intermediate step. 1142 Register ScaledReg = ResultReg; 1143 1144 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), 1145 ScaledReg) 1146 .addImm(ST.getWavefrontSizeLog2()) 1147 .addReg(FrameReg); 1148 1149 const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32; 1150 1151 // TODO: Fold if use instruction is another add of a constant. 1152 if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { 1153 // FIXME: This can fail 1154 MIB.addImm(Offset); 1155 MIB.addReg(ScaledReg, RegState::Kill); 1156 if (!IsVOP2) 1157 MIB.addImm(0); // clamp bit 1158 } else { 1159 assert(MIB->getOpcode() == AMDGPU::V_ADD_I32_e64 && 1160 "Need to reuse carry out register"); 1161 1162 // Use scavenged unused carry out as offset register. 1163 Register ConstOffsetReg; 1164 if (!isWave32) 1165 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); 1166 else 1167 ConstOffsetReg = MIB.getReg(1); 1168 1169 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) 1170 .addImm(Offset); 1171 MIB.addReg(ConstOffsetReg, RegState::Kill); 1172 MIB.addReg(ScaledReg, RegState::Kill); 1173 MIB.addImm(0); // clamp bit 1174 } 1175 } else { 1176 // We have to produce a carry out, and there isn't a free SGPR pair 1177 // for it. We can keep the whole computation on the SALU to avoid 1178 // clobbering an additional register at the cost of an extra mov. 1179 1180 // We may have 1 free scratch SGPR even though a carry out is 1181 // unavailable. Only one additional mov is needed. 1182 Register TmpScaledReg = 1183 RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, false); 1184 Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg; 1185 1186 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg) 1187 .addReg(FrameReg) 1188 .addImm(ST.getWavefrontSizeLog2()); 1189 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), ScaledReg) 1190 .addReg(ScaledReg, RegState::Kill) 1191 .addImm(Offset); 1192 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg) 1193 .addReg(ScaledReg, RegState::Kill); 1194 1195 // If there were truly no free SGPRs, we need to undo everything. 1196 if (!TmpScaledReg.isValid()) { 1197 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScaledReg) 1198 .addReg(ScaledReg, RegState::Kill) 1199 .addImm(Offset); 1200 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) 1201 .addReg(FrameReg) 1202 .addImm(ST.getWavefrontSizeLog2()); 1203 } 1204 } 1205 } 1206 1207 // Don't introduce an extra copy if we're just materializing in a mov. 1208 if (IsCopy) 1209 MI->eraseFromParent(); 1210 else 1211 FIOp.ChangeToRegister(ResultReg, false, false, true); 1212 return; 1213 } 1214 1215 if (IsMUBUF) { 1216 // Disable offen so we don't need a 0 vgpr base. 1217 assert(static_cast<int>(FIOperandNum) == 1218 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1219 AMDGPU::OpName::vaddr)); 1220 1221 auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset); 1222 assert((SOffset.isReg() && 1223 SOffset.getReg() == MFI->getStackPtrOffsetReg()) || 1224 (SOffset.isImm() && SOffset.getImm() == 0)); 1225 if (SOffset.isReg()) { 1226 if (FrameReg == AMDGPU::NoRegister) { 1227 SOffset.ChangeToImmediate(0); 1228 } else { 1229 SOffset.setReg(FrameReg); 1230 } 1231 } 1232 1233 int64_t Offset = FrameInfo.getObjectOffset(Index); 1234 int64_t OldImm 1235 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); 1236 int64_t NewOffset = OldImm + Offset; 1237 1238 if (isUInt<12>(NewOffset) && 1239 buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) { 1240 MI->eraseFromParent(); 1241 return; 1242 } 1243 } 1244 1245 // If the offset is simply too big, don't convert to a scratch wave offset 1246 // relative index. 1247 1248 int64_t Offset = FrameInfo.getObjectOffset(Index); 1249 FIOp.ChangeToImmediate(Offset); 1250 if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) { 1251 Register TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1252 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 1253 .addImm(Offset); 1254 FIOp.ChangeToRegister(TmpReg, false, false, true); 1255 } 1256 } 1257 } 1258 } 1259 1260 StringRef SIRegisterInfo::getRegAsmName(unsigned Reg) const { 1261 return AMDGPUInstPrinter::getRegisterName(Reg); 1262 } 1263 1264 // FIXME: This is very slow. It might be worth creating a map from physreg to 1265 // register class. 1266 const TargetRegisterClass *SIRegisterInfo::getPhysRegClass(unsigned Reg) const { 1267 assert(!Register::isVirtualRegister(Reg)); 1268 1269 static const TargetRegisterClass *const BaseClasses[] = { 1270 &AMDGPU::VGPR_32RegClass, 1271 &AMDGPU::SReg_32RegClass, 1272 &AMDGPU::AGPR_32RegClass, 1273 &AMDGPU::VReg_64RegClass, 1274 &AMDGPU::SReg_64RegClass, 1275 &AMDGPU::AReg_64RegClass, 1276 &AMDGPU::VReg_96RegClass, 1277 &AMDGPU::SReg_96RegClass, 1278 &AMDGPU::VReg_128RegClass, 1279 &AMDGPU::SReg_128RegClass, 1280 &AMDGPU::AReg_128RegClass, 1281 &AMDGPU::VReg_160RegClass, 1282 &AMDGPU::SReg_160RegClass, 1283 &AMDGPU::VReg_256RegClass, 1284 &AMDGPU::SReg_256RegClass, 1285 &AMDGPU::VReg_512RegClass, 1286 &AMDGPU::SReg_512RegClass, 1287 &AMDGPU::AReg_512RegClass, 1288 &AMDGPU::SReg_1024RegClass, 1289 &AMDGPU::VReg_1024RegClass, 1290 &AMDGPU::AReg_1024RegClass, 1291 &AMDGPU::SCC_CLASSRegClass, 1292 &AMDGPU::Pseudo_SReg_32RegClass, 1293 &AMDGPU::Pseudo_SReg_128RegClass, 1294 }; 1295 1296 for (const TargetRegisterClass *BaseClass : BaseClasses) { 1297 if (BaseClass->contains(Reg)) { 1298 return BaseClass; 1299 } 1300 } 1301 return nullptr; 1302 } 1303 1304 // TODO: It might be helpful to have some target specific flags in 1305 // TargetRegisterClass to mark which classes are VGPRs to make this trivial. 1306 bool SIRegisterInfo::hasVGPRs(const TargetRegisterClass *RC) const { 1307 unsigned Size = getRegSizeInBits(*RC); 1308 switch (Size) { 1309 case 32: 1310 return getCommonSubClass(&AMDGPU::VGPR_32RegClass, RC) != nullptr; 1311 case 64: 1312 return getCommonSubClass(&AMDGPU::VReg_64RegClass, RC) != nullptr; 1313 case 96: 1314 return getCommonSubClass(&AMDGPU::VReg_96RegClass, RC) != nullptr; 1315 case 128: 1316 return getCommonSubClass(&AMDGPU::VReg_128RegClass, RC) != nullptr; 1317 case 160: 1318 return getCommonSubClass(&AMDGPU::VReg_160RegClass, RC) != nullptr; 1319 case 256: 1320 return getCommonSubClass(&AMDGPU::VReg_256RegClass, RC) != nullptr; 1321 case 512: 1322 return getCommonSubClass(&AMDGPU::VReg_512RegClass, RC) != nullptr; 1323 case 1024: 1324 return getCommonSubClass(&AMDGPU::VReg_1024RegClass, RC) != nullptr; 1325 case 1: 1326 return getCommonSubClass(&AMDGPU::VReg_1RegClass, RC) != nullptr; 1327 default: 1328 assert(Size < 32 && "Invalid register class size"); 1329 return false; 1330 } 1331 } 1332 1333 bool SIRegisterInfo::hasAGPRs(const TargetRegisterClass *RC) const { 1334 unsigned Size = getRegSizeInBits(*RC); 1335 if (Size < 32) 1336 return false; 1337 switch (Size) { 1338 case 32: 1339 return getCommonSubClass(&AMDGPU::AGPR_32RegClass, RC) != nullptr; 1340 case 64: 1341 return getCommonSubClass(&AMDGPU::AReg_64RegClass, RC) != nullptr; 1342 case 96: 1343 return false; 1344 case 128: 1345 return getCommonSubClass(&AMDGPU::AReg_128RegClass, RC) != nullptr; 1346 case 160: 1347 case 256: 1348 return false; 1349 case 512: 1350 return getCommonSubClass(&AMDGPU::AReg_512RegClass, RC) != nullptr; 1351 case 1024: 1352 return getCommonSubClass(&AMDGPU::AReg_1024RegClass, RC) != nullptr; 1353 default: 1354 llvm_unreachable("Invalid register class size"); 1355 } 1356 } 1357 1358 const TargetRegisterClass *SIRegisterInfo::getEquivalentVGPRClass( 1359 const TargetRegisterClass *SRC) const { 1360 switch (getRegSizeInBits(*SRC)) { 1361 case 32: 1362 return &AMDGPU::VGPR_32RegClass; 1363 case 64: 1364 return &AMDGPU::VReg_64RegClass; 1365 case 96: 1366 return &AMDGPU::VReg_96RegClass; 1367 case 128: 1368 return &AMDGPU::VReg_128RegClass; 1369 case 160: 1370 return &AMDGPU::VReg_160RegClass; 1371 case 256: 1372 return &AMDGPU::VReg_256RegClass; 1373 case 512: 1374 return &AMDGPU::VReg_512RegClass; 1375 case 1024: 1376 return &AMDGPU::VReg_1024RegClass; 1377 case 1: 1378 return &AMDGPU::VReg_1RegClass; 1379 default: 1380 llvm_unreachable("Invalid register class size"); 1381 } 1382 } 1383 1384 const TargetRegisterClass *SIRegisterInfo::getEquivalentAGPRClass( 1385 const TargetRegisterClass *SRC) const { 1386 switch (getRegSizeInBits(*SRC)) { 1387 case 32: 1388 return &AMDGPU::AGPR_32RegClass; 1389 case 64: 1390 return &AMDGPU::AReg_64RegClass; 1391 case 128: 1392 return &AMDGPU::AReg_128RegClass; 1393 case 512: 1394 return &AMDGPU::AReg_512RegClass; 1395 case 1024: 1396 return &AMDGPU::AReg_1024RegClass; 1397 default: 1398 llvm_unreachable("Invalid register class size"); 1399 } 1400 } 1401 1402 const TargetRegisterClass *SIRegisterInfo::getEquivalentSGPRClass( 1403 const TargetRegisterClass *VRC) const { 1404 switch (getRegSizeInBits(*VRC)) { 1405 case 32: 1406 return &AMDGPU::SGPR_32RegClass; 1407 case 64: 1408 return &AMDGPU::SReg_64RegClass; 1409 case 96: 1410 return &AMDGPU::SReg_96RegClass; 1411 case 128: 1412 return &AMDGPU::SGPR_128RegClass; 1413 case 160: 1414 return &AMDGPU::SReg_160RegClass; 1415 case 256: 1416 return &AMDGPU::SReg_256RegClass; 1417 case 512: 1418 return &AMDGPU::SReg_512RegClass; 1419 case 1024: 1420 return &AMDGPU::SReg_1024RegClass; 1421 default: 1422 llvm_unreachable("Invalid register class size"); 1423 } 1424 } 1425 1426 const TargetRegisterClass *SIRegisterInfo::getSubRegClass( 1427 const TargetRegisterClass *RC, unsigned SubIdx) const { 1428 if (SubIdx == AMDGPU::NoSubRegister) 1429 return RC; 1430 1431 // We can assume that each lane corresponds to one 32-bit register. 1432 unsigned Count = getNumChannelsFromSubReg(SubIdx); 1433 if (isSGPRClass(RC)) { 1434 switch (Count) { 1435 case 1: 1436 return &AMDGPU::SGPR_32RegClass; 1437 case 2: 1438 return &AMDGPU::SReg_64RegClass; 1439 case 3: 1440 return &AMDGPU::SReg_96RegClass; 1441 case 4: 1442 return &AMDGPU::SGPR_128RegClass; 1443 case 5: 1444 return &AMDGPU::SReg_160RegClass; 1445 case 8: 1446 return &AMDGPU::SReg_256RegClass; 1447 case 16: 1448 return &AMDGPU::SReg_512RegClass; 1449 case 32: /* fall-through */ 1450 default: 1451 llvm_unreachable("Invalid sub-register class size"); 1452 } 1453 } else if (hasAGPRs(RC)) { 1454 switch (Count) { 1455 case 1: 1456 return &AMDGPU::AGPR_32RegClass; 1457 case 2: 1458 return &AMDGPU::AReg_64RegClass; 1459 case 4: 1460 return &AMDGPU::AReg_128RegClass; 1461 case 16: 1462 return &AMDGPU::AReg_512RegClass; 1463 case 32: /* fall-through */ 1464 default: 1465 llvm_unreachable("Invalid sub-register class size"); 1466 } 1467 } else { 1468 switch (Count) { 1469 case 1: 1470 return &AMDGPU::VGPR_32RegClass; 1471 case 2: 1472 return &AMDGPU::VReg_64RegClass; 1473 case 3: 1474 return &AMDGPU::VReg_96RegClass; 1475 case 4: 1476 return &AMDGPU::VReg_128RegClass; 1477 case 5: 1478 return &AMDGPU::VReg_160RegClass; 1479 case 8: 1480 return &AMDGPU::VReg_256RegClass; 1481 case 16: 1482 return &AMDGPU::VReg_512RegClass; 1483 case 32: /* fall-through */ 1484 default: 1485 llvm_unreachable("Invalid sub-register class size"); 1486 } 1487 } 1488 } 1489 1490 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const { 1491 if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST && 1492 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST) 1493 return !ST.hasMFMAInlineLiteralBug(); 1494 1495 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1496 OpType <= AMDGPU::OPERAND_SRC_LAST; 1497 } 1498 1499 bool SIRegisterInfo::shouldRewriteCopySrc( 1500 const TargetRegisterClass *DefRC, 1501 unsigned DefSubReg, 1502 const TargetRegisterClass *SrcRC, 1503 unsigned SrcSubReg) const { 1504 // We want to prefer the smallest register class possible, so we don't want to 1505 // stop and rewrite on anything that looks like a subregister 1506 // extract. Operations mostly don't care about the super register class, so we 1507 // only want to stop on the most basic of copies between the same register 1508 // class. 1509 // 1510 // e.g. if we have something like 1511 // %0 = ... 1512 // %1 = ... 1513 // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2 1514 // %3 = COPY %2, sub0 1515 // 1516 // We want to look through the COPY to find: 1517 // => %3 = COPY %0 1518 1519 // Plain copy. 1520 return getCommonSubClass(DefRC, SrcRC) != nullptr; 1521 } 1522 1523 /// Returns a register that is not used at any point in the function. 1524 /// If all registers are used, then this function will return 1525 // AMDGPU::NoRegister. 1526 unsigned 1527 SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI, 1528 const TargetRegisterClass *RC, 1529 const MachineFunction &MF) const { 1530 1531 for (unsigned Reg : *RC) 1532 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 1533 return Reg; 1534 return AMDGPU::NoRegister; 1535 } 1536 1537 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC, 1538 unsigned EltSize) const { 1539 if (EltSize == 4) { 1540 static const int16_t Sub0_31[] = { 1541 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1542 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 1543 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 1544 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 1545 AMDGPU::sub16, AMDGPU::sub17, AMDGPU::sub18, AMDGPU::sub19, 1546 AMDGPU::sub20, AMDGPU::sub21, AMDGPU::sub22, AMDGPU::sub23, 1547 AMDGPU::sub24, AMDGPU::sub25, AMDGPU::sub26, AMDGPU::sub27, 1548 AMDGPU::sub28, AMDGPU::sub29, AMDGPU::sub30, AMDGPU::sub31, 1549 }; 1550 1551 static const int16_t Sub0_15[] = { 1552 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1553 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 1554 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 1555 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 1556 }; 1557 1558 static const int16_t Sub0_7[] = { 1559 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1560 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 1561 }; 1562 1563 static const int16_t Sub0_4[] = { 1564 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, AMDGPU::sub4, 1565 }; 1566 1567 static const int16_t Sub0_3[] = { 1568 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1569 }; 1570 1571 static const int16_t Sub0_2[] = { 1572 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, 1573 }; 1574 1575 static const int16_t Sub0_1[] = { 1576 AMDGPU::sub0, AMDGPU::sub1, 1577 }; 1578 1579 switch (AMDGPU::getRegBitWidth(*RC->MC)) { 1580 case 32: 1581 return {}; 1582 case 64: 1583 return makeArrayRef(Sub0_1); 1584 case 96: 1585 return makeArrayRef(Sub0_2); 1586 case 128: 1587 return makeArrayRef(Sub0_3); 1588 case 160: 1589 return makeArrayRef(Sub0_4); 1590 case 256: 1591 return makeArrayRef(Sub0_7); 1592 case 512: 1593 return makeArrayRef(Sub0_15); 1594 case 1024: 1595 return makeArrayRef(Sub0_31); 1596 default: 1597 llvm_unreachable("unhandled register size"); 1598 } 1599 } 1600 1601 if (EltSize == 8) { 1602 static const int16_t Sub0_31_64[] = { 1603 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 1604 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 1605 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 1606 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 1607 AMDGPU::sub16_sub17, AMDGPU::sub18_sub19, 1608 AMDGPU::sub20_sub21, AMDGPU::sub22_sub23, 1609 AMDGPU::sub24_sub25, AMDGPU::sub26_sub27, 1610 AMDGPU::sub28_sub29, AMDGPU::sub30_sub31 1611 }; 1612 1613 static const int16_t Sub0_15_64[] = { 1614 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 1615 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 1616 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 1617 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15 1618 }; 1619 1620 static const int16_t Sub0_7_64[] = { 1621 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 1622 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7 1623 }; 1624 1625 1626 static const int16_t Sub0_3_64[] = { 1627 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3 1628 }; 1629 1630 switch (AMDGPU::getRegBitWidth(*RC->MC)) { 1631 case 64: 1632 return {}; 1633 case 128: 1634 return makeArrayRef(Sub0_3_64); 1635 case 256: 1636 return makeArrayRef(Sub0_7_64); 1637 case 512: 1638 return makeArrayRef(Sub0_15_64); 1639 case 1024: 1640 return makeArrayRef(Sub0_31_64); 1641 default: 1642 llvm_unreachable("unhandled register size"); 1643 } 1644 } 1645 1646 if (EltSize == 16) { 1647 1648 static const int16_t Sub0_31_128[] = { 1649 AMDGPU::sub0_sub1_sub2_sub3, 1650 AMDGPU::sub4_sub5_sub6_sub7, 1651 AMDGPU::sub8_sub9_sub10_sub11, 1652 AMDGPU::sub12_sub13_sub14_sub15, 1653 AMDGPU::sub16_sub17_sub18_sub19, 1654 AMDGPU::sub20_sub21_sub22_sub23, 1655 AMDGPU::sub24_sub25_sub26_sub27, 1656 AMDGPU::sub28_sub29_sub30_sub31 1657 }; 1658 1659 static const int16_t Sub0_15_128[] = { 1660 AMDGPU::sub0_sub1_sub2_sub3, 1661 AMDGPU::sub4_sub5_sub6_sub7, 1662 AMDGPU::sub8_sub9_sub10_sub11, 1663 AMDGPU::sub12_sub13_sub14_sub15 1664 }; 1665 1666 static const int16_t Sub0_7_128[] = { 1667 AMDGPU::sub0_sub1_sub2_sub3, 1668 AMDGPU::sub4_sub5_sub6_sub7 1669 }; 1670 1671 switch (AMDGPU::getRegBitWidth(*RC->MC)) { 1672 case 128: 1673 return {}; 1674 case 256: 1675 return makeArrayRef(Sub0_7_128); 1676 case 512: 1677 return makeArrayRef(Sub0_15_128); 1678 case 1024: 1679 return makeArrayRef(Sub0_31_128); 1680 default: 1681 llvm_unreachable("unhandled register size"); 1682 } 1683 } 1684 1685 if (EltSize == 32) { 1686 static const int16_t Sub0_31_256[] = { 1687 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7, 1688 AMDGPU::sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 1689 AMDGPU::sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23, 1690 AMDGPU::sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31 1691 }; 1692 1693 static const int16_t Sub0_15_256[] = { 1694 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7, 1695 AMDGPU::sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15 1696 }; 1697 1698 switch (AMDGPU::getRegBitWidth(*RC->MC)) { 1699 case 256: 1700 return {}; 1701 case 512: 1702 return makeArrayRef(Sub0_15_256); 1703 case 1024: 1704 return makeArrayRef(Sub0_31_256); 1705 default: 1706 llvm_unreachable("unhandled register size"); 1707 } 1708 } 1709 1710 assert(EltSize == 64 && "unhandled elt size"); 1711 static const int16_t Sub0_31_512[] = { 1712 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 1713 AMDGPU::sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23_sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31 1714 }; 1715 1716 switch (AMDGPU::getRegBitWidth(*RC->MC)) { 1717 case 512: 1718 return {}; 1719 case 1024: 1720 return makeArrayRef(Sub0_31_512); 1721 default: 1722 llvm_unreachable("unhandled register size"); 1723 } 1724 } 1725 1726 const TargetRegisterClass* 1727 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI, 1728 unsigned Reg) const { 1729 if (Register::isVirtualRegister(Reg)) 1730 return MRI.getRegClass(Reg); 1731 1732 return getPhysRegClass(Reg); 1733 } 1734 1735 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI, 1736 unsigned Reg) const { 1737 const TargetRegisterClass * RC = getRegClassForReg(MRI, Reg); 1738 assert(RC && "Register class for the reg not found"); 1739 return hasVGPRs(RC); 1740 } 1741 1742 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI, 1743 unsigned Reg) const { 1744 const TargetRegisterClass * RC = getRegClassForReg(MRI, Reg); 1745 assert(RC && "Register class for the reg not found"); 1746 return hasAGPRs(RC); 1747 } 1748 1749 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI, 1750 const TargetRegisterClass *SrcRC, 1751 unsigned SubReg, 1752 const TargetRegisterClass *DstRC, 1753 unsigned DstSubReg, 1754 const TargetRegisterClass *NewRC, 1755 LiveIntervals &LIS) const { 1756 unsigned SrcSize = getRegSizeInBits(*SrcRC); 1757 unsigned DstSize = getRegSizeInBits(*DstRC); 1758 unsigned NewSize = getRegSizeInBits(*NewRC); 1759 1760 // Do not increase size of registers beyond dword, we would need to allocate 1761 // adjacent registers and constraint regalloc more than needed. 1762 1763 // Always allow dword coalescing. 1764 if (SrcSize <= 32 || DstSize <= 32) 1765 return true; 1766 1767 return NewSize <= DstSize || NewSize <= SrcSize; 1768 } 1769 1770 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 1771 MachineFunction &MF) const { 1772 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1773 1774 unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), 1775 MF.getFunction()); 1776 switch (RC->getID()) { 1777 default: 1778 return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF); 1779 case AMDGPU::VGPR_32RegClassID: 1780 return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF)); 1781 case AMDGPU::SGPR_32RegClassID: 1782 return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF)); 1783 } 1784 } 1785 1786 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 1787 unsigned Idx) const { 1788 if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 || 1789 Idx == AMDGPU::RegisterPressureSets::AGPR_32) 1790 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 1791 const_cast<MachineFunction &>(MF)); 1792 1793 if (Idx == AMDGPU::RegisterPressureSets::SReg_32) 1794 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, 1795 const_cast<MachineFunction &>(MF)); 1796 1797 llvm_unreachable("Unexpected register pressure set!"); 1798 } 1799 1800 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const { 1801 static const int Empty[] = { -1 }; 1802 1803 if (hasRegUnit(AMDGPU::M0, RegUnit)) 1804 return Empty; 1805 return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit); 1806 } 1807 1808 unsigned SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { 1809 // Not a callee saved register. 1810 return AMDGPU::SGPR30_SGPR31; 1811 } 1812 1813 const TargetRegisterClass * 1814 SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, 1815 const RegisterBank &RB, 1816 const MachineRegisterInfo &MRI) const { 1817 switch (Size) { 1818 case 1: { 1819 switch (RB.getID()) { 1820 case AMDGPU::VGPRRegBankID: 1821 return &AMDGPU::VGPR_32RegClass; 1822 case AMDGPU::VCCRegBankID: 1823 return isWave32 ? 1824 &AMDGPU::SReg_32_XM0_XEXECRegClass : &AMDGPU::SReg_64_XEXECRegClass; 1825 case AMDGPU::SGPRRegBankID: 1826 return &AMDGPU::SReg_32RegClass; 1827 default: 1828 llvm_unreachable("unknown register bank"); 1829 } 1830 } 1831 case 32: 1832 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VGPR_32RegClass : 1833 &AMDGPU::SReg_32RegClass; 1834 case 64: 1835 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_64RegClass : 1836 &AMDGPU::SReg_64RegClass; 1837 case 96: 1838 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_96RegClass : 1839 &AMDGPU::SReg_96RegClass; 1840 case 128: 1841 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_128RegClass : 1842 &AMDGPU::SGPR_128RegClass; 1843 case 160: 1844 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_160RegClass : 1845 &AMDGPU::SReg_160RegClass; 1846 case 256: 1847 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_256RegClass : 1848 &AMDGPU::SReg_256RegClass; 1849 case 512: 1850 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_512RegClass : 1851 &AMDGPU::SReg_512RegClass; 1852 case 1024: 1853 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_1024RegClass : 1854 &AMDGPU::SReg_1024RegClass; 1855 default: 1856 if (Size < 32) 1857 return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VGPR_32RegClass : 1858 &AMDGPU::SReg_32RegClass; 1859 return nullptr; 1860 } 1861 } 1862 1863 const TargetRegisterClass * 1864 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO, 1865 const MachineRegisterInfo &MRI) const { 1866 const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg()); 1867 if (const RegisterBank *RB = RCOrRB.dyn_cast<const RegisterBank*>()) 1868 return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB, MRI); 1869 1870 const TargetRegisterClass *RC = RCOrRB.get<const TargetRegisterClass*>(); 1871 return getAllocatableClass(RC); 1872 } 1873 1874 unsigned SIRegisterInfo::getVCC() const { 1875 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; 1876 } 1877 1878 const TargetRegisterClass * 1879 SIRegisterInfo::getRegClass(unsigned RCID) const { 1880 switch ((int)RCID) { 1881 case AMDGPU::SReg_1RegClassID: 1882 return getBoolRC(); 1883 case AMDGPU::SReg_1_XEXECRegClassID: 1884 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 1885 : &AMDGPU::SReg_64_XEXECRegClass; 1886 case -1: 1887 return nullptr; 1888 default: 1889 return AMDGPUGenRegisterInfo::getRegClass(RCID); 1890 } 1891 } 1892 1893 // Find reaching register definition 1894 MachineInstr *SIRegisterInfo::findReachingDef(unsigned Reg, unsigned SubReg, 1895 MachineInstr &Use, 1896 MachineRegisterInfo &MRI, 1897 LiveIntervals *LIS) const { 1898 auto &MDT = LIS->getAnalysis<MachineDominatorTree>(); 1899 SlotIndex UseIdx = LIS->getInstructionIndex(Use); 1900 SlotIndex DefIdx; 1901 1902 if (Register::isVirtualRegister(Reg)) { 1903 if (!LIS->hasInterval(Reg)) 1904 return nullptr; 1905 LiveInterval &LI = LIS->getInterval(Reg); 1906 LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg) 1907 : MRI.getMaxLaneMaskForVReg(Reg); 1908 VNInfo *V = nullptr; 1909 if (LI.hasSubRanges()) { 1910 for (auto &S : LI.subranges()) { 1911 if ((S.LaneMask & SubLanes) == SubLanes) { 1912 V = S.getVNInfoAt(UseIdx); 1913 break; 1914 } 1915 } 1916 } else { 1917 V = LI.getVNInfoAt(UseIdx); 1918 } 1919 if (!V) 1920 return nullptr; 1921 DefIdx = V->def; 1922 } else { 1923 // Find last def. 1924 for (MCRegUnitIterator Units(Reg, this); Units.isValid(); ++Units) { 1925 LiveRange &LR = LIS->getRegUnit(*Units); 1926 if (VNInfo *V = LR.getVNInfoAt(UseIdx)) { 1927 if (!DefIdx.isValid() || 1928 MDT.dominates(LIS->getInstructionFromIndex(DefIdx), 1929 LIS->getInstructionFromIndex(V->def))) 1930 DefIdx = V->def; 1931 } else { 1932 return nullptr; 1933 } 1934 } 1935 } 1936 1937 MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx); 1938 1939 if (!Def || !MDT.dominates(Def, &Use)) 1940 return nullptr; 1941 1942 assert(Def->modifiesRegister(Reg, this)); 1943 1944 return Def; 1945 } 1946