1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI implementation of the TargetRegisterInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIRegisterInfo.h" 15 #include "AMDGPURegisterBankInfo.h" 16 #include "AMDGPUSubtarget.h" 17 #include "SIInstrInfo.h" 18 #include "SIMachineFunctionInfo.h" 19 #include "MCTargetDesc/AMDGPUInstPrinter.h" 20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include "llvm/CodeGen/MachineDominators.h" 23 #include "llvm/CodeGen/MachineFrameInfo.h" 24 #include "llvm/CodeGen/MachineInstrBuilder.h" 25 #include "llvm/CodeGen/RegisterScavenging.h" 26 #include "llvm/CodeGen/SlotIndexes.h" 27 #include "llvm/IR/Function.h" 28 #include "llvm/IR/LLVMContext.h" 29 30 using namespace llvm; 31 32 #define GET_REGINFO_TARGET_DESC 33 #include "AMDGPUGenRegisterInfo.inc" 34 35 static cl::opt<bool> EnableSpillSGPRToVGPR( 36 "amdgpu-spill-sgpr-to-vgpr", 37 cl::desc("Enable spilling VGPRs to SGPRs"), 38 cl::ReallyHidden, 39 cl::init(true)); 40 41 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) 42 : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST), 43 SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) { 44 45 assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 && 46 getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) && 47 (getSubRegIndexLaneMask(AMDGPU::lo16) | 48 getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() == 49 getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() && 50 "getNumCoveredRegs() will not work with generated subreg masks!"); 51 52 RegPressureIgnoredUnits.resize(getNumRegUnits()); 53 RegPressureIgnoredUnits.set(*MCRegUnitIterator(AMDGPU::M0, this)); 54 for (auto Reg : AMDGPU::VGPR_HI16RegClass) 55 RegPressureIgnoredUnits.set(*MCRegUnitIterator(Reg, this)); 56 } 57 58 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, 59 MCRegister Reg) const { 60 MCRegAliasIterator R(Reg, this, true); 61 62 for (; R.isValid(); ++R) 63 Reserved.set(*R); 64 } 65 66 // Forced to be here by one .inc 67 const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs( 68 const MachineFunction *MF) const { 69 CallingConv::ID CC = MF->getFunction().getCallingConv(); 70 switch (CC) { 71 case CallingConv::C: 72 case CallingConv::Fast: 73 case CallingConv::Cold: 74 return CSR_AMDGPU_HighRegs_SaveList; 75 default: { 76 // Dummy to not crash RegisterClassInfo. 77 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister; 78 return &NoCalleeSavedReg; 79 } 80 } 81 } 82 83 const MCPhysReg * 84 SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const { 85 return nullptr; 86 } 87 88 const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF, 89 CallingConv::ID CC) const { 90 switch (CC) { 91 case CallingConv::C: 92 case CallingConv::Fast: 93 case CallingConv::Cold: 94 return CSR_AMDGPU_HighRegs_RegMask; 95 default: 96 return nullptr; 97 } 98 } 99 100 Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const { 101 const SIFrameLowering *TFI = 102 MF.getSubtarget<GCNSubtarget>().getFrameLowering(); 103 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); 104 // During ISel lowering we always reserve the stack pointer in entry 105 // functions, but never actually want to reference it when accessing our own 106 // frame. If we need a frame pointer we use it, but otherwise we can just use 107 // an immediate "0" which we represent by returning NoRegister. 108 if (FuncInfo->isEntryFunction()) { 109 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register(); 110 } 111 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() 112 : FuncInfo->getStackPtrOffsetReg(); 113 } 114 115 const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const { 116 return CSR_AMDGPU_AllVGPRs_RegMask; 117 } 118 119 const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const { 120 return CSR_AMDGPU_AllAllocatableSRegs_RegMask; 121 } 122 123 // FIXME: TableGen should generate something to make this manageable for all 124 // register classes. At a minimum we could use the opposite of 125 // composeSubRegIndices and go up from the base 32-bit subreg. 126 unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel, 127 unsigned NumRegs) { 128 // Table of NumRegs sized pieces at every 32-bit offset. 129 static const uint16_t SubRegFromChannelTable[][32] = { 130 {AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 131 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 132 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 133 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 134 AMDGPU::sub16, AMDGPU::sub17, AMDGPU::sub18, AMDGPU::sub19, 135 AMDGPU::sub20, AMDGPU::sub21, AMDGPU::sub22, AMDGPU::sub23, 136 AMDGPU::sub24, AMDGPU::sub25, AMDGPU::sub26, AMDGPU::sub27, 137 AMDGPU::sub28, AMDGPU::sub29, AMDGPU::sub30, AMDGPU::sub31}, 138 {AMDGPU::sub0_sub1, AMDGPU::sub1_sub2, AMDGPU::sub2_sub3, 139 AMDGPU::sub3_sub4, AMDGPU::sub4_sub5, AMDGPU::sub5_sub6, 140 AMDGPU::sub6_sub7, AMDGPU::sub7_sub8, AMDGPU::sub8_sub9, 141 AMDGPU::sub9_sub10, AMDGPU::sub10_sub11, AMDGPU::sub11_sub12, 142 AMDGPU::sub12_sub13, AMDGPU::sub13_sub14, AMDGPU::sub14_sub15, 143 AMDGPU::sub15_sub16, AMDGPU::sub16_sub17, AMDGPU::sub17_sub18, 144 AMDGPU::sub18_sub19, AMDGPU::sub19_sub20, AMDGPU::sub20_sub21, 145 AMDGPU::sub21_sub22, AMDGPU::sub22_sub23, AMDGPU::sub23_sub24, 146 AMDGPU::sub24_sub25, AMDGPU::sub25_sub26, AMDGPU::sub26_sub27, 147 AMDGPU::sub27_sub28, AMDGPU::sub28_sub29, AMDGPU::sub29_sub30, 148 AMDGPU::sub30_sub31, AMDGPU::NoSubRegister}, 149 {AMDGPU::sub0_sub1_sub2, AMDGPU::sub1_sub2_sub3, 150 AMDGPU::sub2_sub3_sub4, AMDGPU::sub3_sub4_sub5, 151 AMDGPU::sub4_sub5_sub6, AMDGPU::sub5_sub6_sub7, 152 AMDGPU::sub6_sub7_sub8, AMDGPU::sub7_sub8_sub9, 153 AMDGPU::sub8_sub9_sub10, AMDGPU::sub9_sub10_sub11, 154 AMDGPU::sub10_sub11_sub12, AMDGPU::sub11_sub12_sub13, 155 AMDGPU::sub12_sub13_sub14, AMDGPU::sub13_sub14_sub15, 156 AMDGPU::sub14_sub15_sub16, AMDGPU::sub15_sub16_sub17, 157 AMDGPU::sub16_sub17_sub18, AMDGPU::sub17_sub18_sub19, 158 AMDGPU::sub18_sub19_sub20, AMDGPU::sub19_sub20_sub21, 159 AMDGPU::sub20_sub21_sub22, AMDGPU::sub21_sub22_sub23, 160 AMDGPU::sub22_sub23_sub24, AMDGPU::sub23_sub24_sub25, 161 AMDGPU::sub24_sub25_sub26, AMDGPU::sub25_sub26_sub27, 162 AMDGPU::sub26_sub27_sub28, AMDGPU::sub27_sub28_sub29, 163 AMDGPU::sub28_sub29_sub30, AMDGPU::sub29_sub30_sub31, 164 AMDGPU::NoSubRegister, AMDGPU::NoSubRegister}, 165 {AMDGPU::sub0_sub1_sub2_sub3, AMDGPU::sub1_sub2_sub3_sub4, 166 AMDGPU::sub2_sub3_sub4_sub5, AMDGPU::sub3_sub4_sub5_sub6, 167 AMDGPU::sub4_sub5_sub6_sub7, AMDGPU::sub5_sub6_sub7_sub8, 168 AMDGPU::sub6_sub7_sub8_sub9, AMDGPU::sub7_sub8_sub9_sub10, 169 AMDGPU::sub8_sub9_sub10_sub11, AMDGPU::sub9_sub10_sub11_sub12, 170 AMDGPU::sub10_sub11_sub12_sub13, AMDGPU::sub11_sub12_sub13_sub14, 171 AMDGPU::sub12_sub13_sub14_sub15, AMDGPU::sub13_sub14_sub15_sub16, 172 AMDGPU::sub14_sub15_sub16_sub17, AMDGPU::sub15_sub16_sub17_sub18, 173 AMDGPU::sub16_sub17_sub18_sub19, AMDGPU::sub17_sub18_sub19_sub20, 174 AMDGPU::sub18_sub19_sub20_sub21, AMDGPU::sub19_sub20_sub21_sub22, 175 AMDGPU::sub20_sub21_sub22_sub23, AMDGPU::sub21_sub22_sub23_sub24, 176 AMDGPU::sub22_sub23_sub24_sub25, AMDGPU::sub23_sub24_sub25_sub26, 177 AMDGPU::sub24_sub25_sub26_sub27, AMDGPU::sub25_sub26_sub27_sub28, 178 AMDGPU::sub26_sub27_sub28_sub29, AMDGPU::sub27_sub28_sub29_sub30, 179 AMDGPU::sub28_sub29_sub30_sub31, AMDGPU::NoSubRegister, 180 AMDGPU::NoSubRegister, AMDGPU::NoSubRegister}}; 181 182 const unsigned NumRegIndex = NumRegs - 1; 183 184 assert(NumRegIndex < array_lengthof(SubRegFromChannelTable) && 185 "Not implemented"); 186 assert(Channel < array_lengthof(SubRegFromChannelTable[0])); 187 return SubRegFromChannelTable[NumRegIndex][Channel]; 188 } 189 190 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg( 191 const MachineFunction &MF) const { 192 unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4; 193 MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); 194 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass); 195 } 196 197 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 198 BitVector Reserved(getNumRegs()); 199 200 // EXEC_LO and EXEC_HI could be allocated and used as regular register, but 201 // this seems likely to result in bugs, so I'm marking them as reserved. 202 reserveRegisterTuples(Reserved, AMDGPU::EXEC); 203 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); 204 205 // M0 has to be reserved so that llvm accepts it as a live-in into a block. 206 reserveRegisterTuples(Reserved, AMDGPU::M0); 207 208 // Reserve src_vccz, src_execz, src_scc. 209 reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ); 210 reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ); 211 reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC); 212 213 // Reserve the memory aperture registers. 214 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); 215 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); 216 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); 217 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); 218 219 // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen. 220 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); 221 222 // Reserve xnack_mask registers - support is not implemented in Codegen. 223 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); 224 225 // Reserve lds_direct register - support is not implemented in Codegen. 226 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); 227 228 // Reserve Trap Handler registers - support is not implemented in Codegen. 229 reserveRegisterTuples(Reserved, AMDGPU::TBA); 230 reserveRegisterTuples(Reserved, AMDGPU::TMA); 231 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); 232 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); 233 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); 234 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); 235 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); 236 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); 237 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); 238 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); 239 240 // Reserve null register - it shall never be allocated 241 reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL); 242 243 // Disallow vcc_hi allocation in wave32. It may be allocated but most likely 244 // will result in bugs. 245 if (isWave32) { 246 Reserved.set(AMDGPU::VCC); 247 Reserved.set(AMDGPU::VCC_HI); 248 } 249 250 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); 251 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 252 for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { 253 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 254 reserveRegisterTuples(Reserved, Reg); 255 } 256 257 unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF); 258 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); 259 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) { 260 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); 261 reserveRegisterTuples(Reserved, Reg); 262 Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 263 reserveRegisterTuples(Reserved, Reg); 264 } 265 266 for (auto Reg : AMDGPU::SReg_32RegClass) { 267 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 268 Register Low = getSubReg(Reg, AMDGPU::lo16); 269 // This is to prevent BB vcc liveness errors. 270 if (!AMDGPU::SGPR_LO16RegClass.contains(Low)) 271 Reserved.set(Low); 272 } 273 274 for (auto Reg : AMDGPU::AGPR_32RegClass) { 275 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 276 } 277 278 // Reserve all the rest AGPRs if there are no instructions to use it. 279 if (!ST.hasMAIInsts()) { 280 for (unsigned i = 0; i < MaxNumVGPRs; ++i) { 281 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 282 reserveRegisterTuples(Reserved, Reg); 283 } 284 } 285 286 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 287 288 unsigned ScratchRSrcReg = MFI->getScratchRSrcReg(); 289 if (ScratchRSrcReg != AMDGPU::NoRegister) { 290 // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need 291 // to spill. 292 // TODO: May need to reserve a VGPR if doing LDS spilling. 293 reserveRegisterTuples(Reserved, ScratchRSrcReg); 294 } 295 296 // We have to assume the SP is needed in case there are calls in the function, 297 // which is detected after the function is lowered. If we aren't really going 298 // to need SP, don't bother reserving it. 299 MCRegister StackPtrReg = MFI->getStackPtrOffsetReg(); 300 301 if (StackPtrReg) { 302 reserveRegisterTuples(Reserved, StackPtrReg); 303 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg)); 304 } 305 306 MCRegister FrameReg = MFI->getFrameOffsetReg(); 307 if (FrameReg) { 308 reserveRegisterTuples(Reserved, FrameReg); 309 assert(!isSubRegister(ScratchRSrcReg, FrameReg)); 310 } 311 312 for (MCRegister Reg : MFI->WWMReservedRegs) { 313 reserveRegisterTuples(Reserved, Reg); 314 } 315 316 // FIXME: Stop using reserved registers for this. 317 for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs()) 318 reserveRegisterTuples(Reserved, Reg); 319 320 for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs()) 321 reserveRegisterTuples(Reserved, Reg); 322 323 if (MFI->VGPRReservedForSGPRSpill) 324 for (auto SSpill : MFI->getSGPRSpillVGPRs()) 325 reserveRegisterTuples(Reserved, SSpill.VGPR); 326 327 return Reserved; 328 } 329 330 bool SIRegisterInfo::canRealignStack(const MachineFunction &MF) const { 331 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 332 // On entry, the base address is 0, so it can't possibly need any more 333 // alignment. 334 335 // FIXME: Should be able to specify the entry frame alignment per calling 336 // convention instead. 337 if (Info->isEntryFunction()) 338 return false; 339 340 return TargetRegisterInfo::canRealignStack(MF); 341 } 342 343 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 344 const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>(); 345 if (Info->isEntryFunction()) { 346 const MachineFrameInfo &MFI = Fn.getFrameInfo(); 347 return MFI.hasStackObjects() || MFI.hasCalls(); 348 } 349 350 // May need scavenger for dealing with callee saved registers. 351 return true; 352 } 353 354 bool SIRegisterInfo::requiresFrameIndexScavenging( 355 const MachineFunction &MF) const { 356 // Do not use frame virtual registers. They used to be used for SGPRs, but 357 // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the 358 // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a 359 // spill. 360 return false; 361 } 362 363 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( 364 const MachineFunction &MF) const { 365 const MachineFrameInfo &MFI = MF.getFrameInfo(); 366 return MFI.hasStackObjects(); 367 } 368 369 bool SIRegisterInfo::requiresVirtualBaseRegisters( 370 const MachineFunction &) const { 371 // There are no special dedicated stack or frame pointers. 372 return true; 373 } 374 375 int64_t SIRegisterInfo::getMUBUFInstrOffset(const MachineInstr *MI) const { 376 assert(SIInstrInfo::isMUBUF(*MI)); 377 378 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 379 AMDGPU::OpName::offset); 380 return MI->getOperand(OffIdx).getImm(); 381 } 382 383 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI, 384 int Idx) const { 385 if (!SIInstrInfo::isMUBUF(*MI)) 386 return 0; 387 388 assert(Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 389 AMDGPU::OpName::vaddr) && 390 "Should never see frame index on non-address operand"); 391 392 return getMUBUFInstrOffset(MI); 393 } 394 395 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { 396 if (!MI->mayLoadOrStore()) 397 return false; 398 399 int64_t FullOffset = Offset + getMUBUFInstrOffset(MI); 400 401 return !isUInt<12>(FullOffset); 402 } 403 404 void SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 405 Register BaseReg, 406 int FrameIdx, 407 int64_t Offset) const { 408 MachineBasicBlock::iterator Ins = MBB->begin(); 409 DebugLoc DL; // Defaults to "unknown" 410 411 if (Ins != MBB->end()) 412 DL = Ins->getDebugLoc(); 413 414 MachineFunction *MF = MBB->getParent(); 415 const SIInstrInfo *TII = ST.getInstrInfo(); 416 417 if (Offset == 0) { 418 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg) 419 .addFrameIndex(FrameIdx); 420 return; 421 } 422 423 MachineRegisterInfo &MRI = MF->getRegInfo(); 424 Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 425 426 Register FIReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 427 428 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 429 .addImm(Offset); 430 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), FIReg) 431 .addFrameIndex(FrameIdx); 432 433 TII->getAddNoCarry(*MBB, Ins, DL, BaseReg) 434 .addReg(OffsetReg, RegState::Kill) 435 .addReg(FIReg) 436 .addImm(0); // clamp bit 437 } 438 439 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, 440 int64_t Offset) const { 441 const SIInstrInfo *TII = ST.getInstrInfo(); 442 443 #ifndef NDEBUG 444 // FIXME: Is it possible to be storing a frame index to itself? 445 bool SeenFI = false; 446 for (const MachineOperand &MO: MI.operands()) { 447 if (MO.isFI()) { 448 if (SeenFI) 449 llvm_unreachable("should not see multiple frame indices"); 450 451 SeenFI = true; 452 } 453 } 454 #endif 455 456 MachineOperand *FIOp = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr); 457 #ifndef NDEBUG 458 MachineBasicBlock *MBB = MI.getParent(); 459 MachineFunction *MF = MBB->getParent(); 460 #endif 461 assert(FIOp && FIOp->isFI() && "frame index must be address operand"); 462 assert(TII->isMUBUF(MI)); 463 assert(TII->getNamedOperand(MI, AMDGPU::OpName::soffset)->getReg() == 464 MF->getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg() && 465 "should only be seeing stack pointer offset relative FrameIndex"); 466 467 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); 468 int64_t NewOffset = OffsetOp->getImm() + Offset; 469 assert(isUInt<12>(NewOffset) && "offset should be legal"); 470 471 FIOp->ChangeToRegister(BaseReg, false); 472 OffsetOp->setImm(NewOffset); 473 } 474 475 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 476 Register BaseReg, 477 int64_t Offset) const { 478 if (!SIInstrInfo::isMUBUF(*MI)) 479 return false; 480 481 int64_t NewOffset = Offset + getMUBUFInstrOffset(MI); 482 483 return isUInt<12>(NewOffset); 484 } 485 486 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass( 487 const MachineFunction &MF, unsigned Kind) const { 488 // This is inaccurate. It depends on the instruction and address space. The 489 // only place where we should hit this is for dealing with frame indexes / 490 // private accesses, so this is correct in that case. 491 return &AMDGPU::VGPR_32RegClass; 492 } 493 494 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 495 496 switch (Op) { 497 case AMDGPU::SI_SPILL_S1024_SAVE: 498 case AMDGPU::SI_SPILL_S1024_RESTORE: 499 case AMDGPU::SI_SPILL_V1024_SAVE: 500 case AMDGPU::SI_SPILL_V1024_RESTORE: 501 case AMDGPU::SI_SPILL_A1024_SAVE: 502 case AMDGPU::SI_SPILL_A1024_RESTORE: 503 return 32; 504 case AMDGPU::SI_SPILL_S512_SAVE: 505 case AMDGPU::SI_SPILL_S512_RESTORE: 506 case AMDGPU::SI_SPILL_V512_SAVE: 507 case AMDGPU::SI_SPILL_V512_RESTORE: 508 case AMDGPU::SI_SPILL_A512_SAVE: 509 case AMDGPU::SI_SPILL_A512_RESTORE: 510 return 16; 511 case AMDGPU::SI_SPILL_S256_SAVE: 512 case AMDGPU::SI_SPILL_S256_RESTORE: 513 case AMDGPU::SI_SPILL_V256_SAVE: 514 case AMDGPU::SI_SPILL_V256_RESTORE: 515 return 8; 516 case AMDGPU::SI_SPILL_S160_SAVE: 517 case AMDGPU::SI_SPILL_S160_RESTORE: 518 case AMDGPU::SI_SPILL_V160_SAVE: 519 case AMDGPU::SI_SPILL_V160_RESTORE: 520 return 5; 521 case AMDGPU::SI_SPILL_S128_SAVE: 522 case AMDGPU::SI_SPILL_S128_RESTORE: 523 case AMDGPU::SI_SPILL_V128_SAVE: 524 case AMDGPU::SI_SPILL_V128_RESTORE: 525 case AMDGPU::SI_SPILL_A128_SAVE: 526 case AMDGPU::SI_SPILL_A128_RESTORE: 527 return 4; 528 case AMDGPU::SI_SPILL_S96_SAVE: 529 case AMDGPU::SI_SPILL_S96_RESTORE: 530 case AMDGPU::SI_SPILL_V96_SAVE: 531 case AMDGPU::SI_SPILL_V96_RESTORE: 532 return 3; 533 case AMDGPU::SI_SPILL_S64_SAVE: 534 case AMDGPU::SI_SPILL_S64_RESTORE: 535 case AMDGPU::SI_SPILL_V64_SAVE: 536 case AMDGPU::SI_SPILL_V64_RESTORE: 537 case AMDGPU::SI_SPILL_A64_SAVE: 538 case AMDGPU::SI_SPILL_A64_RESTORE: 539 return 2; 540 case AMDGPU::SI_SPILL_S32_SAVE: 541 case AMDGPU::SI_SPILL_S32_RESTORE: 542 case AMDGPU::SI_SPILL_V32_SAVE: 543 case AMDGPU::SI_SPILL_V32_RESTORE: 544 case AMDGPU::SI_SPILL_A32_SAVE: 545 case AMDGPU::SI_SPILL_A32_RESTORE: 546 return 1; 547 default: llvm_unreachable("Invalid spill opcode"); 548 } 549 } 550 551 static int getOffsetMUBUFStore(unsigned Opc) { 552 switch (Opc) { 553 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 554 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; 555 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: 556 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; 557 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: 558 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; 559 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: 560 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; 561 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: 562 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; 563 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: 564 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; 565 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: 566 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; 567 default: 568 return -1; 569 } 570 } 571 572 static int getOffsetMUBUFLoad(unsigned Opc) { 573 switch (Opc) { 574 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 575 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 576 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: 577 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; 578 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: 579 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; 580 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: 581 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; 582 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: 583 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; 584 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: 585 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; 586 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: 587 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; 588 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: 589 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; 590 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: 591 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; 592 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: 593 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; 594 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: 595 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; 596 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: 597 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; 598 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: 599 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; 600 default: 601 return -1; 602 } 603 } 604 605 static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST, 606 MachineBasicBlock::iterator MI, 607 int Index, 608 unsigned Lane, 609 unsigned ValueReg, 610 bool IsKill) { 611 MachineBasicBlock *MBB = MI->getParent(); 612 MachineFunction *MF = MI->getParent()->getParent(); 613 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 614 const SIInstrInfo *TII = ST.getInstrInfo(); 615 616 MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane); 617 618 if (Reg == AMDGPU::NoRegister) 619 return MachineInstrBuilder(); 620 621 bool IsStore = MI->mayStore(); 622 MachineRegisterInfo &MRI = MF->getRegInfo(); 623 auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo()); 624 625 unsigned Dst = IsStore ? Reg : ValueReg; 626 unsigned Src = IsStore ? ValueReg : Reg; 627 unsigned Opc = (IsStore ^ TRI->isVGPR(MRI, Reg)) ? AMDGPU::V_ACCVGPR_WRITE_B32 628 : AMDGPU::V_ACCVGPR_READ_B32; 629 630 return BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(Opc), Dst) 631 .addReg(Src, getKillRegState(IsKill)); 632 } 633 634 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not 635 // need to handle the case where an SGPR may need to be spilled while spilling. 636 static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST, 637 MachineFrameInfo &MFI, 638 MachineBasicBlock::iterator MI, 639 int Index, 640 int64_t Offset) { 641 const SIInstrInfo *TII = ST.getInstrInfo(); 642 MachineBasicBlock *MBB = MI->getParent(); 643 const DebugLoc &DL = MI->getDebugLoc(); 644 bool IsStore = MI->mayStore(); 645 646 unsigned Opc = MI->getOpcode(); 647 int LoadStoreOp = IsStore ? 648 getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc); 649 if (LoadStoreOp == -1) 650 return false; 651 652 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); 653 if (spillVGPRtoAGPR(ST, MI, Index, 0, Reg->getReg(), false).getInstr()) 654 return true; 655 656 MachineInstrBuilder NewMI = 657 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 658 .add(*Reg) 659 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) 660 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) 661 .addImm(Offset) 662 .addImm(0) // glc 663 .addImm(0) // slc 664 .addImm(0) // tfe 665 .addImm(0) // dlc 666 .addImm(0) // swz 667 .cloneMemRefs(*MI); 668 669 const MachineOperand *VDataIn = TII->getNamedOperand(*MI, 670 AMDGPU::OpName::vdata_in); 671 if (VDataIn) 672 NewMI.add(*VDataIn); 673 return true; 674 } 675 676 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI, 677 unsigned LoadStoreOp, 678 int Index, 679 Register ValueReg, 680 bool IsKill, 681 MCRegister ScratchRsrcReg, 682 MCRegister ScratchOffsetReg, 683 int64_t InstOffset, 684 MachineMemOperand *MMO, 685 RegScavenger *RS) const { 686 MachineBasicBlock *MBB = MI->getParent(); 687 MachineFunction *MF = MI->getParent()->getParent(); 688 const SIInstrInfo *TII = ST.getInstrInfo(); 689 const MachineFrameInfo &MFI = MF->getFrameInfo(); 690 691 const MCInstrDesc &Desc = TII->get(LoadStoreOp); 692 const DebugLoc &DL = MI->getDebugLoc(); 693 bool IsStore = Desc.mayStore(); 694 695 bool Scavenged = false; 696 MCRegister SOffset = ScratchOffsetReg; 697 698 const unsigned EltSize = 4; 699 const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg); 700 unsigned NumSubRegs = AMDGPU::getRegBitWidth(RC->getID()) / (EltSize * CHAR_BIT); 701 unsigned Size = NumSubRegs * EltSize; 702 int64_t Offset = InstOffset + MFI.getObjectOffset(Index); 703 int64_t ScratchOffsetRegDelta = 0; 704 705 Align Alignment = MFI.getObjectAlign(Index); 706 const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo(); 707 708 Register TmpReg = 709 hasAGPRs(RC) ? TII->getNamedOperand(*MI, AMDGPU::OpName::tmp)->getReg() 710 : Register(); 711 712 assert((Offset % EltSize) == 0 && "unexpected VGPR spill offset"); 713 714 if (!isUInt<12>(Offset + Size - EltSize)) { 715 SOffset = MCRegister(); 716 717 // We currently only support spilling VGPRs to EltSize boundaries, meaning 718 // we can simplify the adjustment of Offset here to just scale with 719 // WavefrontSize. 720 Offset *= ST.getWavefrontSize(); 721 722 // We don't have access to the register scavenger if this function is called 723 // during PEI::scavengeFrameVirtualRegs(). 724 if (RS) 725 SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false); 726 727 if (!SOffset) { 728 if (!ScratchOffsetReg) { 729 report_fatal_error("could not scavenge SGPR to spill in entry function"); 730 } 731 // There are no free SGPRs, and since we are in the process of spilling 732 // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true 733 // on SI/CI and on VI it is true until we implement spilling using scalar 734 // stores), we have no way to free up an SGPR. Our solution here is to 735 // add the offset directly to the ScratchOffset register, and then 736 // subtract the offset after the spill to return ScratchOffset to it's 737 // original value. 738 SOffset = ScratchOffsetReg; 739 ScratchOffsetRegDelta = Offset; 740 } else { 741 Scavenged = true; 742 } 743 744 if (ScratchOffsetReg == AMDGPU::NoRegister) { 745 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset) 746 .addImm(Offset); 747 } else { 748 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset) 749 .addReg(ScratchOffsetReg) 750 .addImm(Offset); 751 } 752 753 Offset = 0; 754 } 755 756 for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += EltSize) { 757 Register SubReg = NumSubRegs == 1 758 ? Register(ValueReg) 759 : getSubReg(ValueReg, getSubRegFromChannel(i)); 760 761 unsigned SOffsetRegState = 0; 762 unsigned SrcDstRegState = getDefRegState(!IsStore); 763 if (i + 1 == e) { 764 SOffsetRegState |= getKillRegState(Scavenged); 765 // The last implicit use carries the "Kill" flag. 766 SrcDstRegState |= getKillRegState(IsKill); 767 } 768 769 auto MIB = spillVGPRtoAGPR(ST, MI, Index, i, SubReg, IsKill); 770 771 if (!MIB.getInstr()) { 772 unsigned FinalReg = SubReg; 773 if (TmpReg != AMDGPU::NoRegister) { 774 if (IsStore) 775 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_READ_B32), TmpReg) 776 .addReg(SubReg, getKillRegState(IsKill)); 777 SubReg = TmpReg; 778 } 779 780 MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(EltSize * i); 781 MachineMemOperand *NewMMO = 782 MF->getMachineMemOperand(PInfo, MMO->getFlags(), EltSize, 783 commonAlignment(Alignment, EltSize * i)); 784 785 MIB = BuildMI(*MBB, MI, DL, Desc) 786 .addReg(SubReg, 787 getDefRegState(!IsStore) | getKillRegState(IsKill)) 788 .addReg(ScratchRsrcReg); 789 if (SOffset == AMDGPU::NoRegister) { 790 MIB.addImm(0); 791 } else { 792 MIB.addReg(SOffset, SOffsetRegState); 793 } 794 MIB.addImm(Offset) 795 .addImm(0) // glc 796 .addImm(0) // slc 797 .addImm(0) // tfe 798 .addImm(0) // dlc 799 .addImm(0) // swz 800 .addMemOperand(NewMMO); 801 802 if (!IsStore && TmpReg != AMDGPU::NoRegister) 803 MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32), 804 FinalReg) 805 .addReg(TmpReg, RegState::Kill); 806 } 807 808 if (NumSubRegs > 1) 809 MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState); 810 } 811 812 if (ScratchOffsetRegDelta != 0) { 813 // Subtract the offset we added to the ScratchOffset register. 814 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScratchOffsetReg) 815 .addReg(ScratchOffsetReg) 816 .addImm(ScratchOffsetRegDelta); 817 } 818 } 819 820 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, 821 int Index, 822 RegScavenger *RS, 823 bool OnlyToVGPR) const { 824 MachineBasicBlock *MBB = MI->getParent(); 825 MachineFunction *MF = MBB->getParent(); 826 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 827 DenseSet<unsigned> SGPRSpillVGPRDefinedSet; 828 829 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 830 = MFI->getSGPRToVGPRSpills(Index); 831 bool SpillToVGPR = !VGPRSpills.empty(); 832 if (OnlyToVGPR && !SpillToVGPR) 833 return false; 834 835 const SIInstrInfo *TII = ST.getInstrInfo(); 836 837 Register SuperReg = MI->getOperand(0).getReg(); 838 bool IsKill = MI->getOperand(0).isKill(); 839 const DebugLoc &DL = MI->getDebugLoc(); 840 841 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 842 843 assert(SpillToVGPR || (SuperReg != MFI->getStackPtrOffsetReg() && 844 SuperReg != MFI->getFrameOffsetReg())); 845 846 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 847 848 unsigned EltSize = 4; 849 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 850 851 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 852 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 853 854 // Scavenged temporary VGPR to use. It must be scavenged once for any number 855 // of spilled subregs. 856 Register TmpVGPR; 857 858 // SubReg carries the "Kill" flag when SubReg == SuperReg. 859 unsigned SubKillState = getKillRegState((NumSubRegs == 1) && IsKill); 860 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 861 Register SubReg = 862 NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]); 863 864 if (SpillToVGPR) { 865 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 866 867 // During SGPR spilling to VGPR, determine if the VGPR is defined. The 868 // only circumstance in which we say it is undefined is when it is the 869 // first spill to this VGPR in the first basic block. 870 bool VGPRDefined = true; 871 if (MBB == &MF->front()) 872 VGPRDefined = !SGPRSpillVGPRDefinedSet.insert(Spill.VGPR).second; 873 874 // Mark the "old value of vgpr" input undef only if this is the first sgpr 875 // spill to this specific vgpr in the first basic block. 876 BuildMI(*MBB, MI, DL, 877 TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32), 878 Spill.VGPR) 879 .addReg(SubReg, getKillRegState(IsKill)) 880 .addImm(Spill.Lane) 881 .addReg(Spill.VGPR, VGPRDefined ? 0 : RegState::Undef); 882 883 // FIXME: Since this spills to another register instead of an actual 884 // frame index, we should delete the frame index when all references to 885 // it are fixed. 886 } else { 887 // XXX - Can to VGPR spill fail for some subregisters but not others? 888 if (OnlyToVGPR) 889 return false; 890 891 // Spill SGPR to a frame index. 892 if (!TmpVGPR.isValid()) 893 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 894 895 MachineInstrBuilder Mov 896 = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 897 .addReg(SubReg, SubKillState); 898 899 // There could be undef components of a spilled super register. 900 // TODO: Can we detect this and skip the spill? 901 if (NumSubRegs > 1) { 902 // The last implicit use of the SuperReg carries the "Kill" flag. 903 unsigned SuperKillState = 0; 904 if (i + 1 == e) 905 SuperKillState |= getKillRegState(IsKill); 906 Mov.addReg(SuperReg, RegState::Implicit | SuperKillState); 907 } 908 909 Align Alignment = FrameInfo.getObjectAlign(Index); 910 MachinePointerInfo PtrInfo 911 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 912 MachineMemOperand *MMO = 913 MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, EltSize, 914 commonAlignment(Alignment, EltSize * i)); 915 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_SAVE)) 916 .addReg(TmpVGPR, RegState::Kill) // src 917 .addFrameIndex(Index) // vaddr 918 .addReg(MFI->getScratchRSrcReg()) // srrsrc 919 .addReg(MFI->getStackPtrOffsetReg()) // soffset 920 .addImm(i * 4) // offset 921 .addMemOperand(MMO); 922 } 923 } 924 925 MI->eraseFromParent(); 926 MFI->addToSpilledSGPRs(NumSubRegs); 927 return true; 928 } 929 930 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, 931 int Index, 932 RegScavenger *RS, 933 bool OnlyToVGPR) const { 934 MachineFunction *MF = MI->getParent()->getParent(); 935 MachineBasicBlock *MBB = MI->getParent(); 936 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 937 938 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 939 = MFI->getSGPRToVGPRSpills(Index); 940 bool SpillToVGPR = !VGPRSpills.empty(); 941 if (OnlyToVGPR && !SpillToVGPR) 942 return false; 943 944 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 945 const SIInstrInfo *TII = ST.getInstrInfo(); 946 const DebugLoc &DL = MI->getDebugLoc(); 947 948 Register SuperReg = MI->getOperand(0).getReg(); 949 950 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 951 952 unsigned EltSize = 4; 953 954 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 955 956 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 957 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 958 959 Register TmpVGPR; 960 961 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 962 Register SubReg = 963 NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]); 964 965 if (SpillToVGPR) { 966 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 967 auto MIB = 968 BuildMI(*MBB, MI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32), 969 SubReg) 970 .addReg(Spill.VGPR) 971 .addImm(Spill.Lane); 972 973 if (NumSubRegs > 1 && i == 0) 974 MIB.addReg(SuperReg, RegState::ImplicitDefine); 975 } else { 976 if (OnlyToVGPR) 977 return false; 978 979 // Restore SGPR from a stack slot. 980 // FIXME: We should use S_LOAD_DWORD here for VI. 981 if (!TmpVGPR.isValid()) 982 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 983 Align Alignment = FrameInfo.getObjectAlign(Index); 984 985 MachinePointerInfo PtrInfo 986 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 987 988 MachineMemOperand *MMO = 989 MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, EltSize, 990 commonAlignment(Alignment, EltSize * i)); 991 992 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_RESTORE), TmpVGPR) 993 .addFrameIndex(Index) // vaddr 994 .addReg(MFI->getScratchRSrcReg()) // srsrc 995 .addReg(MFI->getStackPtrOffsetReg()) // soffset 996 .addImm(i * 4) // offset 997 .addMemOperand(MMO); 998 999 auto MIB = 1000 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg) 1001 .addReg(TmpVGPR, RegState::Kill); 1002 1003 if (NumSubRegs > 1) 1004 MIB.addReg(MI->getOperand(0).getReg(), RegState::ImplicitDefine); 1005 } 1006 } 1007 1008 MI->eraseFromParent(); 1009 return true; 1010 } 1011 1012 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to 1013 /// a VGPR and the stack slot can be safely eliminated when all other users are 1014 /// handled. 1015 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex( 1016 MachineBasicBlock::iterator MI, 1017 int FI, 1018 RegScavenger *RS) const { 1019 switch (MI->getOpcode()) { 1020 case AMDGPU::SI_SPILL_S1024_SAVE: 1021 case AMDGPU::SI_SPILL_S512_SAVE: 1022 case AMDGPU::SI_SPILL_S256_SAVE: 1023 case AMDGPU::SI_SPILL_S160_SAVE: 1024 case AMDGPU::SI_SPILL_S128_SAVE: 1025 case AMDGPU::SI_SPILL_S96_SAVE: 1026 case AMDGPU::SI_SPILL_S64_SAVE: 1027 case AMDGPU::SI_SPILL_S32_SAVE: 1028 return spillSGPR(MI, FI, RS, true); 1029 case AMDGPU::SI_SPILL_S1024_RESTORE: 1030 case AMDGPU::SI_SPILL_S512_RESTORE: 1031 case AMDGPU::SI_SPILL_S256_RESTORE: 1032 case AMDGPU::SI_SPILL_S160_RESTORE: 1033 case AMDGPU::SI_SPILL_S128_RESTORE: 1034 case AMDGPU::SI_SPILL_S96_RESTORE: 1035 case AMDGPU::SI_SPILL_S64_RESTORE: 1036 case AMDGPU::SI_SPILL_S32_RESTORE: 1037 return restoreSGPR(MI, FI, RS, true); 1038 default: 1039 llvm_unreachable("not an SGPR spill instruction"); 1040 } 1041 } 1042 1043 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 1044 int SPAdj, unsigned FIOperandNum, 1045 RegScavenger *RS) const { 1046 MachineFunction *MF = MI->getParent()->getParent(); 1047 MachineBasicBlock *MBB = MI->getParent(); 1048 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1049 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1050 const SIInstrInfo *TII = ST.getInstrInfo(); 1051 DebugLoc DL = MI->getDebugLoc(); 1052 1053 assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?"); 1054 1055 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 1056 int Index = MI->getOperand(FIOperandNum).getIndex(); 1057 1058 Register FrameReg = getFrameRegister(*MF); 1059 1060 switch (MI->getOpcode()) { 1061 // SGPR register spill 1062 case AMDGPU::SI_SPILL_S1024_SAVE: 1063 case AMDGPU::SI_SPILL_S512_SAVE: 1064 case AMDGPU::SI_SPILL_S256_SAVE: 1065 case AMDGPU::SI_SPILL_S160_SAVE: 1066 case AMDGPU::SI_SPILL_S128_SAVE: 1067 case AMDGPU::SI_SPILL_S96_SAVE: 1068 case AMDGPU::SI_SPILL_S64_SAVE: 1069 case AMDGPU::SI_SPILL_S32_SAVE: { 1070 spillSGPR(MI, Index, RS); 1071 break; 1072 } 1073 1074 // SGPR register restore 1075 case AMDGPU::SI_SPILL_S1024_RESTORE: 1076 case AMDGPU::SI_SPILL_S512_RESTORE: 1077 case AMDGPU::SI_SPILL_S256_RESTORE: 1078 case AMDGPU::SI_SPILL_S160_RESTORE: 1079 case AMDGPU::SI_SPILL_S128_RESTORE: 1080 case AMDGPU::SI_SPILL_S96_RESTORE: 1081 case AMDGPU::SI_SPILL_S64_RESTORE: 1082 case AMDGPU::SI_SPILL_S32_RESTORE: { 1083 restoreSGPR(MI, Index, RS); 1084 break; 1085 } 1086 1087 // VGPR register spill 1088 case AMDGPU::SI_SPILL_V1024_SAVE: 1089 case AMDGPU::SI_SPILL_V512_SAVE: 1090 case AMDGPU::SI_SPILL_V256_SAVE: 1091 case AMDGPU::SI_SPILL_V160_SAVE: 1092 case AMDGPU::SI_SPILL_V128_SAVE: 1093 case AMDGPU::SI_SPILL_V96_SAVE: 1094 case AMDGPU::SI_SPILL_V64_SAVE: 1095 case AMDGPU::SI_SPILL_V32_SAVE: 1096 case AMDGPU::SI_SPILL_A1024_SAVE: 1097 case AMDGPU::SI_SPILL_A512_SAVE: 1098 case AMDGPU::SI_SPILL_A128_SAVE: 1099 case AMDGPU::SI_SPILL_A64_SAVE: 1100 case AMDGPU::SI_SPILL_A32_SAVE: { 1101 const MachineOperand *VData = TII->getNamedOperand(*MI, 1102 AMDGPU::OpName::vdata); 1103 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 1104 MFI->getStackPtrOffsetReg()); 1105 1106 buildSpillLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET, 1107 Index, 1108 VData->getReg(), VData->isKill(), 1109 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1110 FrameReg, 1111 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1112 *MI->memoperands_begin(), 1113 RS); 1114 MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode())); 1115 MI->eraseFromParent(); 1116 break; 1117 } 1118 case AMDGPU::SI_SPILL_V32_RESTORE: 1119 case AMDGPU::SI_SPILL_V64_RESTORE: 1120 case AMDGPU::SI_SPILL_V96_RESTORE: 1121 case AMDGPU::SI_SPILL_V128_RESTORE: 1122 case AMDGPU::SI_SPILL_V160_RESTORE: 1123 case AMDGPU::SI_SPILL_V256_RESTORE: 1124 case AMDGPU::SI_SPILL_V512_RESTORE: 1125 case AMDGPU::SI_SPILL_V1024_RESTORE: 1126 case AMDGPU::SI_SPILL_A32_RESTORE: 1127 case AMDGPU::SI_SPILL_A64_RESTORE: 1128 case AMDGPU::SI_SPILL_A128_RESTORE: 1129 case AMDGPU::SI_SPILL_A512_RESTORE: 1130 case AMDGPU::SI_SPILL_A1024_RESTORE: { 1131 const MachineOperand *VData = TII->getNamedOperand(*MI, 1132 AMDGPU::OpName::vdata); 1133 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 1134 MFI->getStackPtrOffsetReg()); 1135 1136 buildSpillLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET, 1137 Index, 1138 VData->getReg(), VData->isKill(), 1139 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1140 FrameReg, 1141 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1142 *MI->memoperands_begin(), 1143 RS); 1144 MI->eraseFromParent(); 1145 break; 1146 } 1147 1148 default: { 1149 const DebugLoc &DL = MI->getDebugLoc(); 1150 bool IsMUBUF = TII->isMUBUF(*MI); 1151 1152 if (!IsMUBUF && !MFI->isEntryFunction()) { 1153 // Convert to a swizzled stack address by scaling by the wave size. 1154 // 1155 // In an entry function/kernel the offset is already swizzled. 1156 1157 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32; 1158 Register ResultReg = 1159 IsCopy ? MI->getOperand(0).getReg() 1160 : RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1161 1162 int64_t Offset = FrameInfo.getObjectOffset(Index); 1163 if (Offset == 0) { 1164 // XXX - This never happens because of emergency scavenging slot at 0? 1165 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ResultReg) 1166 .addImm(ST.getWavefrontSizeLog2()) 1167 .addReg(FrameReg); 1168 } else { 1169 if (auto MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) { 1170 // Reuse ResultReg in intermediate step. 1171 Register ScaledReg = ResultReg; 1172 1173 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), 1174 ScaledReg) 1175 .addImm(ST.getWavefrontSizeLog2()) 1176 .addReg(FrameReg); 1177 1178 const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32; 1179 1180 // TODO: Fold if use instruction is another add of a constant. 1181 if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { 1182 // FIXME: This can fail 1183 MIB.addImm(Offset); 1184 MIB.addReg(ScaledReg, RegState::Kill); 1185 if (!IsVOP2) 1186 MIB.addImm(0); // clamp bit 1187 } else { 1188 assert(MIB->getOpcode() == AMDGPU::V_ADD_I32_e64 && 1189 "Need to reuse carry out register"); 1190 1191 // Use scavenged unused carry out as offset register. 1192 Register ConstOffsetReg; 1193 if (!isWave32) 1194 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); 1195 else 1196 ConstOffsetReg = MIB.getReg(1); 1197 1198 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) 1199 .addImm(Offset); 1200 MIB.addReg(ConstOffsetReg, RegState::Kill); 1201 MIB.addReg(ScaledReg, RegState::Kill); 1202 MIB.addImm(0); // clamp bit 1203 } 1204 } else { 1205 // We have to produce a carry out, and there isn't a free SGPR pair 1206 // for it. We can keep the whole computation on the SALU to avoid 1207 // clobbering an additional register at the cost of an extra mov. 1208 1209 // We may have 1 free scratch SGPR even though a carry out is 1210 // unavailable. Only one additional mov is needed. 1211 Register TmpScaledReg = 1212 RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, false); 1213 Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg; 1214 1215 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg) 1216 .addReg(FrameReg) 1217 .addImm(ST.getWavefrontSizeLog2()); 1218 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), ScaledReg) 1219 .addReg(ScaledReg, RegState::Kill) 1220 .addImm(Offset); 1221 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg) 1222 .addReg(ScaledReg, RegState::Kill); 1223 1224 // If there were truly no free SGPRs, we need to undo everything. 1225 if (!TmpScaledReg.isValid()) { 1226 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScaledReg) 1227 .addReg(ScaledReg, RegState::Kill) 1228 .addImm(Offset); 1229 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) 1230 .addReg(FrameReg) 1231 .addImm(ST.getWavefrontSizeLog2()); 1232 } 1233 } 1234 } 1235 1236 // Don't introduce an extra copy if we're just materializing in a mov. 1237 if (IsCopy) 1238 MI->eraseFromParent(); 1239 else 1240 FIOp.ChangeToRegister(ResultReg, false, false, true); 1241 return; 1242 } 1243 1244 if (IsMUBUF) { 1245 // Disable offen so we don't need a 0 vgpr base. 1246 assert(static_cast<int>(FIOperandNum) == 1247 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1248 AMDGPU::OpName::vaddr)); 1249 1250 auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset); 1251 assert((SOffset.isReg() && 1252 SOffset.getReg() == MFI->getStackPtrOffsetReg()) || 1253 (SOffset.isImm() && SOffset.getImm() == 0)); 1254 if (SOffset.isReg()) { 1255 if (FrameReg == AMDGPU::NoRegister) { 1256 SOffset.ChangeToImmediate(0); 1257 } else { 1258 SOffset.setReg(FrameReg); 1259 } 1260 } 1261 1262 int64_t Offset = FrameInfo.getObjectOffset(Index); 1263 int64_t OldImm 1264 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); 1265 int64_t NewOffset = OldImm + Offset; 1266 1267 if (isUInt<12>(NewOffset) && 1268 buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) { 1269 MI->eraseFromParent(); 1270 return; 1271 } 1272 } 1273 1274 // If the offset is simply too big, don't convert to a scratch wave offset 1275 // relative index. 1276 1277 int64_t Offset = FrameInfo.getObjectOffset(Index); 1278 FIOp.ChangeToImmediate(Offset); 1279 if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) { 1280 Register TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1281 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 1282 .addImm(Offset); 1283 FIOp.ChangeToRegister(TmpReg, false, false, true); 1284 } 1285 } 1286 } 1287 } 1288 1289 StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const { 1290 return AMDGPUInstPrinter::getRegisterName(Reg); 1291 } 1292 1293 const TargetRegisterClass * 1294 SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) { 1295 switch (BitWidth) { 1296 case 1: 1297 return &AMDGPU::VReg_1RegClass; 1298 case 16: 1299 return &AMDGPU::VGPR_LO16RegClass; 1300 case 32: 1301 return &AMDGPU::VGPR_32RegClass; 1302 case 64: 1303 return &AMDGPU::VReg_64RegClass; 1304 case 96: 1305 return &AMDGPU::VReg_96RegClass; 1306 case 128: 1307 return &AMDGPU::VReg_128RegClass; 1308 case 160: 1309 return &AMDGPU::VReg_160RegClass; 1310 case 192: 1311 return &AMDGPU::VReg_192RegClass; 1312 case 256: 1313 return &AMDGPU::VReg_256RegClass; 1314 case 512: 1315 return &AMDGPU::VReg_512RegClass; 1316 case 1024: 1317 return &AMDGPU::VReg_1024RegClass; 1318 default: 1319 return nullptr; 1320 } 1321 } 1322 1323 const TargetRegisterClass * 1324 SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) { 1325 switch (BitWidth) { 1326 case 16: 1327 return &AMDGPU::AGPR_LO16RegClass; 1328 case 32: 1329 return &AMDGPU::AGPR_32RegClass; 1330 case 64: 1331 return &AMDGPU::AReg_64RegClass; 1332 case 96: 1333 return &AMDGPU::AReg_96RegClass; 1334 case 128: 1335 return &AMDGPU::AReg_128RegClass; 1336 case 160: 1337 return &AMDGPU::AReg_160RegClass; 1338 case 192: 1339 return &AMDGPU::AReg_192RegClass; 1340 case 256: 1341 return &AMDGPU::AReg_256RegClass; 1342 case 512: 1343 return &AMDGPU::AReg_512RegClass; 1344 case 1024: 1345 return &AMDGPU::AReg_1024RegClass; 1346 default: 1347 return nullptr; 1348 } 1349 } 1350 1351 const TargetRegisterClass * 1352 SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) { 1353 switch (BitWidth) { 1354 case 16: 1355 return &AMDGPU::SGPR_LO16RegClass; 1356 case 32: 1357 return &AMDGPU::SReg_32RegClass; 1358 case 64: 1359 return &AMDGPU::SReg_64RegClass; 1360 case 96: 1361 return &AMDGPU::SGPR_96RegClass; 1362 case 128: 1363 return &AMDGPU::SGPR_128RegClass; 1364 case 160: 1365 return &AMDGPU::SGPR_160RegClass; 1366 case 192: 1367 return &AMDGPU::SGPR_192RegClass; 1368 case 256: 1369 return &AMDGPU::SGPR_256RegClass; 1370 case 512: 1371 return &AMDGPU::SGPR_512RegClass; 1372 case 1024: 1373 return &AMDGPU::SGPR_1024RegClass; 1374 default: 1375 return nullptr; 1376 } 1377 } 1378 1379 // FIXME: This is very slow. It might be worth creating a map from physreg to 1380 // register class. 1381 const TargetRegisterClass * 1382 SIRegisterInfo::getPhysRegClass(MCRegister Reg) const { 1383 static const TargetRegisterClass *const BaseClasses[] = { 1384 &AMDGPU::VGPR_LO16RegClass, 1385 &AMDGPU::VGPR_HI16RegClass, 1386 &AMDGPU::SReg_LO16RegClass, 1387 &AMDGPU::AGPR_LO16RegClass, 1388 &AMDGPU::VGPR_32RegClass, 1389 &AMDGPU::SReg_32RegClass, 1390 &AMDGPU::AGPR_32RegClass, 1391 &AMDGPU::VReg_64RegClass, 1392 &AMDGPU::SReg_64RegClass, 1393 &AMDGPU::AReg_64RegClass, 1394 &AMDGPU::VReg_96RegClass, 1395 &AMDGPU::SReg_96RegClass, 1396 &AMDGPU::AReg_96RegClass, 1397 &AMDGPU::VReg_128RegClass, 1398 &AMDGPU::SReg_128RegClass, 1399 &AMDGPU::AReg_128RegClass, 1400 &AMDGPU::VReg_160RegClass, 1401 &AMDGPU::SReg_160RegClass, 1402 &AMDGPU::AReg_160RegClass, 1403 &AMDGPU::VReg_192RegClass, 1404 &AMDGPU::SReg_192RegClass, 1405 &AMDGPU::AReg_192RegClass, 1406 &AMDGPU::VReg_256RegClass, 1407 &AMDGPU::SReg_256RegClass, 1408 &AMDGPU::AReg_256RegClass, 1409 &AMDGPU::VReg_512RegClass, 1410 &AMDGPU::SReg_512RegClass, 1411 &AMDGPU::AReg_512RegClass, 1412 &AMDGPU::SReg_1024RegClass, 1413 &AMDGPU::VReg_1024RegClass, 1414 &AMDGPU::AReg_1024RegClass, 1415 &AMDGPU::SCC_CLASSRegClass, 1416 &AMDGPU::Pseudo_SReg_32RegClass, 1417 &AMDGPU::Pseudo_SReg_128RegClass, 1418 }; 1419 1420 for (const TargetRegisterClass *BaseClass : BaseClasses) { 1421 if (BaseClass->contains(Reg)) { 1422 return BaseClass; 1423 } 1424 } 1425 return nullptr; 1426 } 1427 1428 // TODO: It might be helpful to have some target specific flags in 1429 // TargetRegisterClass to mark which classes are VGPRs to make this trivial. 1430 bool SIRegisterInfo::hasVGPRs(const TargetRegisterClass *RC) const { 1431 unsigned Size = getRegSizeInBits(*RC); 1432 if (Size == 16) { 1433 return getCommonSubClass(&AMDGPU::VGPR_LO16RegClass, RC) != nullptr || 1434 getCommonSubClass(&AMDGPU::VGPR_HI16RegClass, RC) != nullptr; 1435 } 1436 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 1437 if (!VRC) { 1438 assert(Size < 32 && "Invalid register class size"); 1439 return false; 1440 } 1441 return getCommonSubClass(VRC, RC) != nullptr; 1442 } 1443 1444 bool SIRegisterInfo::hasAGPRs(const TargetRegisterClass *RC) const { 1445 unsigned Size = getRegSizeInBits(*RC); 1446 if (Size < 16) 1447 return false; 1448 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 1449 if (!ARC) { 1450 assert(getVGPRClassForBitWidth(Size) && "Invalid register class size"); 1451 return false; 1452 } 1453 return getCommonSubClass(ARC, RC) != nullptr; 1454 } 1455 1456 const TargetRegisterClass * 1457 SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const { 1458 unsigned Size = getRegSizeInBits(*SRC); 1459 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 1460 assert(VRC && "Invalid register class size"); 1461 return VRC; 1462 } 1463 1464 const TargetRegisterClass * 1465 SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const { 1466 unsigned Size = getRegSizeInBits(*SRC); 1467 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 1468 assert(ARC && "Invalid register class size"); 1469 return ARC; 1470 } 1471 1472 const TargetRegisterClass * 1473 SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const { 1474 unsigned Size = getRegSizeInBits(*VRC); 1475 if (Size == 32) 1476 return &AMDGPU::SGPR_32RegClass; 1477 const TargetRegisterClass *SRC = getSGPRClassForBitWidth(Size); 1478 assert(SRC && "Invalid register class size"); 1479 return SRC; 1480 } 1481 1482 const TargetRegisterClass *SIRegisterInfo::getSubRegClass( 1483 const TargetRegisterClass *RC, unsigned SubIdx) const { 1484 if (SubIdx == AMDGPU::NoSubRegister) 1485 return RC; 1486 1487 // We can assume that each lane corresponds to one 32-bit register. 1488 unsigned Size = getNumChannelsFromSubReg(SubIdx) * 32; 1489 if (isSGPRClass(RC)) { 1490 if (Size == 32) 1491 RC = &AMDGPU::SGPR_32RegClass; 1492 else 1493 RC = getSGPRClassForBitWidth(Size); 1494 } else if (hasAGPRs(RC)) { 1495 RC = getAGPRClassForBitWidth(Size); 1496 } else { 1497 RC = getVGPRClassForBitWidth(Size); 1498 } 1499 assert(RC && "Invalid sub-register class size"); 1500 return RC; 1501 } 1502 1503 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const { 1504 if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST && 1505 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST) 1506 return !ST.hasMFMAInlineLiteralBug(); 1507 1508 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1509 OpType <= AMDGPU::OPERAND_SRC_LAST; 1510 } 1511 1512 bool SIRegisterInfo::shouldRewriteCopySrc( 1513 const TargetRegisterClass *DefRC, 1514 unsigned DefSubReg, 1515 const TargetRegisterClass *SrcRC, 1516 unsigned SrcSubReg) const { 1517 // We want to prefer the smallest register class possible, so we don't want to 1518 // stop and rewrite on anything that looks like a subregister 1519 // extract. Operations mostly don't care about the super register class, so we 1520 // only want to stop on the most basic of copies between the same register 1521 // class. 1522 // 1523 // e.g. if we have something like 1524 // %0 = ... 1525 // %1 = ... 1526 // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2 1527 // %3 = COPY %2, sub0 1528 // 1529 // We want to look through the COPY to find: 1530 // => %3 = COPY %0 1531 1532 // Plain copy. 1533 return getCommonSubClass(DefRC, SrcRC) != nullptr; 1534 } 1535 1536 /// Returns a lowest register that is not used at any point in the function. 1537 /// If all registers are used, then this function will return 1538 /// AMDGPU::NoRegister. If \p ReserveHighestVGPR = true, then return 1539 /// highest unused register. 1540 MCRegister SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI, 1541 const TargetRegisterClass *RC, 1542 const MachineFunction &MF, 1543 bool ReserveHighestVGPR) const { 1544 if (ReserveHighestVGPR) { 1545 for (MCRegister Reg : reverse(*RC)) 1546 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 1547 return Reg; 1548 } else { 1549 for (MCRegister Reg : *RC) 1550 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 1551 return Reg; 1552 } 1553 return MCRegister(); 1554 } 1555 1556 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC, 1557 unsigned EltSize) const { 1558 const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC->MC); 1559 assert(RegBitWidth >= 32 && RegBitWidth <= 1024); 1560 1561 const unsigned EltBitWidth = EltSize * 8; 1562 assert(EltBitWidth >= 32 && EltBitWidth < 1024 && isPowerOf2_32(EltBitWidth)); 1563 const unsigned LogEltBitWidth = Log2_32(EltBitWidth); 1564 1565 assert(RegBitWidth % EltBitWidth == 0); 1566 1567 if (RegBitWidth == EltBitWidth) 1568 return {}; 1569 1570 static const int16_t Sub_32[] = { 1571 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1572 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 1573 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 1574 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 1575 AMDGPU::sub16, AMDGPU::sub17, AMDGPU::sub18, AMDGPU::sub19, 1576 AMDGPU::sub20, AMDGPU::sub21, AMDGPU::sub22, AMDGPU::sub23, 1577 AMDGPU::sub24, AMDGPU::sub25, AMDGPU::sub26, AMDGPU::sub27, 1578 AMDGPU::sub28, AMDGPU::sub29, AMDGPU::sub30, AMDGPU::sub31 1579 }; 1580 1581 static const int16_t Sub_64[] = { 1582 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 1583 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 1584 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 1585 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 1586 AMDGPU::sub16_sub17, AMDGPU::sub18_sub19, 1587 AMDGPU::sub20_sub21, AMDGPU::sub22_sub23, 1588 AMDGPU::sub24_sub25, AMDGPU::sub26_sub27, 1589 AMDGPU::sub28_sub29, AMDGPU::sub30_sub31 1590 }; 1591 1592 static const int16_t Sub_128[] = { 1593 AMDGPU::sub0_sub1_sub2_sub3, 1594 AMDGPU::sub4_sub5_sub6_sub7, 1595 AMDGPU::sub8_sub9_sub10_sub11, 1596 AMDGPU::sub12_sub13_sub14_sub15, 1597 AMDGPU::sub16_sub17_sub18_sub19, 1598 AMDGPU::sub20_sub21_sub22_sub23, 1599 AMDGPU::sub24_sub25_sub26_sub27, 1600 AMDGPU::sub28_sub29_sub30_sub31 1601 }; 1602 1603 static const int16_t Sub_256[] = { 1604 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7, 1605 AMDGPU::sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 1606 AMDGPU::sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23, 1607 AMDGPU::sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31 1608 }; 1609 1610 static const int16_t Sub_512[] = { 1611 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 1612 AMDGPU::sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23_sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31 1613 }; 1614 1615 static const int16_t *const Subs[] = { 1616 Sub_32, Sub_64, Sub_128, Sub_256, Sub_512 1617 }; 1618 1619 return makeArrayRef(Subs[LogEltBitWidth - 5], RegBitWidth >> LogEltBitWidth); 1620 } 1621 1622 const TargetRegisterClass* 1623 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI, 1624 Register Reg) const { 1625 return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegClass(Reg); 1626 } 1627 1628 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI, 1629 Register Reg) const { 1630 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 1631 assert(RC && "Register class for the reg not found"); 1632 return hasVGPRs(RC); 1633 } 1634 1635 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI, 1636 Register Reg) const { 1637 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 1638 assert(RC && "Register class for the reg not found"); 1639 return hasAGPRs(RC); 1640 } 1641 1642 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI, 1643 const TargetRegisterClass *SrcRC, 1644 unsigned SubReg, 1645 const TargetRegisterClass *DstRC, 1646 unsigned DstSubReg, 1647 const TargetRegisterClass *NewRC, 1648 LiveIntervals &LIS) const { 1649 unsigned SrcSize = getRegSizeInBits(*SrcRC); 1650 unsigned DstSize = getRegSizeInBits(*DstRC); 1651 unsigned NewSize = getRegSizeInBits(*NewRC); 1652 1653 // Do not increase size of registers beyond dword, we would need to allocate 1654 // adjacent registers and constraint regalloc more than needed. 1655 1656 // Always allow dword coalescing. 1657 if (SrcSize <= 32 || DstSize <= 32) 1658 return true; 1659 1660 return NewSize <= DstSize || NewSize <= SrcSize; 1661 } 1662 1663 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 1664 MachineFunction &MF) const { 1665 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1666 1667 unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), 1668 MF.getFunction()); 1669 switch (RC->getID()) { 1670 default: 1671 return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF); 1672 case AMDGPU::VGPR_32RegClassID: 1673 case AMDGPU::VGPR_LO16RegClassID: 1674 case AMDGPU::VGPR_HI16RegClassID: 1675 return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF)); 1676 case AMDGPU::SGPR_32RegClassID: 1677 case AMDGPU::SGPR_LO16RegClassID: 1678 return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF)); 1679 } 1680 } 1681 1682 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 1683 unsigned Idx) const { 1684 if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 || 1685 Idx == AMDGPU::RegisterPressureSets::AGPR_32) 1686 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 1687 const_cast<MachineFunction &>(MF)); 1688 1689 if (Idx == AMDGPU::RegisterPressureSets::SReg_32) 1690 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, 1691 const_cast<MachineFunction &>(MF)); 1692 1693 llvm_unreachable("Unexpected register pressure set!"); 1694 } 1695 1696 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const { 1697 static const int Empty[] = { -1 }; 1698 1699 if (RegPressureIgnoredUnits[RegUnit]) 1700 return Empty; 1701 1702 return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit); 1703 } 1704 1705 MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { 1706 // Not a callee saved register. 1707 return AMDGPU::SGPR30_SGPR31; 1708 } 1709 1710 const TargetRegisterClass * 1711 SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, 1712 const RegisterBank &RB, 1713 const MachineRegisterInfo &MRI) const { 1714 switch (RB.getID()) { 1715 case AMDGPU::VGPRRegBankID: 1716 return getVGPRClassForBitWidth(std::max(32u, Size)); 1717 case AMDGPU::VCCRegBankID: 1718 assert(Size == 1); 1719 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 1720 : &AMDGPU::SReg_64_XEXECRegClass; 1721 case AMDGPU::SGPRRegBankID: 1722 return getSGPRClassForBitWidth(std::max(32u, Size)); 1723 case AMDGPU::AGPRRegBankID: 1724 return getAGPRClassForBitWidth(std::max(32u, Size)); 1725 default: 1726 llvm_unreachable("unknown register bank"); 1727 } 1728 } 1729 1730 const TargetRegisterClass * 1731 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO, 1732 const MachineRegisterInfo &MRI) const { 1733 const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg()); 1734 if (const RegisterBank *RB = RCOrRB.dyn_cast<const RegisterBank*>()) 1735 return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB, MRI); 1736 1737 const TargetRegisterClass *RC = RCOrRB.get<const TargetRegisterClass*>(); 1738 return getAllocatableClass(RC); 1739 } 1740 1741 MCRegister SIRegisterInfo::getVCC() const { 1742 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; 1743 } 1744 1745 const TargetRegisterClass * 1746 SIRegisterInfo::getRegClass(unsigned RCID) const { 1747 switch ((int)RCID) { 1748 case AMDGPU::SReg_1RegClassID: 1749 return getBoolRC(); 1750 case AMDGPU::SReg_1_XEXECRegClassID: 1751 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 1752 : &AMDGPU::SReg_64_XEXECRegClass; 1753 case -1: 1754 return nullptr; 1755 default: 1756 return AMDGPUGenRegisterInfo::getRegClass(RCID); 1757 } 1758 } 1759 1760 // Find reaching register definition 1761 MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg, 1762 MachineInstr &Use, 1763 MachineRegisterInfo &MRI, 1764 LiveIntervals *LIS) const { 1765 auto &MDT = LIS->getAnalysis<MachineDominatorTree>(); 1766 SlotIndex UseIdx = LIS->getInstructionIndex(Use); 1767 SlotIndex DefIdx; 1768 1769 if (Reg.isVirtual()) { 1770 if (!LIS->hasInterval(Reg)) 1771 return nullptr; 1772 LiveInterval &LI = LIS->getInterval(Reg); 1773 LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg) 1774 : MRI.getMaxLaneMaskForVReg(Reg); 1775 VNInfo *V = nullptr; 1776 if (LI.hasSubRanges()) { 1777 for (auto &S : LI.subranges()) { 1778 if ((S.LaneMask & SubLanes) == SubLanes) { 1779 V = S.getVNInfoAt(UseIdx); 1780 break; 1781 } 1782 } 1783 } else { 1784 V = LI.getVNInfoAt(UseIdx); 1785 } 1786 if (!V) 1787 return nullptr; 1788 DefIdx = V->def; 1789 } else { 1790 // Find last def. 1791 for (MCRegUnitIterator Units(Reg, this); Units.isValid(); ++Units) { 1792 LiveRange &LR = LIS->getRegUnit(*Units); 1793 if (VNInfo *V = LR.getVNInfoAt(UseIdx)) { 1794 if (!DefIdx.isValid() || 1795 MDT.dominates(LIS->getInstructionFromIndex(DefIdx), 1796 LIS->getInstructionFromIndex(V->def))) 1797 DefIdx = V->def; 1798 } else { 1799 return nullptr; 1800 } 1801 } 1802 } 1803 1804 MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx); 1805 1806 if (!Def || !MDT.dominates(Def, &Use)) 1807 return nullptr; 1808 1809 assert(Def->modifiesRegister(Reg, this)); 1810 1811 return Def; 1812 } 1813 1814 MCPhysReg SIRegisterInfo::get32BitRegister(MCPhysReg Reg) const { 1815 assert(getRegSizeInBits(*getPhysRegClass(Reg)) <= 32); 1816 1817 for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass, 1818 AMDGPU::SReg_32RegClass, 1819 AMDGPU::AGPR_32RegClass } ) { 1820 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC)) 1821 return Super; 1822 } 1823 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16, 1824 &AMDGPU::VGPR_32RegClass)) { 1825 return Super; 1826 } 1827 1828 return AMDGPU::NoRegister; 1829 } 1830