1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI implementation of the TargetRegisterInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIRegisterInfo.h" 15 #include "AMDGPURegisterBankInfo.h" 16 #include "AMDGPUSubtarget.h" 17 #include "SIInstrInfo.h" 18 #include "SIMachineFunctionInfo.h" 19 #include "MCTargetDesc/AMDGPUInstPrinter.h" 20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include "llvm/CodeGen/MachineDominators.h" 23 #include "llvm/CodeGen/MachineFrameInfo.h" 24 #include "llvm/CodeGen/MachineInstrBuilder.h" 25 #include "llvm/CodeGen/RegisterScavenging.h" 26 #include "llvm/CodeGen/SlotIndexes.h" 27 #include "llvm/IR/Function.h" 28 #include "llvm/IR/LLVMContext.h" 29 30 using namespace llvm; 31 32 #define GET_REGINFO_TARGET_DESC 33 #include "AMDGPUGenRegisterInfo.inc" 34 35 static cl::opt<bool> EnableSpillSGPRToVGPR( 36 "amdgpu-spill-sgpr-to-vgpr", 37 cl::desc("Enable spilling VGPRs to SGPRs"), 38 cl::ReallyHidden, 39 cl::init(true)); 40 41 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) 42 : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST), 43 SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) { 44 45 assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 && 46 getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) && 47 (getSubRegIndexLaneMask(AMDGPU::lo16) | 48 getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() == 49 getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() && 50 "getNumCoveredRegs() will not work with generated subreg masks!"); 51 52 RegPressureIgnoredUnits.resize(getNumRegUnits()); 53 RegPressureIgnoredUnits.set(*MCRegUnitIterator(AMDGPU::M0, this)); 54 for (auto Reg : AMDGPU::VGPR_HI16RegClass) 55 RegPressureIgnoredUnits.set(*MCRegUnitIterator(Reg, this)); 56 } 57 58 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, 59 MCRegister Reg) const { 60 MCRegAliasIterator R(Reg, this, true); 61 62 for (; R.isValid(); ++R) 63 Reserved.set(*R); 64 } 65 66 // Forced to be here by one .inc 67 const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs( 68 const MachineFunction *MF) const { 69 CallingConv::ID CC = MF->getFunction().getCallingConv(); 70 switch (CC) { 71 case CallingConv::C: 72 case CallingConv::Fast: 73 case CallingConv::Cold: 74 return CSR_AMDGPU_HighRegs_SaveList; 75 default: { 76 // Dummy to not crash RegisterClassInfo. 77 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister; 78 return &NoCalleeSavedReg; 79 } 80 } 81 } 82 83 const MCPhysReg * 84 SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const { 85 return nullptr; 86 } 87 88 const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF, 89 CallingConv::ID CC) const { 90 switch (CC) { 91 case CallingConv::C: 92 case CallingConv::Fast: 93 case CallingConv::Cold: 94 return CSR_AMDGPU_HighRegs_RegMask; 95 default: 96 return nullptr; 97 } 98 } 99 100 Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const { 101 const SIFrameLowering *TFI = 102 MF.getSubtarget<GCNSubtarget>().getFrameLowering(); 103 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); 104 // During ISel lowering we always reserve the stack pointer in entry 105 // functions, but never actually want to reference it when accessing our own 106 // frame. If we need a frame pointer we use it, but otherwise we can just use 107 // an immediate "0" which we represent by returning NoRegister. 108 if (FuncInfo->isEntryFunction()) { 109 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register(); 110 } 111 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() 112 : FuncInfo->getStackPtrOffsetReg(); 113 } 114 115 const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const { 116 return CSR_AMDGPU_AllVGPRs_RegMask; 117 } 118 119 const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const { 120 return CSR_AMDGPU_AllAllocatableSRegs_RegMask; 121 } 122 123 // FIXME: TableGen should generate something to make this manageable for all 124 // register classes. At a minimum we could use the opposite of 125 // composeSubRegIndices and go up from the base 32-bit subreg. 126 unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel, 127 unsigned NumRegs) { 128 // Table of NumRegs sized pieces at every 32-bit offset. 129 static const uint16_t SubRegFromChannelTable[][32] = { 130 {AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 131 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 132 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 133 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 134 AMDGPU::sub16, AMDGPU::sub17, AMDGPU::sub18, AMDGPU::sub19, 135 AMDGPU::sub20, AMDGPU::sub21, AMDGPU::sub22, AMDGPU::sub23, 136 AMDGPU::sub24, AMDGPU::sub25, AMDGPU::sub26, AMDGPU::sub27, 137 AMDGPU::sub28, AMDGPU::sub29, AMDGPU::sub30, AMDGPU::sub31}, 138 {AMDGPU::sub0_sub1, AMDGPU::sub1_sub2, AMDGPU::sub2_sub3, 139 AMDGPU::sub3_sub4, AMDGPU::sub4_sub5, AMDGPU::sub5_sub6, 140 AMDGPU::sub6_sub7, AMDGPU::sub7_sub8, AMDGPU::sub8_sub9, 141 AMDGPU::sub9_sub10, AMDGPU::sub10_sub11, AMDGPU::sub11_sub12, 142 AMDGPU::sub12_sub13, AMDGPU::sub13_sub14, AMDGPU::sub14_sub15, 143 AMDGPU::sub15_sub16, AMDGPU::sub16_sub17, AMDGPU::sub17_sub18, 144 AMDGPU::sub18_sub19, AMDGPU::sub19_sub20, AMDGPU::sub20_sub21, 145 AMDGPU::sub21_sub22, AMDGPU::sub22_sub23, AMDGPU::sub23_sub24, 146 AMDGPU::sub24_sub25, AMDGPU::sub25_sub26, AMDGPU::sub26_sub27, 147 AMDGPU::sub27_sub28, AMDGPU::sub28_sub29, AMDGPU::sub29_sub30, 148 AMDGPU::sub30_sub31, AMDGPU::NoSubRegister}, 149 {AMDGPU::sub0_sub1_sub2, AMDGPU::sub1_sub2_sub3, 150 AMDGPU::sub2_sub3_sub4, AMDGPU::sub3_sub4_sub5, 151 AMDGPU::sub4_sub5_sub6, AMDGPU::sub5_sub6_sub7, 152 AMDGPU::sub6_sub7_sub8, AMDGPU::sub7_sub8_sub9, 153 AMDGPU::sub8_sub9_sub10, AMDGPU::sub9_sub10_sub11, 154 AMDGPU::sub10_sub11_sub12, AMDGPU::sub11_sub12_sub13, 155 AMDGPU::sub12_sub13_sub14, AMDGPU::sub13_sub14_sub15, 156 AMDGPU::sub14_sub15_sub16, AMDGPU::sub15_sub16_sub17, 157 AMDGPU::sub16_sub17_sub18, AMDGPU::sub17_sub18_sub19, 158 AMDGPU::sub18_sub19_sub20, AMDGPU::sub19_sub20_sub21, 159 AMDGPU::sub20_sub21_sub22, AMDGPU::sub21_sub22_sub23, 160 AMDGPU::sub22_sub23_sub24, AMDGPU::sub23_sub24_sub25, 161 AMDGPU::sub24_sub25_sub26, AMDGPU::sub25_sub26_sub27, 162 AMDGPU::sub26_sub27_sub28, AMDGPU::sub27_sub28_sub29, 163 AMDGPU::sub28_sub29_sub30, AMDGPU::sub29_sub30_sub31, 164 AMDGPU::NoSubRegister, AMDGPU::NoSubRegister}, 165 {AMDGPU::sub0_sub1_sub2_sub3, AMDGPU::sub1_sub2_sub3_sub4, 166 AMDGPU::sub2_sub3_sub4_sub5, AMDGPU::sub3_sub4_sub5_sub6, 167 AMDGPU::sub4_sub5_sub6_sub7, AMDGPU::sub5_sub6_sub7_sub8, 168 AMDGPU::sub6_sub7_sub8_sub9, AMDGPU::sub7_sub8_sub9_sub10, 169 AMDGPU::sub8_sub9_sub10_sub11, AMDGPU::sub9_sub10_sub11_sub12, 170 AMDGPU::sub10_sub11_sub12_sub13, AMDGPU::sub11_sub12_sub13_sub14, 171 AMDGPU::sub12_sub13_sub14_sub15, AMDGPU::sub13_sub14_sub15_sub16, 172 AMDGPU::sub14_sub15_sub16_sub17, AMDGPU::sub15_sub16_sub17_sub18, 173 AMDGPU::sub16_sub17_sub18_sub19, AMDGPU::sub17_sub18_sub19_sub20, 174 AMDGPU::sub18_sub19_sub20_sub21, AMDGPU::sub19_sub20_sub21_sub22, 175 AMDGPU::sub20_sub21_sub22_sub23, AMDGPU::sub21_sub22_sub23_sub24, 176 AMDGPU::sub22_sub23_sub24_sub25, AMDGPU::sub23_sub24_sub25_sub26, 177 AMDGPU::sub24_sub25_sub26_sub27, AMDGPU::sub25_sub26_sub27_sub28, 178 AMDGPU::sub26_sub27_sub28_sub29, AMDGPU::sub27_sub28_sub29_sub30, 179 AMDGPU::sub28_sub29_sub30_sub31, AMDGPU::NoSubRegister, 180 AMDGPU::NoSubRegister, AMDGPU::NoSubRegister}}; 181 182 const unsigned NumRegIndex = NumRegs - 1; 183 184 assert(NumRegIndex < array_lengthof(SubRegFromChannelTable) && 185 "Not implemented"); 186 assert(Channel < array_lengthof(SubRegFromChannelTable[0])); 187 return SubRegFromChannelTable[NumRegIndex][Channel]; 188 } 189 190 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg( 191 const MachineFunction &MF) const { 192 unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4; 193 MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); 194 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass); 195 } 196 197 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 198 BitVector Reserved(getNumRegs()); 199 200 // EXEC_LO and EXEC_HI could be allocated and used as regular register, but 201 // this seems likely to result in bugs, so I'm marking them as reserved. 202 reserveRegisterTuples(Reserved, AMDGPU::EXEC); 203 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); 204 205 // M0 has to be reserved so that llvm accepts it as a live-in into a block. 206 reserveRegisterTuples(Reserved, AMDGPU::M0); 207 208 // Reserve src_vccz, src_execz, src_scc. 209 reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ); 210 reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ); 211 reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC); 212 213 // Reserve the memory aperture registers. 214 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); 215 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); 216 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); 217 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); 218 219 // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen. 220 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); 221 222 // Reserve xnack_mask registers - support is not implemented in Codegen. 223 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); 224 225 // Reserve lds_direct register - support is not implemented in Codegen. 226 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); 227 228 // Reserve Trap Handler registers - support is not implemented in Codegen. 229 reserveRegisterTuples(Reserved, AMDGPU::TBA); 230 reserveRegisterTuples(Reserved, AMDGPU::TMA); 231 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); 232 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); 233 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); 234 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); 235 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); 236 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); 237 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); 238 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); 239 240 // Reserve null register - it shall never be allocated 241 reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL); 242 243 // Disallow vcc_hi allocation in wave32. It may be allocated but most likely 244 // will result in bugs. 245 if (isWave32) { 246 Reserved.set(AMDGPU::VCC); 247 Reserved.set(AMDGPU::VCC_HI); 248 } 249 250 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); 251 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 252 for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { 253 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 254 reserveRegisterTuples(Reserved, Reg); 255 } 256 257 unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF); 258 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); 259 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) { 260 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); 261 reserveRegisterTuples(Reserved, Reg); 262 Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 263 reserveRegisterTuples(Reserved, Reg); 264 } 265 266 for (auto Reg : AMDGPU::SReg_32RegClass) { 267 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 268 Register Low = getSubReg(Reg, AMDGPU::lo16); 269 // This is to prevent BB vcc liveness errors. 270 if (!AMDGPU::SGPR_LO16RegClass.contains(Low)) 271 Reserved.set(Low); 272 } 273 274 for (auto Reg : AMDGPU::AGPR_32RegClass) { 275 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 276 } 277 278 // Reserve all the rest AGPRs if there are no instructions to use it. 279 if (!ST.hasMAIInsts()) { 280 for (unsigned i = 0; i < MaxNumVGPRs; ++i) { 281 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 282 reserveRegisterTuples(Reserved, Reg); 283 } 284 } 285 286 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 287 288 unsigned ScratchRSrcReg = MFI->getScratchRSrcReg(); 289 if (ScratchRSrcReg != AMDGPU::NoRegister) { 290 // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need 291 // to spill. 292 // TODO: May need to reserve a VGPR if doing LDS spilling. 293 reserveRegisterTuples(Reserved, ScratchRSrcReg); 294 } 295 296 // We have to assume the SP is needed in case there are calls in the function, 297 // which is detected after the function is lowered. If we aren't really going 298 // to need SP, don't bother reserving it. 299 MCRegister StackPtrReg = MFI->getStackPtrOffsetReg(); 300 301 if (StackPtrReg) { 302 reserveRegisterTuples(Reserved, StackPtrReg); 303 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg)); 304 } 305 306 MCRegister FrameReg = MFI->getFrameOffsetReg(); 307 if (FrameReg) { 308 reserveRegisterTuples(Reserved, FrameReg); 309 assert(!isSubRegister(ScratchRSrcReg, FrameReg)); 310 } 311 312 for (MCRegister Reg : MFI->WWMReservedRegs) { 313 reserveRegisterTuples(Reserved, Reg); 314 } 315 316 // FIXME: Stop using reserved registers for this. 317 for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs()) 318 reserveRegisterTuples(Reserved, Reg); 319 320 for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs()) 321 reserveRegisterTuples(Reserved, Reg); 322 323 return Reserved; 324 } 325 326 bool SIRegisterInfo::canRealignStack(const MachineFunction &MF) const { 327 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 328 // On entry, the base address is 0, so it can't possibly need any more 329 // alignment. 330 331 // FIXME: Should be able to specify the entry frame alignment per calling 332 // convention instead. 333 if (Info->isEntryFunction()) 334 return false; 335 336 return TargetRegisterInfo::canRealignStack(MF); 337 } 338 339 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 340 const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>(); 341 if (Info->isEntryFunction()) { 342 const MachineFrameInfo &MFI = Fn.getFrameInfo(); 343 return MFI.hasStackObjects() || MFI.hasCalls(); 344 } 345 346 // May need scavenger for dealing with callee saved registers. 347 return true; 348 } 349 350 bool SIRegisterInfo::requiresFrameIndexScavenging( 351 const MachineFunction &MF) const { 352 // Do not use frame virtual registers. They used to be used for SGPRs, but 353 // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the 354 // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a 355 // spill. 356 return false; 357 } 358 359 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( 360 const MachineFunction &MF) const { 361 const MachineFrameInfo &MFI = MF.getFrameInfo(); 362 return MFI.hasStackObjects(); 363 } 364 365 bool SIRegisterInfo::requiresVirtualBaseRegisters( 366 const MachineFunction &) const { 367 // There are no special dedicated stack or frame pointers. 368 return true; 369 } 370 371 int64_t SIRegisterInfo::getMUBUFInstrOffset(const MachineInstr *MI) const { 372 assert(SIInstrInfo::isMUBUF(*MI)); 373 374 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 375 AMDGPU::OpName::offset); 376 return MI->getOperand(OffIdx).getImm(); 377 } 378 379 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI, 380 int Idx) const { 381 if (!SIInstrInfo::isMUBUF(*MI)) 382 return 0; 383 384 assert(Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 385 AMDGPU::OpName::vaddr) && 386 "Should never see frame index on non-address operand"); 387 388 return getMUBUFInstrOffset(MI); 389 } 390 391 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { 392 if (!MI->mayLoadOrStore()) 393 return false; 394 395 int64_t FullOffset = Offset + getMUBUFInstrOffset(MI); 396 397 return !isUInt<12>(FullOffset); 398 } 399 400 void SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 401 Register BaseReg, 402 int FrameIdx, 403 int64_t Offset) const { 404 MachineBasicBlock::iterator Ins = MBB->begin(); 405 DebugLoc DL; // Defaults to "unknown" 406 407 if (Ins != MBB->end()) 408 DL = Ins->getDebugLoc(); 409 410 MachineFunction *MF = MBB->getParent(); 411 const SIInstrInfo *TII = ST.getInstrInfo(); 412 413 if (Offset == 0) { 414 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg) 415 .addFrameIndex(FrameIdx); 416 return; 417 } 418 419 MachineRegisterInfo &MRI = MF->getRegInfo(); 420 Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 421 422 Register FIReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 423 424 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 425 .addImm(Offset); 426 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), FIReg) 427 .addFrameIndex(FrameIdx); 428 429 TII->getAddNoCarry(*MBB, Ins, DL, BaseReg) 430 .addReg(OffsetReg, RegState::Kill) 431 .addReg(FIReg) 432 .addImm(0); // clamp bit 433 } 434 435 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, 436 int64_t Offset) const { 437 const SIInstrInfo *TII = ST.getInstrInfo(); 438 439 #ifndef NDEBUG 440 // FIXME: Is it possible to be storing a frame index to itself? 441 bool SeenFI = false; 442 for (const MachineOperand &MO: MI.operands()) { 443 if (MO.isFI()) { 444 if (SeenFI) 445 llvm_unreachable("should not see multiple frame indices"); 446 447 SeenFI = true; 448 } 449 } 450 #endif 451 452 MachineOperand *FIOp = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr); 453 #ifndef NDEBUG 454 MachineBasicBlock *MBB = MI.getParent(); 455 MachineFunction *MF = MBB->getParent(); 456 #endif 457 assert(FIOp && FIOp->isFI() && "frame index must be address operand"); 458 assert(TII->isMUBUF(MI)); 459 assert(TII->getNamedOperand(MI, AMDGPU::OpName::soffset)->getReg() == 460 MF->getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg() && 461 "should only be seeing stack pointer offset relative FrameIndex"); 462 463 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); 464 int64_t NewOffset = OffsetOp->getImm() + Offset; 465 assert(isUInt<12>(NewOffset) && "offset should be legal"); 466 467 FIOp->ChangeToRegister(BaseReg, false); 468 OffsetOp->setImm(NewOffset); 469 } 470 471 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 472 Register BaseReg, 473 int64_t Offset) const { 474 if (!SIInstrInfo::isMUBUF(*MI)) 475 return false; 476 477 int64_t NewOffset = Offset + getMUBUFInstrOffset(MI); 478 479 return isUInt<12>(NewOffset); 480 } 481 482 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass( 483 const MachineFunction &MF, unsigned Kind) const { 484 // This is inaccurate. It depends on the instruction and address space. The 485 // only place where we should hit this is for dealing with frame indexes / 486 // private accesses, so this is correct in that case. 487 return &AMDGPU::VGPR_32RegClass; 488 } 489 490 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 491 492 switch (Op) { 493 case AMDGPU::SI_SPILL_S1024_SAVE: 494 case AMDGPU::SI_SPILL_S1024_RESTORE: 495 case AMDGPU::SI_SPILL_V1024_SAVE: 496 case AMDGPU::SI_SPILL_V1024_RESTORE: 497 case AMDGPU::SI_SPILL_A1024_SAVE: 498 case AMDGPU::SI_SPILL_A1024_RESTORE: 499 return 32; 500 case AMDGPU::SI_SPILL_S512_SAVE: 501 case AMDGPU::SI_SPILL_S512_RESTORE: 502 case AMDGPU::SI_SPILL_V512_SAVE: 503 case AMDGPU::SI_SPILL_V512_RESTORE: 504 case AMDGPU::SI_SPILL_A512_SAVE: 505 case AMDGPU::SI_SPILL_A512_RESTORE: 506 return 16; 507 case AMDGPU::SI_SPILL_S256_SAVE: 508 case AMDGPU::SI_SPILL_S256_RESTORE: 509 case AMDGPU::SI_SPILL_V256_SAVE: 510 case AMDGPU::SI_SPILL_V256_RESTORE: 511 return 8; 512 case AMDGPU::SI_SPILL_S160_SAVE: 513 case AMDGPU::SI_SPILL_S160_RESTORE: 514 case AMDGPU::SI_SPILL_V160_SAVE: 515 case AMDGPU::SI_SPILL_V160_RESTORE: 516 return 5; 517 case AMDGPU::SI_SPILL_S128_SAVE: 518 case AMDGPU::SI_SPILL_S128_RESTORE: 519 case AMDGPU::SI_SPILL_V128_SAVE: 520 case AMDGPU::SI_SPILL_V128_RESTORE: 521 case AMDGPU::SI_SPILL_A128_SAVE: 522 case AMDGPU::SI_SPILL_A128_RESTORE: 523 return 4; 524 case AMDGPU::SI_SPILL_S96_SAVE: 525 case AMDGPU::SI_SPILL_S96_RESTORE: 526 case AMDGPU::SI_SPILL_V96_SAVE: 527 case AMDGPU::SI_SPILL_V96_RESTORE: 528 return 3; 529 case AMDGPU::SI_SPILL_S64_SAVE: 530 case AMDGPU::SI_SPILL_S64_RESTORE: 531 case AMDGPU::SI_SPILL_V64_SAVE: 532 case AMDGPU::SI_SPILL_V64_RESTORE: 533 case AMDGPU::SI_SPILL_A64_SAVE: 534 case AMDGPU::SI_SPILL_A64_RESTORE: 535 return 2; 536 case AMDGPU::SI_SPILL_S32_SAVE: 537 case AMDGPU::SI_SPILL_S32_RESTORE: 538 case AMDGPU::SI_SPILL_V32_SAVE: 539 case AMDGPU::SI_SPILL_V32_RESTORE: 540 case AMDGPU::SI_SPILL_A32_SAVE: 541 case AMDGPU::SI_SPILL_A32_RESTORE: 542 return 1; 543 default: llvm_unreachable("Invalid spill opcode"); 544 } 545 } 546 547 static int getOffsetMUBUFStore(unsigned Opc) { 548 switch (Opc) { 549 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 550 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; 551 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: 552 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; 553 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: 554 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; 555 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: 556 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; 557 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: 558 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; 559 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: 560 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; 561 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: 562 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; 563 default: 564 return -1; 565 } 566 } 567 568 static int getOffsetMUBUFLoad(unsigned Opc) { 569 switch (Opc) { 570 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 571 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 572 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: 573 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; 574 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: 575 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; 576 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: 577 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; 578 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: 579 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; 580 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: 581 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; 582 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: 583 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; 584 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: 585 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; 586 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: 587 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; 588 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: 589 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; 590 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: 591 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; 592 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: 593 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; 594 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: 595 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; 596 default: 597 return -1; 598 } 599 } 600 601 static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST, 602 MachineBasicBlock::iterator MI, 603 int Index, 604 unsigned Lane, 605 unsigned ValueReg, 606 bool IsKill) { 607 MachineBasicBlock *MBB = MI->getParent(); 608 MachineFunction *MF = MI->getParent()->getParent(); 609 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 610 const SIInstrInfo *TII = ST.getInstrInfo(); 611 612 MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane); 613 614 if (Reg == AMDGPU::NoRegister) 615 return MachineInstrBuilder(); 616 617 bool IsStore = MI->mayStore(); 618 MachineRegisterInfo &MRI = MF->getRegInfo(); 619 auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo()); 620 621 unsigned Dst = IsStore ? Reg : ValueReg; 622 unsigned Src = IsStore ? ValueReg : Reg; 623 unsigned Opc = (IsStore ^ TRI->isVGPR(MRI, Reg)) ? AMDGPU::V_ACCVGPR_WRITE_B32 624 : AMDGPU::V_ACCVGPR_READ_B32; 625 626 return BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(Opc), Dst) 627 .addReg(Src, getKillRegState(IsKill)); 628 } 629 630 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not 631 // need to handle the case where an SGPR may need to be spilled while spilling. 632 static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST, 633 MachineFrameInfo &MFI, 634 MachineBasicBlock::iterator MI, 635 int Index, 636 int64_t Offset) { 637 const SIInstrInfo *TII = ST.getInstrInfo(); 638 MachineBasicBlock *MBB = MI->getParent(); 639 const DebugLoc &DL = MI->getDebugLoc(); 640 bool IsStore = MI->mayStore(); 641 642 unsigned Opc = MI->getOpcode(); 643 int LoadStoreOp = IsStore ? 644 getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc); 645 if (LoadStoreOp == -1) 646 return false; 647 648 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); 649 if (spillVGPRtoAGPR(ST, MI, Index, 0, Reg->getReg(), false).getInstr()) 650 return true; 651 652 MachineInstrBuilder NewMI = 653 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 654 .add(*Reg) 655 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) 656 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) 657 .addImm(Offset) 658 .addImm(0) // glc 659 .addImm(0) // slc 660 .addImm(0) // tfe 661 .addImm(0) // dlc 662 .addImm(0) // swz 663 .cloneMemRefs(*MI); 664 665 const MachineOperand *VDataIn = TII->getNamedOperand(*MI, 666 AMDGPU::OpName::vdata_in); 667 if (VDataIn) 668 NewMI.add(*VDataIn); 669 return true; 670 } 671 672 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI, 673 unsigned LoadStoreOp, 674 int Index, 675 Register ValueReg, 676 bool IsKill, 677 MCRegister ScratchRsrcReg, 678 MCRegister ScratchOffsetReg, 679 int64_t InstOffset, 680 MachineMemOperand *MMO, 681 RegScavenger *RS) const { 682 MachineBasicBlock *MBB = MI->getParent(); 683 MachineFunction *MF = MI->getParent()->getParent(); 684 const SIInstrInfo *TII = ST.getInstrInfo(); 685 const MachineFrameInfo &MFI = MF->getFrameInfo(); 686 687 const MCInstrDesc &Desc = TII->get(LoadStoreOp); 688 const DebugLoc &DL = MI->getDebugLoc(); 689 bool IsStore = Desc.mayStore(); 690 691 bool Scavenged = false; 692 MCRegister SOffset = ScratchOffsetReg; 693 694 const unsigned EltSize = 4; 695 const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg); 696 unsigned NumSubRegs = AMDGPU::getRegBitWidth(RC->getID()) / (EltSize * CHAR_BIT); 697 unsigned Size = NumSubRegs * EltSize; 698 int64_t Offset = InstOffset + MFI.getObjectOffset(Index); 699 int64_t ScratchOffsetRegDelta = 0; 700 701 Align Alignment = MFI.getObjectAlign(Index); 702 const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo(); 703 704 Register TmpReg = 705 hasAGPRs(RC) ? TII->getNamedOperand(*MI, AMDGPU::OpName::tmp)->getReg() 706 : Register(); 707 708 assert((Offset % EltSize) == 0 && "unexpected VGPR spill offset"); 709 710 if (!isUInt<12>(Offset + Size - EltSize)) { 711 SOffset = MCRegister(); 712 713 // We currently only support spilling VGPRs to EltSize boundaries, meaning 714 // we can simplify the adjustment of Offset here to just scale with 715 // WavefrontSize. 716 Offset *= ST.getWavefrontSize(); 717 718 // We don't have access to the register scavenger if this function is called 719 // during PEI::scavengeFrameVirtualRegs(). 720 if (RS) 721 SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false); 722 723 if (!SOffset) { 724 if (!ScratchOffsetReg) { 725 report_fatal_error("could not scavenge SGPR to spill in entry function"); 726 } 727 // There are no free SGPRs, and since we are in the process of spilling 728 // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true 729 // on SI/CI and on VI it is true until we implement spilling using scalar 730 // stores), we have no way to free up an SGPR. Our solution here is to 731 // add the offset directly to the ScratchOffset register, and then 732 // subtract the offset after the spill to return ScratchOffset to it's 733 // original value. 734 SOffset = ScratchOffsetReg; 735 ScratchOffsetRegDelta = Offset; 736 } else { 737 Scavenged = true; 738 } 739 740 if (ScratchOffsetReg == AMDGPU::NoRegister) { 741 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset) 742 .addImm(Offset); 743 } else { 744 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset) 745 .addReg(ScratchOffsetReg) 746 .addImm(Offset); 747 } 748 749 Offset = 0; 750 } 751 752 for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += EltSize) { 753 Register SubReg = NumSubRegs == 1 754 ? Register(ValueReg) 755 : getSubReg(ValueReg, getSubRegFromChannel(i)); 756 757 unsigned SOffsetRegState = 0; 758 unsigned SrcDstRegState = getDefRegState(!IsStore); 759 if (i + 1 == e) { 760 SOffsetRegState |= getKillRegState(Scavenged); 761 // The last implicit use carries the "Kill" flag. 762 SrcDstRegState |= getKillRegState(IsKill); 763 } 764 765 auto MIB = spillVGPRtoAGPR(ST, MI, Index, i, SubReg, IsKill); 766 767 if (!MIB.getInstr()) { 768 unsigned FinalReg = SubReg; 769 if (TmpReg != AMDGPU::NoRegister) { 770 if (IsStore) 771 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_READ_B32), TmpReg) 772 .addReg(SubReg, getKillRegState(IsKill)); 773 SubReg = TmpReg; 774 } 775 776 MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(EltSize * i); 777 MachineMemOperand *NewMMO = 778 MF->getMachineMemOperand(PInfo, MMO->getFlags(), EltSize, 779 commonAlignment(Alignment, EltSize * i)); 780 781 MIB = BuildMI(*MBB, MI, DL, Desc) 782 .addReg(SubReg, 783 getDefRegState(!IsStore) | getKillRegState(IsKill)) 784 .addReg(ScratchRsrcReg); 785 if (SOffset == AMDGPU::NoRegister) { 786 MIB.addImm(0); 787 } else { 788 MIB.addReg(SOffset, SOffsetRegState); 789 } 790 MIB.addImm(Offset) 791 .addImm(0) // glc 792 .addImm(0) // slc 793 .addImm(0) // tfe 794 .addImm(0) // dlc 795 .addImm(0) // swz 796 .addMemOperand(NewMMO); 797 798 if (!IsStore && TmpReg != AMDGPU::NoRegister) 799 MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32), 800 FinalReg) 801 .addReg(TmpReg, RegState::Kill); 802 } 803 804 if (NumSubRegs > 1) 805 MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState); 806 } 807 808 if (ScratchOffsetRegDelta != 0) { 809 // Subtract the offset we added to the ScratchOffset register. 810 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScratchOffsetReg) 811 .addReg(ScratchOffsetReg) 812 .addImm(ScratchOffsetRegDelta); 813 } 814 } 815 816 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, 817 int Index, 818 RegScavenger *RS, 819 bool OnlyToVGPR) const { 820 MachineBasicBlock *MBB = MI->getParent(); 821 MachineFunction *MF = MBB->getParent(); 822 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 823 DenseSet<unsigned> SGPRSpillVGPRDefinedSet; 824 825 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 826 = MFI->getSGPRToVGPRSpills(Index); 827 bool SpillToVGPR = !VGPRSpills.empty(); 828 if (OnlyToVGPR && !SpillToVGPR) 829 return false; 830 831 const SIInstrInfo *TII = ST.getInstrInfo(); 832 833 Register SuperReg = MI->getOperand(0).getReg(); 834 bool IsKill = MI->getOperand(0).isKill(); 835 const DebugLoc &DL = MI->getDebugLoc(); 836 837 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 838 839 assert(SpillToVGPR || (SuperReg != MFI->getStackPtrOffsetReg() && 840 SuperReg != MFI->getFrameOffsetReg())); 841 842 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 843 844 unsigned EltSize = 4; 845 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 846 847 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 848 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 849 850 // Scavenged temporary VGPR to use. It must be scavenged once for any number 851 // of spilled subregs. 852 Register TmpVGPR; 853 854 // SubReg carries the "Kill" flag when SubReg == SuperReg. 855 unsigned SubKillState = getKillRegState((NumSubRegs == 1) && IsKill); 856 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 857 Register SubReg = 858 NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]); 859 860 if (SpillToVGPR) { 861 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 862 863 // During SGPR spilling to VGPR, determine if the VGPR is defined. The 864 // only circumstance in which we say it is undefined is when it is the 865 // first spill to this VGPR in the first basic block. 866 bool VGPRDefined = true; 867 if (MBB == &MF->front()) 868 VGPRDefined = !SGPRSpillVGPRDefinedSet.insert(Spill.VGPR).second; 869 870 // Mark the "old value of vgpr" input undef only if this is the first sgpr 871 // spill to this specific vgpr in the first basic block. 872 BuildMI(*MBB, MI, DL, 873 TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32), 874 Spill.VGPR) 875 .addReg(SubReg, getKillRegState(IsKill)) 876 .addImm(Spill.Lane) 877 .addReg(Spill.VGPR, VGPRDefined ? 0 : RegState::Undef); 878 879 // FIXME: Since this spills to another register instead of an actual 880 // frame index, we should delete the frame index when all references to 881 // it are fixed. 882 } else { 883 // XXX - Can to VGPR spill fail for some subregisters but not others? 884 if (OnlyToVGPR) 885 return false; 886 887 // Spill SGPR to a frame index. 888 if (!TmpVGPR.isValid()) 889 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 890 891 MachineInstrBuilder Mov 892 = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 893 .addReg(SubReg, SubKillState); 894 895 // There could be undef components of a spilled super register. 896 // TODO: Can we detect this and skip the spill? 897 if (NumSubRegs > 1) { 898 // The last implicit use of the SuperReg carries the "Kill" flag. 899 unsigned SuperKillState = 0; 900 if (i + 1 == e) 901 SuperKillState |= getKillRegState(IsKill); 902 Mov.addReg(SuperReg, RegState::Implicit | SuperKillState); 903 } 904 905 Align Alignment = FrameInfo.getObjectAlign(Index); 906 MachinePointerInfo PtrInfo 907 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 908 MachineMemOperand *MMO = 909 MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, EltSize, 910 commonAlignment(Alignment, EltSize * i)); 911 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_SAVE)) 912 .addReg(TmpVGPR, RegState::Kill) // src 913 .addFrameIndex(Index) // vaddr 914 .addReg(MFI->getScratchRSrcReg()) // srrsrc 915 .addReg(MFI->getStackPtrOffsetReg()) // soffset 916 .addImm(i * 4) // offset 917 .addMemOperand(MMO); 918 } 919 } 920 921 MI->eraseFromParent(); 922 MFI->addToSpilledSGPRs(NumSubRegs); 923 return true; 924 } 925 926 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, 927 int Index, 928 RegScavenger *RS, 929 bool OnlyToVGPR) const { 930 MachineFunction *MF = MI->getParent()->getParent(); 931 MachineBasicBlock *MBB = MI->getParent(); 932 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 933 934 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 935 = MFI->getSGPRToVGPRSpills(Index); 936 bool SpillToVGPR = !VGPRSpills.empty(); 937 if (OnlyToVGPR && !SpillToVGPR) 938 return false; 939 940 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 941 const SIInstrInfo *TII = ST.getInstrInfo(); 942 const DebugLoc &DL = MI->getDebugLoc(); 943 944 Register SuperReg = MI->getOperand(0).getReg(); 945 946 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 947 948 unsigned EltSize = 4; 949 950 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 951 952 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 953 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 954 955 Register TmpVGPR; 956 957 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 958 Register SubReg = 959 NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]); 960 961 if (SpillToVGPR) { 962 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 963 auto MIB = 964 BuildMI(*MBB, MI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32), 965 SubReg) 966 .addReg(Spill.VGPR) 967 .addImm(Spill.Lane); 968 969 if (NumSubRegs > 1 && i == 0) 970 MIB.addReg(SuperReg, RegState::ImplicitDefine); 971 } else { 972 if (OnlyToVGPR) 973 return false; 974 975 // Restore SGPR from a stack slot. 976 // FIXME: We should use S_LOAD_DWORD here for VI. 977 if (!TmpVGPR.isValid()) 978 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 979 Align Alignment = FrameInfo.getObjectAlign(Index); 980 981 MachinePointerInfo PtrInfo 982 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 983 984 MachineMemOperand *MMO = 985 MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, EltSize, 986 commonAlignment(Alignment, EltSize * i)); 987 988 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_RESTORE), TmpVGPR) 989 .addFrameIndex(Index) // vaddr 990 .addReg(MFI->getScratchRSrcReg()) // srsrc 991 .addReg(MFI->getStackPtrOffsetReg()) // soffset 992 .addImm(i * 4) // offset 993 .addMemOperand(MMO); 994 995 auto MIB = 996 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg) 997 .addReg(TmpVGPR, RegState::Kill); 998 999 if (NumSubRegs > 1) 1000 MIB.addReg(MI->getOperand(0).getReg(), RegState::ImplicitDefine); 1001 } 1002 } 1003 1004 MI->eraseFromParent(); 1005 return true; 1006 } 1007 1008 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to 1009 /// a VGPR and the stack slot can be safely eliminated when all other users are 1010 /// handled. 1011 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex( 1012 MachineBasicBlock::iterator MI, 1013 int FI, 1014 RegScavenger *RS) const { 1015 switch (MI->getOpcode()) { 1016 case AMDGPU::SI_SPILL_S1024_SAVE: 1017 case AMDGPU::SI_SPILL_S512_SAVE: 1018 case AMDGPU::SI_SPILL_S256_SAVE: 1019 case AMDGPU::SI_SPILL_S160_SAVE: 1020 case AMDGPU::SI_SPILL_S128_SAVE: 1021 case AMDGPU::SI_SPILL_S96_SAVE: 1022 case AMDGPU::SI_SPILL_S64_SAVE: 1023 case AMDGPU::SI_SPILL_S32_SAVE: 1024 return spillSGPR(MI, FI, RS, true); 1025 case AMDGPU::SI_SPILL_S1024_RESTORE: 1026 case AMDGPU::SI_SPILL_S512_RESTORE: 1027 case AMDGPU::SI_SPILL_S256_RESTORE: 1028 case AMDGPU::SI_SPILL_S160_RESTORE: 1029 case AMDGPU::SI_SPILL_S128_RESTORE: 1030 case AMDGPU::SI_SPILL_S96_RESTORE: 1031 case AMDGPU::SI_SPILL_S64_RESTORE: 1032 case AMDGPU::SI_SPILL_S32_RESTORE: 1033 return restoreSGPR(MI, FI, RS, true); 1034 default: 1035 llvm_unreachable("not an SGPR spill instruction"); 1036 } 1037 } 1038 1039 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 1040 int SPAdj, unsigned FIOperandNum, 1041 RegScavenger *RS) const { 1042 MachineFunction *MF = MI->getParent()->getParent(); 1043 MachineBasicBlock *MBB = MI->getParent(); 1044 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1045 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1046 const SIInstrInfo *TII = ST.getInstrInfo(); 1047 DebugLoc DL = MI->getDebugLoc(); 1048 1049 assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?"); 1050 1051 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 1052 int Index = MI->getOperand(FIOperandNum).getIndex(); 1053 1054 Register FrameReg = getFrameRegister(*MF); 1055 1056 switch (MI->getOpcode()) { 1057 // SGPR register spill 1058 case AMDGPU::SI_SPILL_S1024_SAVE: 1059 case AMDGPU::SI_SPILL_S512_SAVE: 1060 case AMDGPU::SI_SPILL_S256_SAVE: 1061 case AMDGPU::SI_SPILL_S160_SAVE: 1062 case AMDGPU::SI_SPILL_S128_SAVE: 1063 case AMDGPU::SI_SPILL_S96_SAVE: 1064 case AMDGPU::SI_SPILL_S64_SAVE: 1065 case AMDGPU::SI_SPILL_S32_SAVE: { 1066 spillSGPR(MI, Index, RS); 1067 break; 1068 } 1069 1070 // SGPR register restore 1071 case AMDGPU::SI_SPILL_S1024_RESTORE: 1072 case AMDGPU::SI_SPILL_S512_RESTORE: 1073 case AMDGPU::SI_SPILL_S256_RESTORE: 1074 case AMDGPU::SI_SPILL_S160_RESTORE: 1075 case AMDGPU::SI_SPILL_S128_RESTORE: 1076 case AMDGPU::SI_SPILL_S96_RESTORE: 1077 case AMDGPU::SI_SPILL_S64_RESTORE: 1078 case AMDGPU::SI_SPILL_S32_RESTORE: { 1079 restoreSGPR(MI, Index, RS); 1080 break; 1081 } 1082 1083 // VGPR register spill 1084 case AMDGPU::SI_SPILL_V1024_SAVE: 1085 case AMDGPU::SI_SPILL_V512_SAVE: 1086 case AMDGPU::SI_SPILL_V256_SAVE: 1087 case AMDGPU::SI_SPILL_V160_SAVE: 1088 case AMDGPU::SI_SPILL_V128_SAVE: 1089 case AMDGPU::SI_SPILL_V96_SAVE: 1090 case AMDGPU::SI_SPILL_V64_SAVE: 1091 case AMDGPU::SI_SPILL_V32_SAVE: 1092 case AMDGPU::SI_SPILL_A1024_SAVE: 1093 case AMDGPU::SI_SPILL_A512_SAVE: 1094 case AMDGPU::SI_SPILL_A128_SAVE: 1095 case AMDGPU::SI_SPILL_A64_SAVE: 1096 case AMDGPU::SI_SPILL_A32_SAVE: { 1097 const MachineOperand *VData = TII->getNamedOperand(*MI, 1098 AMDGPU::OpName::vdata); 1099 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 1100 MFI->getStackPtrOffsetReg()); 1101 1102 buildSpillLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET, 1103 Index, 1104 VData->getReg(), VData->isKill(), 1105 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1106 FrameReg, 1107 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1108 *MI->memoperands_begin(), 1109 RS); 1110 MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode())); 1111 MI->eraseFromParent(); 1112 break; 1113 } 1114 case AMDGPU::SI_SPILL_V32_RESTORE: 1115 case AMDGPU::SI_SPILL_V64_RESTORE: 1116 case AMDGPU::SI_SPILL_V96_RESTORE: 1117 case AMDGPU::SI_SPILL_V128_RESTORE: 1118 case AMDGPU::SI_SPILL_V160_RESTORE: 1119 case AMDGPU::SI_SPILL_V256_RESTORE: 1120 case AMDGPU::SI_SPILL_V512_RESTORE: 1121 case AMDGPU::SI_SPILL_V1024_RESTORE: 1122 case AMDGPU::SI_SPILL_A32_RESTORE: 1123 case AMDGPU::SI_SPILL_A64_RESTORE: 1124 case AMDGPU::SI_SPILL_A128_RESTORE: 1125 case AMDGPU::SI_SPILL_A512_RESTORE: 1126 case AMDGPU::SI_SPILL_A1024_RESTORE: { 1127 const MachineOperand *VData = TII->getNamedOperand(*MI, 1128 AMDGPU::OpName::vdata); 1129 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 1130 MFI->getStackPtrOffsetReg()); 1131 1132 buildSpillLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET, 1133 Index, 1134 VData->getReg(), VData->isKill(), 1135 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1136 FrameReg, 1137 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1138 *MI->memoperands_begin(), 1139 RS); 1140 MI->eraseFromParent(); 1141 break; 1142 } 1143 1144 default: { 1145 const DebugLoc &DL = MI->getDebugLoc(); 1146 bool IsMUBUF = TII->isMUBUF(*MI); 1147 1148 if (!IsMUBUF && !MFI->isEntryFunction()) { 1149 // Convert to a swizzled stack address by scaling by the wave size. 1150 // 1151 // In an entry function/kernel the offset is already swizzled. 1152 1153 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32; 1154 Register ResultReg = 1155 IsCopy ? MI->getOperand(0).getReg() 1156 : RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1157 1158 int64_t Offset = FrameInfo.getObjectOffset(Index); 1159 if (Offset == 0) { 1160 // XXX - This never happens because of emergency scavenging slot at 0? 1161 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ResultReg) 1162 .addImm(ST.getWavefrontSizeLog2()) 1163 .addReg(FrameReg); 1164 } else { 1165 if (auto MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) { 1166 // Reuse ResultReg in intermediate step. 1167 Register ScaledReg = ResultReg; 1168 1169 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), 1170 ScaledReg) 1171 .addImm(ST.getWavefrontSizeLog2()) 1172 .addReg(FrameReg); 1173 1174 const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32; 1175 1176 // TODO: Fold if use instruction is another add of a constant. 1177 if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { 1178 // FIXME: This can fail 1179 MIB.addImm(Offset); 1180 MIB.addReg(ScaledReg, RegState::Kill); 1181 if (!IsVOP2) 1182 MIB.addImm(0); // clamp bit 1183 } else { 1184 assert(MIB->getOpcode() == AMDGPU::V_ADD_I32_e64 && 1185 "Need to reuse carry out register"); 1186 1187 // Use scavenged unused carry out as offset register. 1188 Register ConstOffsetReg; 1189 if (!isWave32) 1190 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); 1191 else 1192 ConstOffsetReg = MIB.getReg(1); 1193 1194 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) 1195 .addImm(Offset); 1196 MIB.addReg(ConstOffsetReg, RegState::Kill); 1197 MIB.addReg(ScaledReg, RegState::Kill); 1198 MIB.addImm(0); // clamp bit 1199 } 1200 } else { 1201 // We have to produce a carry out, and there isn't a free SGPR pair 1202 // for it. We can keep the whole computation on the SALU to avoid 1203 // clobbering an additional register at the cost of an extra mov. 1204 1205 // We may have 1 free scratch SGPR even though a carry out is 1206 // unavailable. Only one additional mov is needed. 1207 Register TmpScaledReg = 1208 RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, false); 1209 Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg; 1210 1211 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg) 1212 .addReg(FrameReg) 1213 .addImm(ST.getWavefrontSizeLog2()); 1214 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), ScaledReg) 1215 .addReg(ScaledReg, RegState::Kill) 1216 .addImm(Offset); 1217 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg) 1218 .addReg(ScaledReg, RegState::Kill); 1219 1220 // If there were truly no free SGPRs, we need to undo everything. 1221 if (!TmpScaledReg.isValid()) { 1222 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScaledReg) 1223 .addReg(ScaledReg, RegState::Kill) 1224 .addImm(Offset); 1225 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) 1226 .addReg(FrameReg) 1227 .addImm(ST.getWavefrontSizeLog2()); 1228 } 1229 } 1230 } 1231 1232 // Don't introduce an extra copy if we're just materializing in a mov. 1233 if (IsCopy) 1234 MI->eraseFromParent(); 1235 else 1236 FIOp.ChangeToRegister(ResultReg, false, false, true); 1237 return; 1238 } 1239 1240 if (IsMUBUF) { 1241 // Disable offen so we don't need a 0 vgpr base. 1242 assert(static_cast<int>(FIOperandNum) == 1243 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1244 AMDGPU::OpName::vaddr)); 1245 1246 auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset); 1247 assert((SOffset.isReg() && 1248 SOffset.getReg() == MFI->getStackPtrOffsetReg()) || 1249 (SOffset.isImm() && SOffset.getImm() == 0)); 1250 if (SOffset.isReg()) { 1251 if (FrameReg == AMDGPU::NoRegister) { 1252 SOffset.ChangeToImmediate(0); 1253 } else { 1254 SOffset.setReg(FrameReg); 1255 } 1256 } 1257 1258 int64_t Offset = FrameInfo.getObjectOffset(Index); 1259 int64_t OldImm 1260 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); 1261 int64_t NewOffset = OldImm + Offset; 1262 1263 if (isUInt<12>(NewOffset) && 1264 buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) { 1265 MI->eraseFromParent(); 1266 return; 1267 } 1268 } 1269 1270 // If the offset is simply too big, don't convert to a scratch wave offset 1271 // relative index. 1272 1273 int64_t Offset = FrameInfo.getObjectOffset(Index); 1274 FIOp.ChangeToImmediate(Offset); 1275 if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) { 1276 Register TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1277 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 1278 .addImm(Offset); 1279 FIOp.ChangeToRegister(TmpReg, false, false, true); 1280 } 1281 } 1282 } 1283 } 1284 1285 StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const { 1286 return AMDGPUInstPrinter::getRegisterName(Reg); 1287 } 1288 1289 const TargetRegisterClass * 1290 SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) { 1291 switch (BitWidth) { 1292 case 1: 1293 return &AMDGPU::VReg_1RegClass; 1294 case 16: 1295 return &AMDGPU::VGPR_LO16RegClass; 1296 case 32: 1297 return &AMDGPU::VGPR_32RegClass; 1298 case 64: 1299 return &AMDGPU::VReg_64RegClass; 1300 case 96: 1301 return &AMDGPU::VReg_96RegClass; 1302 case 128: 1303 return &AMDGPU::VReg_128RegClass; 1304 case 160: 1305 return &AMDGPU::VReg_160RegClass; 1306 case 192: 1307 return &AMDGPU::VReg_192RegClass; 1308 case 256: 1309 return &AMDGPU::VReg_256RegClass; 1310 case 512: 1311 return &AMDGPU::VReg_512RegClass; 1312 case 1024: 1313 return &AMDGPU::VReg_1024RegClass; 1314 default: 1315 return nullptr; 1316 } 1317 } 1318 1319 const TargetRegisterClass * 1320 SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) { 1321 switch (BitWidth) { 1322 case 16: 1323 return &AMDGPU::AGPR_LO16RegClass; 1324 case 32: 1325 return &AMDGPU::AGPR_32RegClass; 1326 case 64: 1327 return &AMDGPU::AReg_64RegClass; 1328 case 96: 1329 return &AMDGPU::AReg_96RegClass; 1330 case 128: 1331 return &AMDGPU::AReg_128RegClass; 1332 case 160: 1333 return &AMDGPU::AReg_160RegClass; 1334 case 192: 1335 return &AMDGPU::AReg_192RegClass; 1336 case 256: 1337 return &AMDGPU::AReg_256RegClass; 1338 case 512: 1339 return &AMDGPU::AReg_512RegClass; 1340 case 1024: 1341 return &AMDGPU::AReg_1024RegClass; 1342 default: 1343 return nullptr; 1344 } 1345 } 1346 1347 const TargetRegisterClass * 1348 SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) { 1349 switch (BitWidth) { 1350 case 16: 1351 return &AMDGPU::SGPR_LO16RegClass; 1352 case 32: 1353 return &AMDGPU::SReg_32RegClass; 1354 case 64: 1355 return &AMDGPU::SReg_64RegClass; 1356 case 96: 1357 return &AMDGPU::SGPR_96RegClass; 1358 case 128: 1359 return &AMDGPU::SGPR_128RegClass; 1360 case 160: 1361 return &AMDGPU::SGPR_160RegClass; 1362 case 192: 1363 return &AMDGPU::SGPR_192RegClass; 1364 case 256: 1365 return &AMDGPU::SGPR_256RegClass; 1366 case 512: 1367 return &AMDGPU::SGPR_512RegClass; 1368 case 1024: 1369 return &AMDGPU::SGPR_1024RegClass; 1370 default: 1371 return nullptr; 1372 } 1373 } 1374 1375 // FIXME: This is very slow. It might be worth creating a map from physreg to 1376 // register class. 1377 const TargetRegisterClass * 1378 SIRegisterInfo::getPhysRegClass(MCRegister Reg) const { 1379 static const TargetRegisterClass *const BaseClasses[] = { 1380 &AMDGPU::VGPR_LO16RegClass, 1381 &AMDGPU::VGPR_HI16RegClass, 1382 &AMDGPU::SReg_LO16RegClass, 1383 &AMDGPU::AGPR_LO16RegClass, 1384 &AMDGPU::VGPR_32RegClass, 1385 &AMDGPU::SReg_32RegClass, 1386 &AMDGPU::AGPR_32RegClass, 1387 &AMDGPU::VReg_64RegClass, 1388 &AMDGPU::SReg_64RegClass, 1389 &AMDGPU::AReg_64RegClass, 1390 &AMDGPU::VReg_96RegClass, 1391 &AMDGPU::SReg_96RegClass, 1392 &AMDGPU::AReg_96RegClass, 1393 &AMDGPU::VReg_128RegClass, 1394 &AMDGPU::SReg_128RegClass, 1395 &AMDGPU::AReg_128RegClass, 1396 &AMDGPU::VReg_160RegClass, 1397 &AMDGPU::SReg_160RegClass, 1398 &AMDGPU::AReg_160RegClass, 1399 &AMDGPU::VReg_192RegClass, 1400 &AMDGPU::SReg_192RegClass, 1401 &AMDGPU::AReg_192RegClass, 1402 &AMDGPU::VReg_256RegClass, 1403 &AMDGPU::SReg_256RegClass, 1404 &AMDGPU::AReg_256RegClass, 1405 &AMDGPU::VReg_512RegClass, 1406 &AMDGPU::SReg_512RegClass, 1407 &AMDGPU::AReg_512RegClass, 1408 &AMDGPU::SReg_1024RegClass, 1409 &AMDGPU::VReg_1024RegClass, 1410 &AMDGPU::AReg_1024RegClass, 1411 &AMDGPU::SCC_CLASSRegClass, 1412 &AMDGPU::Pseudo_SReg_32RegClass, 1413 &AMDGPU::Pseudo_SReg_128RegClass, 1414 }; 1415 1416 for (const TargetRegisterClass *BaseClass : BaseClasses) { 1417 if (BaseClass->contains(Reg)) { 1418 return BaseClass; 1419 } 1420 } 1421 return nullptr; 1422 } 1423 1424 // TODO: It might be helpful to have some target specific flags in 1425 // TargetRegisterClass to mark which classes are VGPRs to make this trivial. 1426 bool SIRegisterInfo::hasVGPRs(const TargetRegisterClass *RC) const { 1427 unsigned Size = getRegSizeInBits(*RC); 1428 if (Size == 16) { 1429 return getCommonSubClass(&AMDGPU::VGPR_LO16RegClass, RC) != nullptr || 1430 getCommonSubClass(&AMDGPU::VGPR_HI16RegClass, RC) != nullptr; 1431 } 1432 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 1433 if (!VRC) { 1434 assert(Size < 32 && "Invalid register class size"); 1435 return false; 1436 } 1437 return getCommonSubClass(VRC, RC) != nullptr; 1438 } 1439 1440 bool SIRegisterInfo::hasAGPRs(const TargetRegisterClass *RC) const { 1441 unsigned Size = getRegSizeInBits(*RC); 1442 if (Size < 16) 1443 return false; 1444 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 1445 if (!ARC) { 1446 assert(getVGPRClassForBitWidth(Size) && "Invalid register class size"); 1447 return false; 1448 } 1449 return getCommonSubClass(ARC, RC) != nullptr; 1450 } 1451 1452 const TargetRegisterClass * 1453 SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const { 1454 unsigned Size = getRegSizeInBits(*SRC); 1455 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 1456 assert(VRC && "Invalid register class size"); 1457 return VRC; 1458 } 1459 1460 const TargetRegisterClass * 1461 SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const { 1462 unsigned Size = getRegSizeInBits(*SRC); 1463 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 1464 assert(ARC && "Invalid register class size"); 1465 return ARC; 1466 } 1467 1468 const TargetRegisterClass * 1469 SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const { 1470 unsigned Size = getRegSizeInBits(*VRC); 1471 if (Size == 32) 1472 return &AMDGPU::SGPR_32RegClass; 1473 const TargetRegisterClass *SRC = getSGPRClassForBitWidth(Size); 1474 assert(SRC && "Invalid register class size"); 1475 return SRC; 1476 } 1477 1478 const TargetRegisterClass *SIRegisterInfo::getSubRegClass( 1479 const TargetRegisterClass *RC, unsigned SubIdx) const { 1480 if (SubIdx == AMDGPU::NoSubRegister) 1481 return RC; 1482 1483 // We can assume that each lane corresponds to one 32-bit register. 1484 unsigned Size = getNumChannelsFromSubReg(SubIdx) * 32; 1485 if (isSGPRClass(RC)) { 1486 if (Size == 32) 1487 RC = &AMDGPU::SGPR_32RegClass; 1488 else 1489 RC = getSGPRClassForBitWidth(Size); 1490 } else if (hasAGPRs(RC)) { 1491 RC = getAGPRClassForBitWidth(Size); 1492 } else { 1493 RC = getVGPRClassForBitWidth(Size); 1494 } 1495 assert(RC && "Invalid sub-register class size"); 1496 return RC; 1497 } 1498 1499 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const { 1500 if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST && 1501 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST) 1502 return !ST.hasMFMAInlineLiteralBug(); 1503 1504 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1505 OpType <= AMDGPU::OPERAND_SRC_LAST; 1506 } 1507 1508 bool SIRegisterInfo::shouldRewriteCopySrc( 1509 const TargetRegisterClass *DefRC, 1510 unsigned DefSubReg, 1511 const TargetRegisterClass *SrcRC, 1512 unsigned SrcSubReg) const { 1513 // We want to prefer the smallest register class possible, so we don't want to 1514 // stop and rewrite on anything that looks like a subregister 1515 // extract. Operations mostly don't care about the super register class, so we 1516 // only want to stop on the most basic of copies between the same register 1517 // class. 1518 // 1519 // e.g. if we have something like 1520 // %0 = ... 1521 // %1 = ... 1522 // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2 1523 // %3 = COPY %2, sub0 1524 // 1525 // We want to look through the COPY to find: 1526 // => %3 = COPY %0 1527 1528 // Plain copy. 1529 return getCommonSubClass(DefRC, SrcRC) != nullptr; 1530 } 1531 1532 /// Returns a register that is not used at any point in the function. 1533 /// If all registers are used, then this function will return 1534 // AMDGPU::NoRegister. 1535 MCRegister 1536 SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI, 1537 const TargetRegisterClass *RC, 1538 const MachineFunction &MF) const { 1539 1540 for (MCRegister Reg : *RC) 1541 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 1542 return Reg; 1543 return MCRegister(); 1544 } 1545 1546 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC, 1547 unsigned EltSize) const { 1548 const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC->MC); 1549 assert(RegBitWidth >= 32 && RegBitWidth <= 1024); 1550 1551 const unsigned EltBitWidth = EltSize * 8; 1552 assert(EltBitWidth >= 32 && EltBitWidth < 1024 && isPowerOf2_32(EltBitWidth)); 1553 const unsigned LogEltBitWidth = Log2_32(EltBitWidth); 1554 1555 assert(RegBitWidth % EltBitWidth == 0); 1556 1557 if (RegBitWidth == EltBitWidth) 1558 return {}; 1559 1560 static const int16_t Sub_32[] = { 1561 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1562 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 1563 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 1564 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 1565 AMDGPU::sub16, AMDGPU::sub17, AMDGPU::sub18, AMDGPU::sub19, 1566 AMDGPU::sub20, AMDGPU::sub21, AMDGPU::sub22, AMDGPU::sub23, 1567 AMDGPU::sub24, AMDGPU::sub25, AMDGPU::sub26, AMDGPU::sub27, 1568 AMDGPU::sub28, AMDGPU::sub29, AMDGPU::sub30, AMDGPU::sub31 1569 }; 1570 1571 static const int16_t Sub_64[] = { 1572 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 1573 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 1574 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 1575 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 1576 AMDGPU::sub16_sub17, AMDGPU::sub18_sub19, 1577 AMDGPU::sub20_sub21, AMDGPU::sub22_sub23, 1578 AMDGPU::sub24_sub25, AMDGPU::sub26_sub27, 1579 AMDGPU::sub28_sub29, AMDGPU::sub30_sub31 1580 }; 1581 1582 static const int16_t Sub_128[] = { 1583 AMDGPU::sub0_sub1_sub2_sub3, 1584 AMDGPU::sub4_sub5_sub6_sub7, 1585 AMDGPU::sub8_sub9_sub10_sub11, 1586 AMDGPU::sub12_sub13_sub14_sub15, 1587 AMDGPU::sub16_sub17_sub18_sub19, 1588 AMDGPU::sub20_sub21_sub22_sub23, 1589 AMDGPU::sub24_sub25_sub26_sub27, 1590 AMDGPU::sub28_sub29_sub30_sub31 1591 }; 1592 1593 static const int16_t Sub_256[] = { 1594 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7, 1595 AMDGPU::sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 1596 AMDGPU::sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23, 1597 AMDGPU::sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31 1598 }; 1599 1600 static const int16_t Sub_512[] = { 1601 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 1602 AMDGPU::sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23_sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31 1603 }; 1604 1605 static const int16_t *const Subs[] = { 1606 Sub_32, Sub_64, Sub_128, Sub_256, Sub_512 1607 }; 1608 1609 return makeArrayRef(Subs[LogEltBitWidth - 5], RegBitWidth >> LogEltBitWidth); 1610 } 1611 1612 const TargetRegisterClass* 1613 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI, 1614 Register Reg) const { 1615 return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegClass(Reg); 1616 } 1617 1618 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI, 1619 Register Reg) const { 1620 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 1621 assert(RC && "Register class for the reg not found"); 1622 return hasVGPRs(RC); 1623 } 1624 1625 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI, 1626 Register Reg) const { 1627 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 1628 assert(RC && "Register class for the reg not found"); 1629 return hasAGPRs(RC); 1630 } 1631 1632 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI, 1633 const TargetRegisterClass *SrcRC, 1634 unsigned SubReg, 1635 const TargetRegisterClass *DstRC, 1636 unsigned DstSubReg, 1637 const TargetRegisterClass *NewRC, 1638 LiveIntervals &LIS) const { 1639 unsigned SrcSize = getRegSizeInBits(*SrcRC); 1640 unsigned DstSize = getRegSizeInBits(*DstRC); 1641 unsigned NewSize = getRegSizeInBits(*NewRC); 1642 1643 // Do not increase size of registers beyond dword, we would need to allocate 1644 // adjacent registers and constraint regalloc more than needed. 1645 1646 // Always allow dword coalescing. 1647 if (SrcSize <= 32 || DstSize <= 32) 1648 return true; 1649 1650 return NewSize <= DstSize || NewSize <= SrcSize; 1651 } 1652 1653 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 1654 MachineFunction &MF) const { 1655 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1656 1657 unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), 1658 MF.getFunction()); 1659 switch (RC->getID()) { 1660 default: 1661 return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF); 1662 case AMDGPU::VGPR_32RegClassID: 1663 case AMDGPU::VGPR_LO16RegClassID: 1664 case AMDGPU::VGPR_HI16RegClassID: 1665 return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF)); 1666 case AMDGPU::SGPR_32RegClassID: 1667 case AMDGPU::SGPR_LO16RegClassID: 1668 return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF)); 1669 } 1670 } 1671 1672 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 1673 unsigned Idx) const { 1674 if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 || 1675 Idx == AMDGPU::RegisterPressureSets::AGPR_32) 1676 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 1677 const_cast<MachineFunction &>(MF)); 1678 1679 if (Idx == AMDGPU::RegisterPressureSets::SReg_32) 1680 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, 1681 const_cast<MachineFunction &>(MF)); 1682 1683 llvm_unreachable("Unexpected register pressure set!"); 1684 } 1685 1686 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const { 1687 static const int Empty[] = { -1 }; 1688 1689 if (RegPressureIgnoredUnits[RegUnit]) 1690 return Empty; 1691 1692 return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit); 1693 } 1694 1695 MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { 1696 // Not a callee saved register. 1697 return AMDGPU::SGPR30_SGPR31; 1698 } 1699 1700 const TargetRegisterClass * 1701 SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, 1702 const RegisterBank &RB, 1703 const MachineRegisterInfo &MRI) const { 1704 switch (RB.getID()) { 1705 case AMDGPU::VGPRRegBankID: 1706 return getVGPRClassForBitWidth(std::max(32u, Size)); 1707 case AMDGPU::VCCRegBankID: 1708 assert(Size == 1); 1709 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 1710 : &AMDGPU::SReg_64_XEXECRegClass; 1711 case AMDGPU::SGPRRegBankID: 1712 return getSGPRClassForBitWidth(std::max(32u, Size)); 1713 default: 1714 llvm_unreachable("unknown register bank"); 1715 } 1716 } 1717 1718 const TargetRegisterClass * 1719 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO, 1720 const MachineRegisterInfo &MRI) const { 1721 const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg()); 1722 if (const RegisterBank *RB = RCOrRB.dyn_cast<const RegisterBank*>()) 1723 return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB, MRI); 1724 1725 const TargetRegisterClass *RC = RCOrRB.get<const TargetRegisterClass*>(); 1726 return getAllocatableClass(RC); 1727 } 1728 1729 MCRegister SIRegisterInfo::getVCC() const { 1730 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; 1731 } 1732 1733 const TargetRegisterClass * 1734 SIRegisterInfo::getRegClass(unsigned RCID) const { 1735 switch ((int)RCID) { 1736 case AMDGPU::SReg_1RegClassID: 1737 return getBoolRC(); 1738 case AMDGPU::SReg_1_XEXECRegClassID: 1739 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 1740 : &AMDGPU::SReg_64_XEXECRegClass; 1741 case -1: 1742 return nullptr; 1743 default: 1744 return AMDGPUGenRegisterInfo::getRegClass(RCID); 1745 } 1746 } 1747 1748 // Find reaching register definition 1749 MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg, 1750 MachineInstr &Use, 1751 MachineRegisterInfo &MRI, 1752 LiveIntervals *LIS) const { 1753 auto &MDT = LIS->getAnalysis<MachineDominatorTree>(); 1754 SlotIndex UseIdx = LIS->getInstructionIndex(Use); 1755 SlotIndex DefIdx; 1756 1757 if (Reg.isVirtual()) { 1758 if (!LIS->hasInterval(Reg)) 1759 return nullptr; 1760 LiveInterval &LI = LIS->getInterval(Reg); 1761 LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg) 1762 : MRI.getMaxLaneMaskForVReg(Reg); 1763 VNInfo *V = nullptr; 1764 if (LI.hasSubRanges()) { 1765 for (auto &S : LI.subranges()) { 1766 if ((S.LaneMask & SubLanes) == SubLanes) { 1767 V = S.getVNInfoAt(UseIdx); 1768 break; 1769 } 1770 } 1771 } else { 1772 V = LI.getVNInfoAt(UseIdx); 1773 } 1774 if (!V) 1775 return nullptr; 1776 DefIdx = V->def; 1777 } else { 1778 // Find last def. 1779 for (MCRegUnitIterator Units(Reg, this); Units.isValid(); ++Units) { 1780 LiveRange &LR = LIS->getRegUnit(*Units); 1781 if (VNInfo *V = LR.getVNInfoAt(UseIdx)) { 1782 if (!DefIdx.isValid() || 1783 MDT.dominates(LIS->getInstructionFromIndex(DefIdx), 1784 LIS->getInstructionFromIndex(V->def))) 1785 DefIdx = V->def; 1786 } else { 1787 return nullptr; 1788 } 1789 } 1790 } 1791 1792 MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx); 1793 1794 if (!Def || !MDT.dominates(Def, &Use)) 1795 return nullptr; 1796 1797 assert(Def->modifiesRegister(Reg, this)); 1798 1799 return Def; 1800 } 1801 1802 MCPhysReg SIRegisterInfo::get32BitRegister(MCPhysReg Reg) const { 1803 assert(getRegSizeInBits(*getPhysRegClass(Reg)) <= 32); 1804 1805 for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass, 1806 AMDGPU::SReg_32RegClass, 1807 AMDGPU::AGPR_32RegClass } ) { 1808 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC)) 1809 return Super; 1810 } 1811 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16, 1812 &AMDGPU::VGPR_32RegClass)) { 1813 return Super; 1814 } 1815 1816 return AMDGPU::NoRegister; 1817 } 1818