1 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI implementation of the TargetRegisterInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIRegisterInfo.h" 15 #include "AMDGPURegisterBankInfo.h" 16 #include "AMDGPUSubtarget.h" 17 #include "SIInstrInfo.h" 18 #include "SIMachineFunctionInfo.h" 19 #include "MCTargetDesc/AMDGPUInstPrinter.h" 20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include "llvm/CodeGen/MachineDominators.h" 23 #include "llvm/CodeGen/MachineFrameInfo.h" 24 #include "llvm/CodeGen/MachineInstrBuilder.h" 25 #include "llvm/CodeGen/RegisterScavenging.h" 26 #include "llvm/CodeGen/SlotIndexes.h" 27 #include "llvm/IR/Function.h" 28 #include "llvm/IR/LLVMContext.h" 29 30 using namespace llvm; 31 32 #define GET_REGINFO_TARGET_DESC 33 #include "AMDGPUGenRegisterInfo.inc" 34 35 static cl::opt<bool> EnableSpillSGPRToVGPR( 36 "amdgpu-spill-sgpr-to-vgpr", 37 cl::desc("Enable spilling VGPRs to SGPRs"), 38 cl::ReallyHidden, 39 cl::init(true)); 40 41 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) 42 : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST), 43 SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) { 44 45 assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 && 46 getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) && 47 (getSubRegIndexLaneMask(AMDGPU::lo16) | 48 getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() == 49 getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() && 50 "getNumCoveredRegs() will not work with generated subreg masks!"); 51 52 RegPressureIgnoredUnits.resize(getNumRegUnits()); 53 RegPressureIgnoredUnits.set(*MCRegUnitIterator(AMDGPU::M0, this)); 54 for (auto Reg : AMDGPU::VGPR_HI16RegClass) 55 RegPressureIgnoredUnits.set(*MCRegUnitIterator(Reg, this)); 56 } 57 58 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, 59 MCRegister Reg) const { 60 MCRegAliasIterator R(Reg, this, true); 61 62 for (; R.isValid(); ++R) 63 Reserved.set(*R); 64 } 65 66 // Forced to be here by one .inc 67 const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs( 68 const MachineFunction *MF) const { 69 CallingConv::ID CC = MF->getFunction().getCallingConv(); 70 switch (CC) { 71 case CallingConv::C: 72 case CallingConv::Fast: 73 case CallingConv::Cold: 74 return CSR_AMDGPU_HighRegs_SaveList; 75 default: { 76 // Dummy to not crash RegisterClassInfo. 77 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister; 78 return &NoCalleeSavedReg; 79 } 80 } 81 } 82 83 const MCPhysReg * 84 SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const { 85 return nullptr; 86 } 87 88 const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF, 89 CallingConv::ID CC) const { 90 switch (CC) { 91 case CallingConv::C: 92 case CallingConv::Fast: 93 case CallingConv::Cold: 94 return CSR_AMDGPU_HighRegs_RegMask; 95 default: 96 return nullptr; 97 } 98 } 99 100 Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const { 101 const SIFrameLowering *TFI = 102 MF.getSubtarget<GCNSubtarget>().getFrameLowering(); 103 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); 104 // During ISel lowering we always reserve the stack pointer in entry 105 // functions, but never actually want to reference it when accessing our own 106 // frame. If we need a frame pointer we use it, but otherwise we can just use 107 // an immediate "0" which we represent by returning NoRegister. 108 if (FuncInfo->isEntryFunction()) { 109 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() : Register(); 110 } 111 return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg() 112 : FuncInfo->getStackPtrOffsetReg(); 113 } 114 115 bool SIRegisterInfo::hasBasePointer(const MachineFunction &MF) const { 116 // When we need stack realignment, we can't reference off of the 117 // stack pointer, so we reserve a base pointer. 118 const MachineFrameInfo &MFI = MF.getFrameInfo(); 119 return MFI.getNumFixedObjects() && needsStackRealignment(MF); 120 } 121 122 Register SIRegisterInfo::getBaseRegister() const { return AMDGPU::SGPR34; } 123 124 const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const { 125 return CSR_AMDGPU_AllVGPRs_RegMask; 126 } 127 128 const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const { 129 return CSR_AMDGPU_AllAllocatableSRegs_RegMask; 130 } 131 132 // FIXME: TableGen should generate something to make this manageable for all 133 // register classes. At a minimum we could use the opposite of 134 // composeSubRegIndices and go up from the base 32-bit subreg. 135 unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel, 136 unsigned NumRegs) { 137 // Table of NumRegs sized pieces at every 32-bit offset. 138 static const uint16_t SubRegFromChannelTable[][32] = { 139 {AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 140 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 141 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 142 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 143 AMDGPU::sub16, AMDGPU::sub17, AMDGPU::sub18, AMDGPU::sub19, 144 AMDGPU::sub20, AMDGPU::sub21, AMDGPU::sub22, AMDGPU::sub23, 145 AMDGPU::sub24, AMDGPU::sub25, AMDGPU::sub26, AMDGPU::sub27, 146 AMDGPU::sub28, AMDGPU::sub29, AMDGPU::sub30, AMDGPU::sub31}, 147 {AMDGPU::sub0_sub1, AMDGPU::sub1_sub2, AMDGPU::sub2_sub3, 148 AMDGPU::sub3_sub4, AMDGPU::sub4_sub5, AMDGPU::sub5_sub6, 149 AMDGPU::sub6_sub7, AMDGPU::sub7_sub8, AMDGPU::sub8_sub9, 150 AMDGPU::sub9_sub10, AMDGPU::sub10_sub11, AMDGPU::sub11_sub12, 151 AMDGPU::sub12_sub13, AMDGPU::sub13_sub14, AMDGPU::sub14_sub15, 152 AMDGPU::sub15_sub16, AMDGPU::sub16_sub17, AMDGPU::sub17_sub18, 153 AMDGPU::sub18_sub19, AMDGPU::sub19_sub20, AMDGPU::sub20_sub21, 154 AMDGPU::sub21_sub22, AMDGPU::sub22_sub23, AMDGPU::sub23_sub24, 155 AMDGPU::sub24_sub25, AMDGPU::sub25_sub26, AMDGPU::sub26_sub27, 156 AMDGPU::sub27_sub28, AMDGPU::sub28_sub29, AMDGPU::sub29_sub30, 157 AMDGPU::sub30_sub31, AMDGPU::NoSubRegister}, 158 {AMDGPU::sub0_sub1_sub2, AMDGPU::sub1_sub2_sub3, 159 AMDGPU::sub2_sub3_sub4, AMDGPU::sub3_sub4_sub5, 160 AMDGPU::sub4_sub5_sub6, AMDGPU::sub5_sub6_sub7, 161 AMDGPU::sub6_sub7_sub8, AMDGPU::sub7_sub8_sub9, 162 AMDGPU::sub8_sub9_sub10, AMDGPU::sub9_sub10_sub11, 163 AMDGPU::sub10_sub11_sub12, AMDGPU::sub11_sub12_sub13, 164 AMDGPU::sub12_sub13_sub14, AMDGPU::sub13_sub14_sub15, 165 AMDGPU::sub14_sub15_sub16, AMDGPU::sub15_sub16_sub17, 166 AMDGPU::sub16_sub17_sub18, AMDGPU::sub17_sub18_sub19, 167 AMDGPU::sub18_sub19_sub20, AMDGPU::sub19_sub20_sub21, 168 AMDGPU::sub20_sub21_sub22, AMDGPU::sub21_sub22_sub23, 169 AMDGPU::sub22_sub23_sub24, AMDGPU::sub23_sub24_sub25, 170 AMDGPU::sub24_sub25_sub26, AMDGPU::sub25_sub26_sub27, 171 AMDGPU::sub26_sub27_sub28, AMDGPU::sub27_sub28_sub29, 172 AMDGPU::sub28_sub29_sub30, AMDGPU::sub29_sub30_sub31, 173 AMDGPU::NoSubRegister, AMDGPU::NoSubRegister}, 174 {AMDGPU::sub0_sub1_sub2_sub3, AMDGPU::sub1_sub2_sub3_sub4, 175 AMDGPU::sub2_sub3_sub4_sub5, AMDGPU::sub3_sub4_sub5_sub6, 176 AMDGPU::sub4_sub5_sub6_sub7, AMDGPU::sub5_sub6_sub7_sub8, 177 AMDGPU::sub6_sub7_sub8_sub9, AMDGPU::sub7_sub8_sub9_sub10, 178 AMDGPU::sub8_sub9_sub10_sub11, AMDGPU::sub9_sub10_sub11_sub12, 179 AMDGPU::sub10_sub11_sub12_sub13, AMDGPU::sub11_sub12_sub13_sub14, 180 AMDGPU::sub12_sub13_sub14_sub15, AMDGPU::sub13_sub14_sub15_sub16, 181 AMDGPU::sub14_sub15_sub16_sub17, AMDGPU::sub15_sub16_sub17_sub18, 182 AMDGPU::sub16_sub17_sub18_sub19, AMDGPU::sub17_sub18_sub19_sub20, 183 AMDGPU::sub18_sub19_sub20_sub21, AMDGPU::sub19_sub20_sub21_sub22, 184 AMDGPU::sub20_sub21_sub22_sub23, AMDGPU::sub21_sub22_sub23_sub24, 185 AMDGPU::sub22_sub23_sub24_sub25, AMDGPU::sub23_sub24_sub25_sub26, 186 AMDGPU::sub24_sub25_sub26_sub27, AMDGPU::sub25_sub26_sub27_sub28, 187 AMDGPU::sub26_sub27_sub28_sub29, AMDGPU::sub27_sub28_sub29_sub30, 188 AMDGPU::sub28_sub29_sub30_sub31, AMDGPU::NoSubRegister, 189 AMDGPU::NoSubRegister, AMDGPU::NoSubRegister}}; 190 191 const unsigned NumRegIndex = NumRegs - 1; 192 193 assert(NumRegIndex < array_lengthof(SubRegFromChannelTable) && 194 "Not implemented"); 195 assert(Channel < array_lengthof(SubRegFromChannelTable[0])); 196 return SubRegFromChannelTable[NumRegIndex][Channel]; 197 } 198 199 MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg( 200 const MachineFunction &MF) const { 201 unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4; 202 MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); 203 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass); 204 } 205 206 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const { 207 BitVector Reserved(getNumRegs()); 208 Reserved.set(AMDGPU::MODE); 209 210 // EXEC_LO and EXEC_HI could be allocated and used as regular register, but 211 // this seems likely to result in bugs, so I'm marking them as reserved. 212 reserveRegisterTuples(Reserved, AMDGPU::EXEC); 213 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); 214 215 // M0 has to be reserved so that llvm accepts it as a live-in into a block. 216 reserveRegisterTuples(Reserved, AMDGPU::M0); 217 218 // Reserve src_vccz, src_execz, src_scc. 219 reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ); 220 reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ); 221 reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC); 222 223 // Reserve the memory aperture registers. 224 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); 225 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); 226 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); 227 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); 228 229 // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen. 230 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); 231 232 // Reserve xnack_mask registers - support is not implemented in Codegen. 233 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); 234 235 // Reserve lds_direct register - support is not implemented in Codegen. 236 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); 237 238 // Reserve Trap Handler registers - support is not implemented in Codegen. 239 reserveRegisterTuples(Reserved, AMDGPU::TBA); 240 reserveRegisterTuples(Reserved, AMDGPU::TMA); 241 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); 242 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); 243 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); 244 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); 245 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); 246 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); 247 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); 248 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); 249 250 // Reserve null register - it shall never be allocated 251 reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL); 252 253 // Disallow vcc_hi allocation in wave32. It may be allocated but most likely 254 // will result in bugs. 255 if (isWave32) { 256 Reserved.set(AMDGPU::VCC); 257 Reserved.set(AMDGPU::VCC_HI); 258 } 259 260 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); 261 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); 262 for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { 263 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); 264 reserveRegisterTuples(Reserved, Reg); 265 } 266 267 unsigned MaxNumVGPRs = ST.getMaxNumVGPRs(MF); 268 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); 269 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) { 270 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); 271 reserveRegisterTuples(Reserved, Reg); 272 Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 273 reserveRegisterTuples(Reserved, Reg); 274 } 275 276 for (auto Reg : AMDGPU::SReg_32RegClass) { 277 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 278 Register Low = getSubReg(Reg, AMDGPU::lo16); 279 // This is to prevent BB vcc liveness errors. 280 if (!AMDGPU::SGPR_LO16RegClass.contains(Low)) 281 Reserved.set(Low); 282 } 283 284 for (auto Reg : AMDGPU::AGPR_32RegClass) { 285 Reserved.set(getSubReg(Reg, AMDGPU::hi16)); 286 } 287 288 // Reserve all the rest AGPRs if there are no instructions to use it. 289 if (!ST.hasMAIInsts()) { 290 for (unsigned i = 0; i < MaxNumVGPRs; ++i) { 291 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); 292 reserveRegisterTuples(Reserved, Reg); 293 } 294 } 295 296 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 297 298 unsigned ScratchRSrcReg = MFI->getScratchRSrcReg(); 299 if (ScratchRSrcReg != AMDGPU::NoRegister) { 300 // Reserve 4 SGPRs for the scratch buffer resource descriptor in case we need 301 // to spill. 302 // TODO: May need to reserve a VGPR if doing LDS spilling. 303 reserveRegisterTuples(Reserved, ScratchRSrcReg); 304 } 305 306 // We have to assume the SP is needed in case there are calls in the function, 307 // which is detected after the function is lowered. If we aren't really going 308 // to need SP, don't bother reserving it. 309 MCRegister StackPtrReg = MFI->getStackPtrOffsetReg(); 310 311 if (StackPtrReg) { 312 reserveRegisterTuples(Reserved, StackPtrReg); 313 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg)); 314 } 315 316 MCRegister FrameReg = MFI->getFrameOffsetReg(); 317 if (FrameReg) { 318 reserveRegisterTuples(Reserved, FrameReg); 319 assert(!isSubRegister(ScratchRSrcReg, FrameReg)); 320 } 321 322 if (hasBasePointer(MF)) { 323 MCRegister BasePtrReg = getBaseRegister(); 324 reserveRegisterTuples(Reserved, BasePtrReg); 325 assert(!isSubRegister(ScratchRSrcReg, BasePtrReg)); 326 } 327 328 for (MCRegister Reg : MFI->WWMReservedRegs) { 329 reserveRegisterTuples(Reserved, Reg); 330 } 331 332 // FIXME: Stop using reserved registers for this. 333 for (MCPhysReg Reg : MFI->getAGPRSpillVGPRs()) 334 reserveRegisterTuples(Reserved, Reg); 335 336 for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs()) 337 reserveRegisterTuples(Reserved, Reg); 338 339 if (MFI->VGPRReservedForSGPRSpill) 340 for (auto SSpill : MFI->getSGPRSpillVGPRs()) 341 reserveRegisterTuples(Reserved, SSpill.VGPR); 342 343 return Reserved; 344 } 345 346 bool SIRegisterInfo::canRealignStack(const MachineFunction &MF) const { 347 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 348 // On entry, the base address is 0, so it can't possibly need any more 349 // alignment. 350 351 // FIXME: Should be able to specify the entry frame alignment per calling 352 // convention instead. 353 if (Info->isEntryFunction()) 354 return false; 355 356 return TargetRegisterInfo::canRealignStack(MF); 357 } 358 359 bool SIRegisterInfo::requiresRegisterScavenging(const MachineFunction &Fn) const { 360 const SIMachineFunctionInfo *Info = Fn.getInfo<SIMachineFunctionInfo>(); 361 if (Info->isEntryFunction()) { 362 const MachineFrameInfo &MFI = Fn.getFrameInfo(); 363 return MFI.hasStackObjects() || MFI.hasCalls(); 364 } 365 366 // May need scavenger for dealing with callee saved registers. 367 return true; 368 } 369 370 bool SIRegisterInfo::requiresFrameIndexScavenging( 371 const MachineFunction &MF) const { 372 // Do not use frame virtual registers. They used to be used for SGPRs, but 373 // once we reach PrologEpilogInserter, we can no longer spill SGPRs. If the 374 // scavenger fails, we can increment/decrement the necessary SGPRs to avoid a 375 // spill. 376 return false; 377 } 378 379 bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( 380 const MachineFunction &MF) const { 381 const MachineFrameInfo &MFI = MF.getFrameInfo(); 382 return MFI.hasStackObjects(); 383 } 384 385 bool SIRegisterInfo::requiresVirtualBaseRegisters( 386 const MachineFunction &) const { 387 // There are no special dedicated stack or frame pointers. 388 return true; 389 } 390 391 int64_t SIRegisterInfo::getMUBUFInstrOffset(const MachineInstr *MI) const { 392 assert(SIInstrInfo::isMUBUF(*MI)); 393 394 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 395 AMDGPU::OpName::offset); 396 return MI->getOperand(OffIdx).getImm(); 397 } 398 399 int64_t SIRegisterInfo::getFrameIndexInstrOffset(const MachineInstr *MI, 400 int Idx) const { 401 if (!SIInstrInfo::isMUBUF(*MI)) 402 return 0; 403 404 assert(Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), 405 AMDGPU::OpName::vaddr) && 406 "Should never see frame index on non-address operand"); 407 408 return getMUBUFInstrOffset(MI); 409 } 410 411 bool SIRegisterInfo::needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const { 412 if (!MI->mayLoadOrStore()) 413 return false; 414 415 int64_t FullOffset = Offset + getMUBUFInstrOffset(MI); 416 417 return !isUInt<12>(FullOffset); 418 } 419 420 void SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 421 Register BaseReg, 422 int FrameIdx, 423 int64_t Offset) const { 424 MachineBasicBlock::iterator Ins = MBB->begin(); 425 DebugLoc DL; // Defaults to "unknown" 426 427 if (Ins != MBB->end()) 428 DL = Ins->getDebugLoc(); 429 430 MachineFunction *MF = MBB->getParent(); 431 const SIInstrInfo *TII = ST.getInstrInfo(); 432 433 if (Offset == 0) { 434 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg) 435 .addFrameIndex(FrameIdx); 436 return; 437 } 438 439 MachineRegisterInfo &MRI = MF->getRegInfo(); 440 Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 441 442 Register FIReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 443 444 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) 445 .addImm(Offset); 446 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), FIReg) 447 .addFrameIndex(FrameIdx); 448 449 TII->getAddNoCarry(*MBB, Ins, DL, BaseReg) 450 .addReg(OffsetReg, RegState::Kill) 451 .addReg(FIReg) 452 .addImm(0); // clamp bit 453 } 454 455 void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, 456 int64_t Offset) const { 457 const SIInstrInfo *TII = ST.getInstrInfo(); 458 459 #ifndef NDEBUG 460 // FIXME: Is it possible to be storing a frame index to itself? 461 bool SeenFI = false; 462 for (const MachineOperand &MO: MI.operands()) { 463 if (MO.isFI()) { 464 if (SeenFI) 465 llvm_unreachable("should not see multiple frame indices"); 466 467 SeenFI = true; 468 } 469 } 470 #endif 471 472 MachineOperand *FIOp = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr); 473 #ifndef NDEBUG 474 MachineBasicBlock *MBB = MI.getParent(); 475 MachineFunction *MF = MBB->getParent(); 476 #endif 477 assert(FIOp && FIOp->isFI() && "frame index must be address operand"); 478 assert(TII->isMUBUF(MI)); 479 assert(TII->getNamedOperand(MI, AMDGPU::OpName::soffset)->getReg() == 480 MF->getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg() && 481 "should only be seeing stack pointer offset relative FrameIndex"); 482 483 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); 484 int64_t NewOffset = OffsetOp->getImm() + Offset; 485 assert(isUInt<12>(NewOffset) && "offset should be legal"); 486 487 FIOp->ChangeToRegister(BaseReg, false); 488 OffsetOp->setImm(NewOffset); 489 } 490 491 bool SIRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 492 Register BaseReg, 493 int64_t Offset) const { 494 if (!SIInstrInfo::isMUBUF(*MI)) 495 return false; 496 497 int64_t NewOffset = Offset + getMUBUFInstrOffset(MI); 498 499 return isUInt<12>(NewOffset); 500 } 501 502 const TargetRegisterClass *SIRegisterInfo::getPointerRegClass( 503 const MachineFunction &MF, unsigned Kind) const { 504 // This is inaccurate. It depends on the instruction and address space. The 505 // only place where we should hit this is for dealing with frame indexes / 506 // private accesses, so this is correct in that case. 507 return &AMDGPU::VGPR_32RegClass; 508 } 509 510 static unsigned getNumSubRegsForSpillOp(unsigned Op) { 511 512 switch (Op) { 513 case AMDGPU::SI_SPILL_S1024_SAVE: 514 case AMDGPU::SI_SPILL_S1024_RESTORE: 515 case AMDGPU::SI_SPILL_V1024_SAVE: 516 case AMDGPU::SI_SPILL_V1024_RESTORE: 517 case AMDGPU::SI_SPILL_A1024_SAVE: 518 case AMDGPU::SI_SPILL_A1024_RESTORE: 519 return 32; 520 case AMDGPU::SI_SPILL_S512_SAVE: 521 case AMDGPU::SI_SPILL_S512_RESTORE: 522 case AMDGPU::SI_SPILL_V512_SAVE: 523 case AMDGPU::SI_SPILL_V512_RESTORE: 524 case AMDGPU::SI_SPILL_A512_SAVE: 525 case AMDGPU::SI_SPILL_A512_RESTORE: 526 return 16; 527 case AMDGPU::SI_SPILL_S256_SAVE: 528 case AMDGPU::SI_SPILL_S256_RESTORE: 529 case AMDGPU::SI_SPILL_V256_SAVE: 530 case AMDGPU::SI_SPILL_V256_RESTORE: 531 return 8; 532 case AMDGPU::SI_SPILL_S160_SAVE: 533 case AMDGPU::SI_SPILL_S160_RESTORE: 534 case AMDGPU::SI_SPILL_V160_SAVE: 535 case AMDGPU::SI_SPILL_V160_RESTORE: 536 return 5; 537 case AMDGPU::SI_SPILL_S128_SAVE: 538 case AMDGPU::SI_SPILL_S128_RESTORE: 539 case AMDGPU::SI_SPILL_V128_SAVE: 540 case AMDGPU::SI_SPILL_V128_RESTORE: 541 case AMDGPU::SI_SPILL_A128_SAVE: 542 case AMDGPU::SI_SPILL_A128_RESTORE: 543 return 4; 544 case AMDGPU::SI_SPILL_S96_SAVE: 545 case AMDGPU::SI_SPILL_S96_RESTORE: 546 case AMDGPU::SI_SPILL_V96_SAVE: 547 case AMDGPU::SI_SPILL_V96_RESTORE: 548 return 3; 549 case AMDGPU::SI_SPILL_S64_SAVE: 550 case AMDGPU::SI_SPILL_S64_RESTORE: 551 case AMDGPU::SI_SPILL_V64_SAVE: 552 case AMDGPU::SI_SPILL_V64_RESTORE: 553 case AMDGPU::SI_SPILL_A64_SAVE: 554 case AMDGPU::SI_SPILL_A64_RESTORE: 555 return 2; 556 case AMDGPU::SI_SPILL_S32_SAVE: 557 case AMDGPU::SI_SPILL_S32_RESTORE: 558 case AMDGPU::SI_SPILL_V32_SAVE: 559 case AMDGPU::SI_SPILL_V32_RESTORE: 560 case AMDGPU::SI_SPILL_A32_SAVE: 561 case AMDGPU::SI_SPILL_A32_RESTORE: 562 return 1; 563 default: llvm_unreachable("Invalid spill opcode"); 564 } 565 } 566 567 static int getOffsetMUBUFStore(unsigned Opc) { 568 switch (Opc) { 569 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 570 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; 571 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: 572 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; 573 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: 574 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; 575 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: 576 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; 577 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: 578 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; 579 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: 580 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; 581 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: 582 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; 583 default: 584 return -1; 585 } 586 } 587 588 static int getOffsetMUBUFLoad(unsigned Opc) { 589 switch (Opc) { 590 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 591 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; 592 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: 593 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; 594 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: 595 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; 596 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: 597 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; 598 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: 599 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; 600 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: 601 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; 602 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: 603 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; 604 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: 605 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; 606 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: 607 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; 608 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: 609 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; 610 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: 611 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; 612 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: 613 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; 614 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: 615 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; 616 default: 617 return -1; 618 } 619 } 620 621 static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST, 622 MachineBasicBlock::iterator MI, 623 int Index, 624 unsigned Lane, 625 unsigned ValueReg, 626 bool IsKill) { 627 MachineBasicBlock *MBB = MI->getParent(); 628 MachineFunction *MF = MI->getParent()->getParent(); 629 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 630 const SIInstrInfo *TII = ST.getInstrInfo(); 631 632 MCPhysReg Reg = MFI->getVGPRToAGPRSpill(Index, Lane); 633 634 if (Reg == AMDGPU::NoRegister) 635 return MachineInstrBuilder(); 636 637 bool IsStore = MI->mayStore(); 638 MachineRegisterInfo &MRI = MF->getRegInfo(); 639 auto *TRI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo()); 640 641 unsigned Dst = IsStore ? Reg : ValueReg; 642 unsigned Src = IsStore ? ValueReg : Reg; 643 unsigned Opc = (IsStore ^ TRI->isVGPR(MRI, Reg)) ? AMDGPU::V_ACCVGPR_WRITE_B32 644 : AMDGPU::V_ACCVGPR_READ_B32; 645 646 return BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(Opc), Dst) 647 .addReg(Src, getKillRegState(IsKill)); 648 } 649 650 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not 651 // need to handle the case where an SGPR may need to be spilled while spilling. 652 static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST, 653 MachineFrameInfo &MFI, 654 MachineBasicBlock::iterator MI, 655 int Index, 656 int64_t Offset) { 657 const SIInstrInfo *TII = ST.getInstrInfo(); 658 MachineBasicBlock *MBB = MI->getParent(); 659 const DebugLoc &DL = MI->getDebugLoc(); 660 bool IsStore = MI->mayStore(); 661 662 unsigned Opc = MI->getOpcode(); 663 int LoadStoreOp = IsStore ? 664 getOffsetMUBUFStore(Opc) : getOffsetMUBUFLoad(Opc); 665 if (LoadStoreOp == -1) 666 return false; 667 668 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); 669 if (spillVGPRtoAGPR(ST, MI, Index, 0, Reg->getReg(), false).getInstr()) 670 return true; 671 672 MachineInstrBuilder NewMI = 673 BuildMI(*MBB, MI, DL, TII->get(LoadStoreOp)) 674 .add(*Reg) 675 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) 676 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) 677 .addImm(Offset) 678 .addImm(0) // glc 679 .addImm(0) // slc 680 .addImm(0) // tfe 681 .addImm(0) // dlc 682 .addImm(0) // swz 683 .cloneMemRefs(*MI); 684 685 const MachineOperand *VDataIn = TII->getNamedOperand(*MI, 686 AMDGPU::OpName::vdata_in); 687 if (VDataIn) 688 NewMI.add(*VDataIn); 689 return true; 690 } 691 692 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI, 693 unsigned LoadStoreOp, 694 int Index, 695 Register ValueReg, 696 bool IsKill, 697 MCRegister ScratchRsrcReg, 698 MCRegister ScratchOffsetReg, 699 int64_t InstOffset, 700 MachineMemOperand *MMO, 701 RegScavenger *RS) const { 702 MachineBasicBlock *MBB = MI->getParent(); 703 MachineFunction *MF = MI->getParent()->getParent(); 704 const SIInstrInfo *TII = ST.getInstrInfo(); 705 const MachineFrameInfo &MFI = MF->getFrameInfo(); 706 const SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>(); 707 708 const MCInstrDesc &Desc = TII->get(LoadStoreOp); 709 const DebugLoc &DL = MI->getDebugLoc(); 710 bool IsStore = Desc.mayStore(); 711 712 bool Scavenged = false; 713 MCRegister SOffset = ScratchOffsetReg; 714 715 const unsigned EltSize = 4; 716 const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg); 717 unsigned NumSubRegs = AMDGPU::getRegBitWidth(RC->getID()) / (EltSize * CHAR_BIT); 718 unsigned Size = NumSubRegs * EltSize; 719 int64_t Offset = InstOffset + MFI.getObjectOffset(Index); 720 int64_t ScratchOffsetRegDelta = 0; 721 722 Align Alignment = MFI.getObjectAlign(Index); 723 const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo(); 724 725 Register TmpReg = 726 hasAGPRs(RC) ? TII->getNamedOperand(*MI, AMDGPU::OpName::tmp)->getReg() 727 : Register(); 728 729 assert((Offset % EltSize) == 0 && "unexpected VGPR spill offset"); 730 731 if (!isUInt<12>(Offset + Size - EltSize)) { 732 SOffset = MCRegister(); 733 734 // We currently only support spilling VGPRs to EltSize boundaries, meaning 735 // we can simplify the adjustment of Offset here to just scale with 736 // WavefrontSize. 737 Offset *= ST.getWavefrontSize(); 738 739 // We don't have access to the register scavenger if this function is called 740 // during PEI::scavengeFrameVirtualRegs(). 741 if (RS) 742 SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false); 743 744 if (!SOffset) { 745 // There are no free SGPRs, and since we are in the process of spilling 746 // VGPRs too. Since we need a VGPR in order to spill SGPRs (this is true 747 // on SI/CI and on VI it is true until we implement spilling using scalar 748 // stores), we have no way to free up an SGPR. Our solution here is to 749 // add the offset directly to the ScratchOffset or StackPtrOffset 750 // register, and then subtract the offset after the spill to return the 751 // register to it's original value. 752 if (!ScratchOffsetReg) 753 ScratchOffsetReg = FuncInfo->getStackPtrOffsetReg(); 754 SOffset = ScratchOffsetReg; 755 ScratchOffsetRegDelta = Offset; 756 } else { 757 Scavenged = true; 758 } 759 760 if (!SOffset) 761 report_fatal_error("could not scavenge SGPR to spill in entry function"); 762 763 if (ScratchOffsetReg == AMDGPU::NoRegister) { 764 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset) 765 .addImm(Offset); 766 } else { 767 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), SOffset) 768 .addReg(ScratchOffsetReg) 769 .addImm(Offset); 770 } 771 772 Offset = 0; 773 } 774 775 for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += EltSize) { 776 Register SubReg = NumSubRegs == 1 777 ? Register(ValueReg) 778 : getSubReg(ValueReg, getSubRegFromChannel(i)); 779 780 unsigned SOffsetRegState = 0; 781 unsigned SrcDstRegState = getDefRegState(!IsStore); 782 if (i + 1 == e) { 783 SOffsetRegState |= getKillRegState(Scavenged); 784 // The last implicit use carries the "Kill" flag. 785 SrcDstRegState |= getKillRegState(IsKill); 786 } 787 788 auto MIB = spillVGPRtoAGPR(ST, MI, Index, i, SubReg, IsKill); 789 790 if (!MIB.getInstr()) { 791 unsigned FinalReg = SubReg; 792 if (TmpReg != AMDGPU::NoRegister) { 793 if (IsStore) 794 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_READ_B32), TmpReg) 795 .addReg(SubReg, getKillRegState(IsKill)); 796 SubReg = TmpReg; 797 } 798 799 MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(EltSize * i); 800 MachineMemOperand *NewMMO = 801 MF->getMachineMemOperand(PInfo, MMO->getFlags(), EltSize, 802 commonAlignment(Alignment, EltSize * i)); 803 804 MIB = BuildMI(*MBB, MI, DL, Desc) 805 .addReg(SubReg, 806 getDefRegState(!IsStore) | getKillRegState(IsKill)) 807 .addReg(ScratchRsrcReg); 808 if (SOffset == AMDGPU::NoRegister) { 809 MIB.addImm(0); 810 } else { 811 MIB.addReg(SOffset, SOffsetRegState); 812 } 813 MIB.addImm(Offset) 814 .addImm(0) // glc 815 .addImm(0) // slc 816 .addImm(0) // tfe 817 .addImm(0) // dlc 818 .addImm(0) // swz 819 .addMemOperand(NewMMO); 820 821 if (!IsStore && TmpReg != AMDGPU::NoRegister) 822 MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32), 823 FinalReg) 824 .addReg(TmpReg, RegState::Kill); 825 } 826 827 if (NumSubRegs > 1) 828 MIB.addReg(ValueReg, RegState::Implicit | SrcDstRegState); 829 } 830 831 if (ScratchOffsetRegDelta != 0) { 832 // Subtract the offset we added to the ScratchOffset register. 833 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), SOffset) 834 .addReg(SOffset) 835 .addImm(ScratchOffsetRegDelta); 836 } 837 } 838 839 bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, 840 int Index, 841 RegScavenger *RS, 842 bool OnlyToVGPR) const { 843 MachineBasicBlock *MBB = MI->getParent(); 844 MachineFunction *MF = MBB->getParent(); 845 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 846 DenseSet<unsigned> SGPRSpillVGPRDefinedSet; 847 848 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 849 = MFI->getSGPRToVGPRSpills(Index); 850 bool SpillToVGPR = !VGPRSpills.empty(); 851 if (OnlyToVGPR && !SpillToVGPR) 852 return false; 853 854 const SIInstrInfo *TII = ST.getInstrInfo(); 855 856 Register SuperReg = MI->getOperand(0).getReg(); 857 bool IsKill = MI->getOperand(0).isKill(); 858 const DebugLoc &DL = MI->getDebugLoc(); 859 860 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 861 862 assert(SpillToVGPR || (SuperReg != MFI->getStackPtrOffsetReg() && 863 SuperReg != MFI->getFrameOffsetReg())); 864 865 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 866 867 unsigned EltSize = 4; 868 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 869 870 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 871 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 872 873 // Scavenged temporary VGPR to use. It must be scavenged once for any number 874 // of spilled subregs. 875 Register TmpVGPR; 876 877 // SubReg carries the "Kill" flag when SubReg == SuperReg. 878 unsigned SubKillState = getKillRegState((NumSubRegs == 1) && IsKill); 879 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 880 Register SubReg = 881 NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]); 882 883 if (SpillToVGPR) { 884 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 885 886 // During SGPR spilling to VGPR, determine if the VGPR is defined. The 887 // only circumstance in which we say it is undefined is when it is the 888 // first spill to this VGPR in the first basic block. 889 bool VGPRDefined = true; 890 if (MBB == &MF->front()) 891 VGPRDefined = !SGPRSpillVGPRDefinedSet.insert(Spill.VGPR).second; 892 893 // Mark the "old value of vgpr" input undef only if this is the first sgpr 894 // spill to this specific vgpr in the first basic block. 895 BuildMI(*MBB, MI, DL, 896 TII->getMCOpcodeFromPseudo(AMDGPU::V_WRITELANE_B32), 897 Spill.VGPR) 898 .addReg(SubReg, getKillRegState(IsKill)) 899 .addImm(Spill.Lane) 900 .addReg(Spill.VGPR, VGPRDefined ? 0 : RegState::Undef); 901 902 // FIXME: Since this spills to another register instead of an actual 903 // frame index, we should delete the frame index when all references to 904 // it are fixed. 905 } else { 906 // XXX - Can to VGPR spill fail for some subregisters but not others? 907 if (OnlyToVGPR) 908 return false; 909 910 // Spill SGPR to a frame index. 911 if (!TmpVGPR.isValid()) 912 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 913 914 MachineInstrBuilder Mov 915 = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) 916 .addReg(SubReg, SubKillState); 917 918 // There could be undef components of a spilled super register. 919 // TODO: Can we detect this and skip the spill? 920 if (NumSubRegs > 1) { 921 // The last implicit use of the SuperReg carries the "Kill" flag. 922 unsigned SuperKillState = 0; 923 if (i + 1 == e) 924 SuperKillState |= getKillRegState(IsKill); 925 Mov.addReg(SuperReg, RegState::Implicit | SuperKillState); 926 } 927 928 Align Alignment = FrameInfo.getObjectAlign(Index); 929 MachinePointerInfo PtrInfo 930 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 931 MachineMemOperand *MMO = 932 MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, EltSize, 933 commonAlignment(Alignment, EltSize * i)); 934 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_SAVE)) 935 .addReg(TmpVGPR, RegState::Kill) // src 936 .addFrameIndex(Index) // vaddr 937 .addReg(MFI->getScratchRSrcReg()) // srrsrc 938 .addReg(MFI->getStackPtrOffsetReg()) // soffset 939 .addImm(i * 4) // offset 940 .addMemOperand(MMO); 941 } 942 } 943 944 MI->eraseFromParent(); 945 MFI->addToSpilledSGPRs(NumSubRegs); 946 return true; 947 } 948 949 bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, 950 int Index, 951 RegScavenger *RS, 952 bool OnlyToVGPR) const { 953 MachineFunction *MF = MI->getParent()->getParent(); 954 MachineBasicBlock *MBB = MI->getParent(); 955 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 956 957 ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills 958 = MFI->getSGPRToVGPRSpills(Index); 959 bool SpillToVGPR = !VGPRSpills.empty(); 960 if (OnlyToVGPR && !SpillToVGPR) 961 return false; 962 963 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 964 const SIInstrInfo *TII = ST.getInstrInfo(); 965 const DebugLoc &DL = MI->getDebugLoc(); 966 967 Register SuperReg = MI->getOperand(0).getReg(); 968 969 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); 970 971 unsigned EltSize = 4; 972 973 const TargetRegisterClass *RC = getPhysRegClass(SuperReg); 974 975 ArrayRef<int16_t> SplitParts = getRegSplitParts(RC, EltSize); 976 unsigned NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); 977 978 Register TmpVGPR; 979 980 for (unsigned i = 0, e = NumSubRegs; i < e; ++i) { 981 Register SubReg = 982 NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]); 983 984 if (SpillToVGPR) { 985 SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i]; 986 auto MIB = 987 BuildMI(*MBB, MI, DL, TII->getMCOpcodeFromPseudo(AMDGPU::V_READLANE_B32), 988 SubReg) 989 .addReg(Spill.VGPR) 990 .addImm(Spill.Lane); 991 992 if (NumSubRegs > 1 && i == 0) 993 MIB.addReg(SuperReg, RegState::ImplicitDefine); 994 } else { 995 if (OnlyToVGPR) 996 return false; 997 998 // Restore SGPR from a stack slot. 999 // FIXME: We should use S_LOAD_DWORD here for VI. 1000 if (!TmpVGPR.isValid()) 1001 TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1002 Align Alignment = FrameInfo.getObjectAlign(Index); 1003 1004 MachinePointerInfo PtrInfo 1005 = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i); 1006 1007 MachineMemOperand *MMO = 1008 MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, EltSize, 1009 commonAlignment(Alignment, EltSize * i)); 1010 1011 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_RESTORE), TmpVGPR) 1012 .addFrameIndex(Index) // vaddr 1013 .addReg(MFI->getScratchRSrcReg()) // srsrc 1014 .addReg(MFI->getStackPtrOffsetReg()) // soffset 1015 .addImm(i * 4) // offset 1016 .addMemOperand(MMO); 1017 1018 auto MIB = 1019 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg) 1020 .addReg(TmpVGPR, RegState::Kill); 1021 1022 if (NumSubRegs > 1) 1023 MIB.addReg(MI->getOperand(0).getReg(), RegState::ImplicitDefine); 1024 } 1025 } 1026 1027 MI->eraseFromParent(); 1028 return true; 1029 } 1030 1031 /// Special case of eliminateFrameIndex. Returns true if the SGPR was spilled to 1032 /// a VGPR and the stack slot can be safely eliminated when all other users are 1033 /// handled. 1034 bool SIRegisterInfo::eliminateSGPRToVGPRSpillFrameIndex( 1035 MachineBasicBlock::iterator MI, 1036 int FI, 1037 RegScavenger *RS) const { 1038 switch (MI->getOpcode()) { 1039 case AMDGPU::SI_SPILL_S1024_SAVE: 1040 case AMDGPU::SI_SPILL_S512_SAVE: 1041 case AMDGPU::SI_SPILL_S256_SAVE: 1042 case AMDGPU::SI_SPILL_S160_SAVE: 1043 case AMDGPU::SI_SPILL_S128_SAVE: 1044 case AMDGPU::SI_SPILL_S96_SAVE: 1045 case AMDGPU::SI_SPILL_S64_SAVE: 1046 case AMDGPU::SI_SPILL_S32_SAVE: 1047 return spillSGPR(MI, FI, RS, true); 1048 case AMDGPU::SI_SPILL_S1024_RESTORE: 1049 case AMDGPU::SI_SPILL_S512_RESTORE: 1050 case AMDGPU::SI_SPILL_S256_RESTORE: 1051 case AMDGPU::SI_SPILL_S160_RESTORE: 1052 case AMDGPU::SI_SPILL_S128_RESTORE: 1053 case AMDGPU::SI_SPILL_S96_RESTORE: 1054 case AMDGPU::SI_SPILL_S64_RESTORE: 1055 case AMDGPU::SI_SPILL_S32_RESTORE: 1056 return restoreSGPR(MI, FI, RS, true); 1057 default: 1058 llvm_unreachable("not an SGPR spill instruction"); 1059 } 1060 } 1061 1062 void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, 1063 int SPAdj, unsigned FIOperandNum, 1064 RegScavenger *RS) const { 1065 MachineFunction *MF = MI->getParent()->getParent(); 1066 MachineBasicBlock *MBB = MI->getParent(); 1067 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1068 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1069 const SIInstrInfo *TII = ST.getInstrInfo(); 1070 DebugLoc DL = MI->getDebugLoc(); 1071 1072 assert(SPAdj == 0 && "unhandled SP adjustment in call sequence?"); 1073 1074 MachineOperand &FIOp = MI->getOperand(FIOperandNum); 1075 int Index = MI->getOperand(FIOperandNum).getIndex(); 1076 1077 Register FrameReg = FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(*MF) 1078 ? getBaseRegister() 1079 : getFrameRegister(*MF); 1080 1081 switch (MI->getOpcode()) { 1082 // SGPR register spill 1083 case AMDGPU::SI_SPILL_S1024_SAVE: 1084 case AMDGPU::SI_SPILL_S512_SAVE: 1085 case AMDGPU::SI_SPILL_S256_SAVE: 1086 case AMDGPU::SI_SPILL_S160_SAVE: 1087 case AMDGPU::SI_SPILL_S128_SAVE: 1088 case AMDGPU::SI_SPILL_S96_SAVE: 1089 case AMDGPU::SI_SPILL_S64_SAVE: 1090 case AMDGPU::SI_SPILL_S32_SAVE: { 1091 spillSGPR(MI, Index, RS); 1092 break; 1093 } 1094 1095 // SGPR register restore 1096 case AMDGPU::SI_SPILL_S1024_RESTORE: 1097 case AMDGPU::SI_SPILL_S512_RESTORE: 1098 case AMDGPU::SI_SPILL_S256_RESTORE: 1099 case AMDGPU::SI_SPILL_S160_RESTORE: 1100 case AMDGPU::SI_SPILL_S128_RESTORE: 1101 case AMDGPU::SI_SPILL_S96_RESTORE: 1102 case AMDGPU::SI_SPILL_S64_RESTORE: 1103 case AMDGPU::SI_SPILL_S32_RESTORE: { 1104 restoreSGPR(MI, Index, RS); 1105 break; 1106 } 1107 1108 // VGPR register spill 1109 case AMDGPU::SI_SPILL_V1024_SAVE: 1110 case AMDGPU::SI_SPILL_V512_SAVE: 1111 case AMDGPU::SI_SPILL_V256_SAVE: 1112 case AMDGPU::SI_SPILL_V160_SAVE: 1113 case AMDGPU::SI_SPILL_V128_SAVE: 1114 case AMDGPU::SI_SPILL_V96_SAVE: 1115 case AMDGPU::SI_SPILL_V64_SAVE: 1116 case AMDGPU::SI_SPILL_V32_SAVE: 1117 case AMDGPU::SI_SPILL_A1024_SAVE: 1118 case AMDGPU::SI_SPILL_A512_SAVE: 1119 case AMDGPU::SI_SPILL_A128_SAVE: 1120 case AMDGPU::SI_SPILL_A64_SAVE: 1121 case AMDGPU::SI_SPILL_A32_SAVE: { 1122 const MachineOperand *VData = TII->getNamedOperand(*MI, 1123 AMDGPU::OpName::vdata); 1124 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 1125 MFI->getStackPtrOffsetReg()); 1126 1127 buildSpillLoadStore(MI, AMDGPU::BUFFER_STORE_DWORD_OFFSET, 1128 Index, 1129 VData->getReg(), VData->isKill(), 1130 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1131 FrameReg, 1132 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1133 *MI->memoperands_begin(), 1134 RS); 1135 MFI->addToSpilledVGPRs(getNumSubRegsForSpillOp(MI->getOpcode())); 1136 MI->eraseFromParent(); 1137 break; 1138 } 1139 case AMDGPU::SI_SPILL_V32_RESTORE: 1140 case AMDGPU::SI_SPILL_V64_RESTORE: 1141 case AMDGPU::SI_SPILL_V96_RESTORE: 1142 case AMDGPU::SI_SPILL_V128_RESTORE: 1143 case AMDGPU::SI_SPILL_V160_RESTORE: 1144 case AMDGPU::SI_SPILL_V256_RESTORE: 1145 case AMDGPU::SI_SPILL_V512_RESTORE: 1146 case AMDGPU::SI_SPILL_V1024_RESTORE: 1147 case AMDGPU::SI_SPILL_A32_RESTORE: 1148 case AMDGPU::SI_SPILL_A64_RESTORE: 1149 case AMDGPU::SI_SPILL_A128_RESTORE: 1150 case AMDGPU::SI_SPILL_A512_RESTORE: 1151 case AMDGPU::SI_SPILL_A1024_RESTORE: { 1152 const MachineOperand *VData = TII->getNamedOperand(*MI, 1153 AMDGPU::OpName::vdata); 1154 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == 1155 MFI->getStackPtrOffsetReg()); 1156 1157 buildSpillLoadStore(MI, AMDGPU::BUFFER_LOAD_DWORD_OFFSET, 1158 Index, 1159 VData->getReg(), VData->isKill(), 1160 TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)->getReg(), 1161 FrameReg, 1162 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), 1163 *MI->memoperands_begin(), 1164 RS); 1165 MI->eraseFromParent(); 1166 break; 1167 } 1168 1169 default: { 1170 const DebugLoc &DL = MI->getDebugLoc(); 1171 bool IsMUBUF = TII->isMUBUF(*MI); 1172 1173 if (!IsMUBUF && !MFI->isEntryFunction()) { 1174 // Convert to a swizzled stack address by scaling by the wave size. 1175 // 1176 // In an entry function/kernel the offset is already swizzled. 1177 1178 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32; 1179 Register ResultReg = 1180 IsCopy ? MI->getOperand(0).getReg() 1181 : RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1182 1183 int64_t Offset = FrameInfo.getObjectOffset(Index); 1184 if (Offset == 0) { 1185 // XXX - This never happens because of emergency scavenging slot at 0? 1186 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ResultReg) 1187 .addImm(ST.getWavefrontSizeLog2()) 1188 .addReg(FrameReg); 1189 } else { 1190 if (auto MIB = TII->getAddNoCarry(*MBB, MI, DL, ResultReg, *RS)) { 1191 // Reuse ResultReg in intermediate step. 1192 Register ScaledReg = ResultReg; 1193 1194 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), 1195 ScaledReg) 1196 .addImm(ST.getWavefrontSizeLog2()) 1197 .addReg(FrameReg); 1198 1199 const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32; 1200 1201 // TODO: Fold if use instruction is another add of a constant. 1202 if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { 1203 // FIXME: This can fail 1204 MIB.addImm(Offset); 1205 MIB.addReg(ScaledReg, RegState::Kill); 1206 if (!IsVOP2) 1207 MIB.addImm(0); // clamp bit 1208 } else { 1209 assert(MIB->getOpcode() == AMDGPU::V_ADD_I32_e64 && 1210 "Need to reuse carry out register"); 1211 1212 // Use scavenged unused carry out as offset register. 1213 Register ConstOffsetReg; 1214 if (!isWave32) 1215 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); 1216 else 1217 ConstOffsetReg = MIB.getReg(1); 1218 1219 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) 1220 .addImm(Offset); 1221 MIB.addReg(ConstOffsetReg, RegState::Kill); 1222 MIB.addReg(ScaledReg, RegState::Kill); 1223 MIB.addImm(0); // clamp bit 1224 } 1225 } else { 1226 // We have to produce a carry out, and there isn't a free SGPR pair 1227 // for it. We can keep the whole computation on the SALU to avoid 1228 // clobbering an additional register at the cost of an extra mov. 1229 1230 // We may have 1 free scratch SGPR even though a carry out is 1231 // unavailable. Only one additional mov is needed. 1232 Register TmpScaledReg = 1233 RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MI, 0, false); 1234 Register ScaledReg = TmpScaledReg.isValid() ? TmpScaledReg : FrameReg; 1235 1236 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg) 1237 .addReg(FrameReg) 1238 .addImm(ST.getWavefrontSizeLog2()); 1239 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_U32), ScaledReg) 1240 .addReg(ScaledReg, RegState::Kill) 1241 .addImm(Offset); 1242 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg) 1243 .addReg(ScaledReg, RegState::Kill); 1244 1245 // If there were truly no free SGPRs, we need to undo everything. 1246 if (!TmpScaledReg.isValid()) { 1247 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScaledReg) 1248 .addReg(ScaledReg, RegState::Kill) 1249 .addImm(Offset); 1250 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) 1251 .addReg(FrameReg) 1252 .addImm(ST.getWavefrontSizeLog2()); 1253 } 1254 } 1255 } 1256 1257 // Don't introduce an extra copy if we're just materializing in a mov. 1258 if (IsCopy) 1259 MI->eraseFromParent(); 1260 else 1261 FIOp.ChangeToRegister(ResultReg, false, false, true); 1262 return; 1263 } 1264 1265 if (IsMUBUF) { 1266 // Disable offen so we don't need a 0 vgpr base. 1267 assert(static_cast<int>(FIOperandNum) == 1268 AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1269 AMDGPU::OpName::vaddr)); 1270 1271 auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset); 1272 assert((SOffset.isReg() && 1273 SOffset.getReg() == MFI->getStackPtrOffsetReg()) || 1274 (SOffset.isImm() && SOffset.getImm() == 0)); 1275 if (SOffset.isReg()) { 1276 if (FrameReg == AMDGPU::NoRegister) { 1277 SOffset.ChangeToImmediate(0); 1278 } else { 1279 SOffset.setReg(FrameReg); 1280 } 1281 } 1282 1283 int64_t Offset = FrameInfo.getObjectOffset(Index); 1284 int64_t OldImm 1285 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); 1286 int64_t NewOffset = OldImm + Offset; 1287 1288 if (isUInt<12>(NewOffset) && 1289 buildMUBUFOffsetLoadStore(ST, FrameInfo, MI, Index, NewOffset)) { 1290 MI->eraseFromParent(); 1291 return; 1292 } 1293 } 1294 1295 // If the offset is simply too big, don't convert to a scratch wave offset 1296 // relative index. 1297 1298 int64_t Offset = FrameInfo.getObjectOffset(Index); 1299 FIOp.ChangeToImmediate(Offset); 1300 if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) { 1301 Register TmpReg = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0); 1302 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) 1303 .addImm(Offset); 1304 FIOp.ChangeToRegister(TmpReg, false, false, true); 1305 } 1306 } 1307 } 1308 } 1309 1310 StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const { 1311 return AMDGPUInstPrinter::getRegisterName(Reg); 1312 } 1313 1314 const TargetRegisterClass * 1315 SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) { 1316 switch (BitWidth) { 1317 case 1: 1318 return &AMDGPU::VReg_1RegClass; 1319 case 16: 1320 return &AMDGPU::VGPR_LO16RegClass; 1321 case 32: 1322 return &AMDGPU::VGPR_32RegClass; 1323 case 64: 1324 return &AMDGPU::VReg_64RegClass; 1325 case 96: 1326 return &AMDGPU::VReg_96RegClass; 1327 case 128: 1328 return &AMDGPU::VReg_128RegClass; 1329 case 160: 1330 return &AMDGPU::VReg_160RegClass; 1331 case 192: 1332 return &AMDGPU::VReg_192RegClass; 1333 case 256: 1334 return &AMDGPU::VReg_256RegClass; 1335 case 512: 1336 return &AMDGPU::VReg_512RegClass; 1337 case 1024: 1338 return &AMDGPU::VReg_1024RegClass; 1339 default: 1340 return nullptr; 1341 } 1342 } 1343 1344 const TargetRegisterClass * 1345 SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) { 1346 switch (BitWidth) { 1347 case 16: 1348 return &AMDGPU::AGPR_LO16RegClass; 1349 case 32: 1350 return &AMDGPU::AGPR_32RegClass; 1351 case 64: 1352 return &AMDGPU::AReg_64RegClass; 1353 case 96: 1354 return &AMDGPU::AReg_96RegClass; 1355 case 128: 1356 return &AMDGPU::AReg_128RegClass; 1357 case 160: 1358 return &AMDGPU::AReg_160RegClass; 1359 case 192: 1360 return &AMDGPU::AReg_192RegClass; 1361 case 256: 1362 return &AMDGPU::AReg_256RegClass; 1363 case 512: 1364 return &AMDGPU::AReg_512RegClass; 1365 case 1024: 1366 return &AMDGPU::AReg_1024RegClass; 1367 default: 1368 return nullptr; 1369 } 1370 } 1371 1372 const TargetRegisterClass * 1373 SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) { 1374 switch (BitWidth) { 1375 case 16: 1376 return &AMDGPU::SGPR_LO16RegClass; 1377 case 32: 1378 return &AMDGPU::SReg_32RegClass; 1379 case 64: 1380 return &AMDGPU::SReg_64RegClass; 1381 case 96: 1382 return &AMDGPU::SGPR_96RegClass; 1383 case 128: 1384 return &AMDGPU::SGPR_128RegClass; 1385 case 160: 1386 return &AMDGPU::SGPR_160RegClass; 1387 case 192: 1388 return &AMDGPU::SGPR_192RegClass; 1389 case 256: 1390 return &AMDGPU::SGPR_256RegClass; 1391 case 512: 1392 return &AMDGPU::SGPR_512RegClass; 1393 case 1024: 1394 return &AMDGPU::SGPR_1024RegClass; 1395 default: 1396 return nullptr; 1397 } 1398 } 1399 1400 // FIXME: This is very slow. It might be worth creating a map from physreg to 1401 // register class. 1402 const TargetRegisterClass * 1403 SIRegisterInfo::getPhysRegClass(MCRegister Reg) const { 1404 static const TargetRegisterClass *const BaseClasses[] = { 1405 &AMDGPU::VGPR_LO16RegClass, 1406 &AMDGPU::VGPR_HI16RegClass, 1407 &AMDGPU::SReg_LO16RegClass, 1408 &AMDGPU::AGPR_LO16RegClass, 1409 &AMDGPU::VGPR_32RegClass, 1410 &AMDGPU::SReg_32RegClass, 1411 &AMDGPU::AGPR_32RegClass, 1412 &AMDGPU::VReg_64RegClass, 1413 &AMDGPU::SReg_64RegClass, 1414 &AMDGPU::AReg_64RegClass, 1415 &AMDGPU::VReg_96RegClass, 1416 &AMDGPU::SReg_96RegClass, 1417 &AMDGPU::AReg_96RegClass, 1418 &AMDGPU::VReg_128RegClass, 1419 &AMDGPU::SReg_128RegClass, 1420 &AMDGPU::AReg_128RegClass, 1421 &AMDGPU::VReg_160RegClass, 1422 &AMDGPU::SReg_160RegClass, 1423 &AMDGPU::AReg_160RegClass, 1424 &AMDGPU::VReg_192RegClass, 1425 &AMDGPU::SReg_192RegClass, 1426 &AMDGPU::AReg_192RegClass, 1427 &AMDGPU::VReg_256RegClass, 1428 &AMDGPU::SReg_256RegClass, 1429 &AMDGPU::AReg_256RegClass, 1430 &AMDGPU::VReg_512RegClass, 1431 &AMDGPU::SReg_512RegClass, 1432 &AMDGPU::AReg_512RegClass, 1433 &AMDGPU::SReg_1024RegClass, 1434 &AMDGPU::VReg_1024RegClass, 1435 &AMDGPU::AReg_1024RegClass, 1436 &AMDGPU::SCC_CLASSRegClass, 1437 &AMDGPU::Pseudo_SReg_32RegClass, 1438 &AMDGPU::Pseudo_SReg_128RegClass, 1439 }; 1440 1441 for (const TargetRegisterClass *BaseClass : BaseClasses) { 1442 if (BaseClass->contains(Reg)) { 1443 return BaseClass; 1444 } 1445 } 1446 return nullptr; 1447 } 1448 1449 // TODO: It might be helpful to have some target specific flags in 1450 // TargetRegisterClass to mark which classes are VGPRs to make this trivial. 1451 bool SIRegisterInfo::hasVGPRs(const TargetRegisterClass *RC) const { 1452 unsigned Size = getRegSizeInBits(*RC); 1453 if (Size == 16) { 1454 return getCommonSubClass(&AMDGPU::VGPR_LO16RegClass, RC) != nullptr || 1455 getCommonSubClass(&AMDGPU::VGPR_HI16RegClass, RC) != nullptr; 1456 } 1457 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 1458 if (!VRC) { 1459 assert(Size < 32 && "Invalid register class size"); 1460 return false; 1461 } 1462 return getCommonSubClass(VRC, RC) != nullptr; 1463 } 1464 1465 bool SIRegisterInfo::hasAGPRs(const TargetRegisterClass *RC) const { 1466 unsigned Size = getRegSizeInBits(*RC); 1467 if (Size < 16) 1468 return false; 1469 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 1470 if (!ARC) { 1471 assert(getVGPRClassForBitWidth(Size) && "Invalid register class size"); 1472 return false; 1473 } 1474 return getCommonSubClass(ARC, RC) != nullptr; 1475 } 1476 1477 const TargetRegisterClass * 1478 SIRegisterInfo::getEquivalentVGPRClass(const TargetRegisterClass *SRC) const { 1479 unsigned Size = getRegSizeInBits(*SRC); 1480 const TargetRegisterClass *VRC = getVGPRClassForBitWidth(Size); 1481 assert(VRC && "Invalid register class size"); 1482 return VRC; 1483 } 1484 1485 const TargetRegisterClass * 1486 SIRegisterInfo::getEquivalentAGPRClass(const TargetRegisterClass *SRC) const { 1487 unsigned Size = getRegSizeInBits(*SRC); 1488 const TargetRegisterClass *ARC = getAGPRClassForBitWidth(Size); 1489 assert(ARC && "Invalid register class size"); 1490 return ARC; 1491 } 1492 1493 const TargetRegisterClass * 1494 SIRegisterInfo::getEquivalentSGPRClass(const TargetRegisterClass *VRC) const { 1495 unsigned Size = getRegSizeInBits(*VRC); 1496 if (Size == 32) 1497 return &AMDGPU::SGPR_32RegClass; 1498 const TargetRegisterClass *SRC = getSGPRClassForBitWidth(Size); 1499 assert(SRC && "Invalid register class size"); 1500 return SRC; 1501 } 1502 1503 const TargetRegisterClass *SIRegisterInfo::getSubRegClass( 1504 const TargetRegisterClass *RC, unsigned SubIdx) const { 1505 if (SubIdx == AMDGPU::NoSubRegister) 1506 return RC; 1507 1508 // We can assume that each lane corresponds to one 32-bit register. 1509 unsigned Size = getNumChannelsFromSubReg(SubIdx) * 32; 1510 if (isSGPRClass(RC)) { 1511 if (Size == 32) 1512 RC = &AMDGPU::SGPR_32RegClass; 1513 else 1514 RC = getSGPRClassForBitWidth(Size); 1515 } else if (hasAGPRs(RC)) { 1516 RC = getAGPRClassForBitWidth(Size); 1517 } else { 1518 RC = getVGPRClassForBitWidth(Size); 1519 } 1520 assert(RC && "Invalid sub-register class size"); 1521 return RC; 1522 } 1523 1524 bool SIRegisterInfo::opCanUseInlineConstant(unsigned OpType) const { 1525 if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST && 1526 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST) 1527 return !ST.hasMFMAInlineLiteralBug(); 1528 1529 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1530 OpType <= AMDGPU::OPERAND_SRC_LAST; 1531 } 1532 1533 bool SIRegisterInfo::shouldRewriteCopySrc( 1534 const TargetRegisterClass *DefRC, 1535 unsigned DefSubReg, 1536 const TargetRegisterClass *SrcRC, 1537 unsigned SrcSubReg) const { 1538 // We want to prefer the smallest register class possible, so we don't want to 1539 // stop and rewrite on anything that looks like a subregister 1540 // extract. Operations mostly don't care about the super register class, so we 1541 // only want to stop on the most basic of copies between the same register 1542 // class. 1543 // 1544 // e.g. if we have something like 1545 // %0 = ... 1546 // %1 = ... 1547 // %2 = REG_SEQUENCE %0, sub0, %1, sub1, %2, sub2 1548 // %3 = COPY %2, sub0 1549 // 1550 // We want to look through the COPY to find: 1551 // => %3 = COPY %0 1552 1553 // Plain copy. 1554 return getCommonSubClass(DefRC, SrcRC) != nullptr; 1555 } 1556 1557 /// Returns a lowest register that is not used at any point in the function. 1558 /// If all registers are used, then this function will return 1559 /// AMDGPU::NoRegister. If \p ReserveHighestVGPR = true, then return 1560 /// highest unused register. 1561 MCRegister SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI, 1562 const TargetRegisterClass *RC, 1563 const MachineFunction &MF, 1564 bool ReserveHighestVGPR) const { 1565 if (ReserveHighestVGPR) { 1566 for (MCRegister Reg : reverse(*RC)) 1567 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 1568 return Reg; 1569 } else { 1570 for (MCRegister Reg : *RC) 1571 if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) 1572 return Reg; 1573 } 1574 return MCRegister(); 1575 } 1576 1577 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC, 1578 unsigned EltSize) const { 1579 const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC->MC); 1580 assert(RegBitWidth >= 32 && RegBitWidth <= 1024); 1581 1582 const unsigned EltBitWidth = EltSize * 8; 1583 assert(EltBitWidth >= 32 && EltBitWidth < 1024 && isPowerOf2_32(EltBitWidth)); 1584 const unsigned LogEltBitWidth = Log2_32(EltBitWidth); 1585 1586 assert(RegBitWidth % EltBitWidth == 0); 1587 1588 if (RegBitWidth == EltBitWidth) 1589 return {}; 1590 1591 static const int16_t Sub_32[] = { 1592 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 1593 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 1594 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 1595 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 1596 AMDGPU::sub16, AMDGPU::sub17, AMDGPU::sub18, AMDGPU::sub19, 1597 AMDGPU::sub20, AMDGPU::sub21, AMDGPU::sub22, AMDGPU::sub23, 1598 AMDGPU::sub24, AMDGPU::sub25, AMDGPU::sub26, AMDGPU::sub27, 1599 AMDGPU::sub28, AMDGPU::sub29, AMDGPU::sub30, AMDGPU::sub31 1600 }; 1601 1602 static const int16_t Sub_64[] = { 1603 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 1604 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 1605 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 1606 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 1607 AMDGPU::sub16_sub17, AMDGPU::sub18_sub19, 1608 AMDGPU::sub20_sub21, AMDGPU::sub22_sub23, 1609 AMDGPU::sub24_sub25, AMDGPU::sub26_sub27, 1610 AMDGPU::sub28_sub29, AMDGPU::sub30_sub31 1611 }; 1612 1613 static const int16_t Sub_128[] = { 1614 AMDGPU::sub0_sub1_sub2_sub3, 1615 AMDGPU::sub4_sub5_sub6_sub7, 1616 AMDGPU::sub8_sub9_sub10_sub11, 1617 AMDGPU::sub12_sub13_sub14_sub15, 1618 AMDGPU::sub16_sub17_sub18_sub19, 1619 AMDGPU::sub20_sub21_sub22_sub23, 1620 AMDGPU::sub24_sub25_sub26_sub27, 1621 AMDGPU::sub28_sub29_sub30_sub31 1622 }; 1623 1624 static const int16_t Sub_256[] = { 1625 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7, 1626 AMDGPU::sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 1627 AMDGPU::sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23, 1628 AMDGPU::sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31 1629 }; 1630 1631 static const int16_t Sub_512[] = { 1632 AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 1633 AMDGPU::sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23_sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31 1634 }; 1635 1636 static const int16_t *const Subs[] = { 1637 Sub_32, Sub_64, Sub_128, Sub_256, Sub_512 1638 }; 1639 1640 return makeArrayRef(Subs[LogEltBitWidth - 5], RegBitWidth >> LogEltBitWidth); 1641 } 1642 1643 const TargetRegisterClass* 1644 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI, 1645 Register Reg) const { 1646 return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegClass(Reg); 1647 } 1648 1649 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI, 1650 Register Reg) const { 1651 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 1652 // Registers without classes are unaddressable, SGPR-like registers. 1653 return RC && hasVGPRs(RC); 1654 } 1655 1656 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI, 1657 Register Reg) const { 1658 const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg); 1659 1660 // Registers without classes are unaddressable, SGPR-like registers. 1661 return RC && hasAGPRs(RC); 1662 } 1663 1664 bool SIRegisterInfo::shouldCoalesce(MachineInstr *MI, 1665 const TargetRegisterClass *SrcRC, 1666 unsigned SubReg, 1667 const TargetRegisterClass *DstRC, 1668 unsigned DstSubReg, 1669 const TargetRegisterClass *NewRC, 1670 LiveIntervals &LIS) const { 1671 unsigned SrcSize = getRegSizeInBits(*SrcRC); 1672 unsigned DstSize = getRegSizeInBits(*DstRC); 1673 unsigned NewSize = getRegSizeInBits(*NewRC); 1674 1675 // Do not increase size of registers beyond dword, we would need to allocate 1676 // adjacent registers and constraint regalloc more than needed. 1677 1678 // Always allow dword coalescing. 1679 if (SrcSize <= 32 || DstSize <= 32) 1680 return true; 1681 1682 return NewSize <= DstSize || NewSize <= SrcSize; 1683 } 1684 1685 unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 1686 MachineFunction &MF) const { 1687 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1688 1689 unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), 1690 MF.getFunction()); 1691 switch (RC->getID()) { 1692 default: 1693 return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF); 1694 case AMDGPU::VGPR_32RegClassID: 1695 case AMDGPU::VGPR_LO16RegClassID: 1696 case AMDGPU::VGPR_HI16RegClassID: 1697 return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF)); 1698 case AMDGPU::SGPR_32RegClassID: 1699 case AMDGPU::SGPR_LO16RegClassID: 1700 return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF)); 1701 } 1702 } 1703 1704 unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF, 1705 unsigned Idx) const { 1706 if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 || 1707 Idx == AMDGPU::RegisterPressureSets::AGPR_32) 1708 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 1709 const_cast<MachineFunction &>(MF)); 1710 1711 if (Idx == AMDGPU::RegisterPressureSets::SReg_32) 1712 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, 1713 const_cast<MachineFunction &>(MF)); 1714 1715 llvm_unreachable("Unexpected register pressure set!"); 1716 } 1717 1718 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const { 1719 static const int Empty[] = { -1 }; 1720 1721 if (RegPressureIgnoredUnits[RegUnit]) 1722 return Empty; 1723 1724 return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit); 1725 } 1726 1727 MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { 1728 // Not a callee saved register. 1729 return AMDGPU::SGPR30_SGPR31; 1730 } 1731 1732 const TargetRegisterClass * 1733 SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, 1734 const RegisterBank &RB, 1735 const MachineRegisterInfo &MRI) const { 1736 switch (RB.getID()) { 1737 case AMDGPU::VGPRRegBankID: 1738 return getVGPRClassForBitWidth(std::max(32u, Size)); 1739 case AMDGPU::VCCRegBankID: 1740 assert(Size == 1); 1741 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 1742 : &AMDGPU::SReg_64_XEXECRegClass; 1743 case AMDGPU::SGPRRegBankID: 1744 return getSGPRClassForBitWidth(std::max(32u, Size)); 1745 case AMDGPU::AGPRRegBankID: 1746 return getAGPRClassForBitWidth(std::max(32u, Size)); 1747 default: 1748 llvm_unreachable("unknown register bank"); 1749 } 1750 } 1751 1752 const TargetRegisterClass * 1753 SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO, 1754 const MachineRegisterInfo &MRI) const { 1755 const RegClassOrRegBank &RCOrRB = MRI.getRegClassOrRegBank(MO.getReg()); 1756 if (const RegisterBank *RB = RCOrRB.dyn_cast<const RegisterBank*>()) 1757 return getRegClassForTypeOnBank(MRI.getType(MO.getReg()), *RB, MRI); 1758 1759 const TargetRegisterClass *RC = RCOrRB.get<const TargetRegisterClass*>(); 1760 return getAllocatableClass(RC); 1761 } 1762 1763 MCRegister SIRegisterInfo::getVCC() const { 1764 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; 1765 } 1766 1767 const TargetRegisterClass * 1768 SIRegisterInfo::getRegClass(unsigned RCID) const { 1769 switch ((int)RCID) { 1770 case AMDGPU::SReg_1RegClassID: 1771 return getBoolRC(); 1772 case AMDGPU::SReg_1_XEXECRegClassID: 1773 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass 1774 : &AMDGPU::SReg_64_XEXECRegClass; 1775 case -1: 1776 return nullptr; 1777 default: 1778 return AMDGPUGenRegisterInfo::getRegClass(RCID); 1779 } 1780 } 1781 1782 // Find reaching register definition 1783 MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg, 1784 MachineInstr &Use, 1785 MachineRegisterInfo &MRI, 1786 LiveIntervals *LIS) const { 1787 auto &MDT = LIS->getAnalysis<MachineDominatorTree>(); 1788 SlotIndex UseIdx = LIS->getInstructionIndex(Use); 1789 SlotIndex DefIdx; 1790 1791 if (Reg.isVirtual()) { 1792 if (!LIS->hasInterval(Reg)) 1793 return nullptr; 1794 LiveInterval &LI = LIS->getInterval(Reg); 1795 LaneBitmask SubLanes = SubReg ? getSubRegIndexLaneMask(SubReg) 1796 : MRI.getMaxLaneMaskForVReg(Reg); 1797 VNInfo *V = nullptr; 1798 if (LI.hasSubRanges()) { 1799 for (auto &S : LI.subranges()) { 1800 if ((S.LaneMask & SubLanes) == SubLanes) { 1801 V = S.getVNInfoAt(UseIdx); 1802 break; 1803 } 1804 } 1805 } else { 1806 V = LI.getVNInfoAt(UseIdx); 1807 } 1808 if (!V) 1809 return nullptr; 1810 DefIdx = V->def; 1811 } else { 1812 // Find last def. 1813 for (MCRegUnitIterator Units(Reg, this); Units.isValid(); ++Units) { 1814 LiveRange &LR = LIS->getRegUnit(*Units); 1815 if (VNInfo *V = LR.getVNInfoAt(UseIdx)) { 1816 if (!DefIdx.isValid() || 1817 MDT.dominates(LIS->getInstructionFromIndex(DefIdx), 1818 LIS->getInstructionFromIndex(V->def))) 1819 DefIdx = V->def; 1820 } else { 1821 return nullptr; 1822 } 1823 } 1824 } 1825 1826 MachineInstr *Def = LIS->getInstructionFromIndex(DefIdx); 1827 1828 if (!Def || !MDT.dominates(Def, &Use)) 1829 return nullptr; 1830 1831 assert(Def->modifiesRegister(Reg, this)); 1832 1833 return Def; 1834 } 1835 1836 MCPhysReg SIRegisterInfo::get32BitRegister(MCPhysReg Reg) const { 1837 assert(getRegSizeInBits(*getPhysRegClass(Reg)) <= 32); 1838 1839 for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass, 1840 AMDGPU::SReg_32RegClass, 1841 AMDGPU::AGPR_32RegClass } ) { 1842 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC)) 1843 return Super; 1844 } 1845 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16, 1846 &AMDGPU::VGPR_32RegClass)) { 1847 return Super; 1848 } 1849 1850 return AMDGPU::NoRegister; 1851 } 1852 1853 bool SIRegisterInfo::isConstantPhysReg(MCRegister PhysReg) const { 1854 switch (PhysReg) { 1855 case AMDGPU::SGPR_NULL: 1856 case AMDGPU::SRC_SHARED_BASE: 1857 case AMDGPU::SRC_PRIVATE_BASE: 1858 case AMDGPU::SRC_SHARED_LIMIT: 1859 case AMDGPU::SRC_PRIVATE_LIMIT: 1860 return true; 1861 default: 1862 return false; 1863 } 1864 } 1865